PageRenderTime 66ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 0ms

/demos/appengine/markdown.py

https://github.com/marcelnicolay/tornado
Python | 1235 lines | 1079 code | 49 blank | 107 comment | 56 complexity | 69fce96900e39cc19556e42fd96b30c4 MD5 | raw file
  1. #!/usr/bin/env python
  2. # Copyright (c) 2007-2008 ActiveState Corp.
  3. # License: MIT (http://www.opensource.org/licenses/mit-license.php)
  4. r"""A fast and complete Python implementation of Markdown.
  5. [from http://daringfireball.net/projects/markdown/]
  6. > Markdown is a text-to-HTML filter; it translates an easy-to-read /
  7. > easy-to-write structured text format into HTML. Markdown's text
  8. > format is most similar to that of plain text email, and supports
  9. > features such as headers, *emphasis*, code blocks, blockquotes, and
  10. > links.
  11. >
  12. > Markdown's syntax is designed not as a generic markup language, but
  13. > specifically to serve as a front-end to (X)HTML. You can use span-level
  14. > HTML tags anywhere in a Markdown document, and you can use block level
  15. > HTML tags (like <div> and <table> as well).
  16. Module usage:
  17. >>> import markdown2
  18. >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)`
  19. u'<p><em>boo!</em></p>\n'
  20. >>> markdowner = Markdown()
  21. >>> markdowner.convert("*boo!*")
  22. u'<p><em>boo!</em></p>\n'
  23. >>> markdowner.convert("**boom!**")
  24. u'<p><strong>boom!</strong></p>\n'
  25. This implementation of Markdown implements the full "core" syntax plus a
  26. number of extras (e.g., code syntax coloring, footnotes) as described on
  27. <http://code.google.com/p/python-markdown2/wiki/Extras>.
  28. """
  29. cmdln_desc = """A fast and complete Python implementation of Markdown, a
  30. text-to-HTML conversion tool for web writers.
  31. """
  32. # Dev Notes:
  33. # - There is already a Python markdown processor
  34. # (http://www.freewisdom.org/projects/python-markdown/).
  35. # - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm
  36. # not yet sure if there implications with this. Compare 'pydoc sre'
  37. # and 'perldoc perlre'.
  38. __version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl
  39. __version__ = '1.0.1.14'
  40. __author__ = "Trent Mick"
  41. import os
  42. import sys
  43. from pprint import pprint
  44. import re
  45. import logging
  46. try:
  47. from hashlib import md5
  48. except ImportError:
  49. from md5 import md5
  50. import optparse
  51. from random import random
  52. import codecs
  53. #---- Python version compat
  54. if sys.version_info[:2] < (2,4):
  55. from sets import Set as set
  56. def reversed(sequence):
  57. for i in sequence[::-1]:
  58. yield i
  59. def _unicode_decode(s, encoding, errors='xmlcharrefreplace'):
  60. return unicode(s, encoding, errors)
  61. else:
  62. def _unicode_decode(s, encoding, errors='strict'):
  63. return s.decode(encoding, errors)
  64. #---- globals
  65. DEBUG = False
  66. log = logging.getLogger("markdown")
  67. DEFAULT_TAB_WIDTH = 4
  68. # Table of hash values for escaped characters:
  69. def _escape_hash(s):
  70. # Lame attempt to avoid possible collision with someone actually
  71. # using the MD5 hexdigest of one of these chars in there text.
  72. # Other ideas: random.random(), uuid.uuid()
  73. #return md5(s).hexdigest() # Markdown.pl effectively does this.
  74. return 'md5-'+md5(s).hexdigest()
  75. g_escape_table = dict([(ch, _escape_hash(ch))
  76. for ch in '\\`*_{}[]()>#+-.!'])
  77. #---- exceptions
  78. class MarkdownError(Exception):
  79. pass
  80. #---- public api
  81. def markdown_path(path, encoding="utf-8",
  82. html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
  83. safe_mode=None, extras=None, link_patterns=None,
  84. use_file_vars=False):
  85. text = codecs.open(path, 'r', encoding).read()
  86. return Markdown(html4tags=html4tags, tab_width=tab_width,
  87. safe_mode=safe_mode, extras=extras,
  88. link_patterns=link_patterns,
  89. use_file_vars=use_file_vars).convert(text)
  90. def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
  91. safe_mode=None, extras=None, link_patterns=None,
  92. use_file_vars=False):
  93. return Markdown(html4tags=html4tags, tab_width=tab_width,
  94. safe_mode=safe_mode, extras=extras,
  95. link_patterns=link_patterns,
  96. use_file_vars=use_file_vars).convert(text)
  97. class Markdown(object):
  98. # The dict of "extras" to enable in processing -- a mapping of
  99. # extra name to argument for the extra. Most extras do not have an
  100. # argument, in which case the value is None.
  101. #
  102. # This can be set via (a) subclassing and (b) the constructor
  103. # "extras" argument.
  104. extras = None
  105. urls = None
  106. titles = None
  107. html_blocks = None
  108. html_spans = None
  109. html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py
  110. # Used to track when we're inside an ordered or unordered list
  111. # (see _ProcessListItems() for details):
  112. list_level = 0
  113. _ws_only_line_re = re.compile(r"^[ \t]+$", re.M)
  114. def __init__(self, html4tags=False, tab_width=4, safe_mode=None,
  115. extras=None, link_patterns=None, use_file_vars=False):
  116. if html4tags:
  117. self.empty_element_suffix = ">"
  118. else:
  119. self.empty_element_suffix = " />"
  120. self.tab_width = tab_width
  121. # For compatibility with earlier markdown2.py and with
  122. # markdown.py's safe_mode being a boolean,
  123. # safe_mode == True -> "replace"
  124. if safe_mode is True:
  125. self.safe_mode = "replace"
  126. else:
  127. self.safe_mode = safe_mode
  128. if self.extras is None:
  129. self.extras = {}
  130. elif not isinstance(self.extras, dict):
  131. self.extras = dict([(e, None) for e in self.extras])
  132. if extras:
  133. if not isinstance(extras, dict):
  134. extras = dict([(e, None) for e in extras])
  135. self.extras.update(extras)
  136. assert isinstance(self.extras, dict)
  137. self._instance_extras = self.extras.copy()
  138. self.link_patterns = link_patterns
  139. self.use_file_vars = use_file_vars
  140. self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M)
  141. def reset(self):
  142. self.urls = {}
  143. self.titles = {}
  144. self.html_blocks = {}
  145. self.html_spans = {}
  146. self.list_level = 0
  147. self.extras = self._instance_extras.copy()
  148. if "footnotes" in self.extras:
  149. self.footnotes = {}
  150. self.footnote_ids = []
  151. def convert(self, text):
  152. """Convert the given text."""
  153. # Main function. The order in which other subs are called here is
  154. # essential. Link and image substitutions need to happen before
  155. # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
  156. # and <img> tags get encoded.
  157. # Clear the global hashes. If we don't clear these, you get conflicts
  158. # from other articles when generating a page which contains more than
  159. # one article (e.g. an index page that shows the N most recent
  160. # articles):
  161. self.reset()
  162. if not isinstance(text, unicode):
  163. #TODO: perhaps shouldn't presume UTF-8 for string input?
  164. text = unicode(text, 'utf-8')
  165. if self.use_file_vars:
  166. # Look for emacs-style file variable hints.
  167. emacs_vars = self._get_emacs_vars(text)
  168. if "markdown-extras" in emacs_vars:
  169. splitter = re.compile("[ ,]+")
  170. for e in splitter.split(emacs_vars["markdown-extras"]):
  171. if '=' in e:
  172. ename, earg = e.split('=', 1)
  173. try:
  174. earg = int(earg)
  175. except ValueError:
  176. pass
  177. else:
  178. ename, earg = e, None
  179. self.extras[ename] = earg
  180. # Standardize line endings:
  181. text = re.sub("\r\n|\r", "\n", text)
  182. # Make sure $text ends with a couple of newlines:
  183. text += "\n\n"
  184. # Convert all tabs to spaces.
  185. text = self._detab(text)
  186. # Strip any lines consisting only of spaces and tabs.
  187. # This makes subsequent regexen easier to write, because we can
  188. # match consecutive blank lines with /\n+/ instead of something
  189. # contorted like /[ \t]*\n+/ .
  190. text = self._ws_only_line_re.sub("", text)
  191. if self.safe_mode:
  192. text = self._hash_html_spans(text)
  193. # Turn block-level HTML blocks into hash entries
  194. text = self._hash_html_blocks(text, raw=True)
  195. # Strip link definitions, store in hashes.
  196. if "footnotes" in self.extras:
  197. # Must do footnotes first because an unlucky footnote defn
  198. # looks like a link defn:
  199. # [^4]: this "looks like a link defn"
  200. text = self._strip_footnote_definitions(text)
  201. text = self._strip_link_definitions(text)
  202. text = self._run_block_gamut(text)
  203. if "footnotes" in self.extras:
  204. text = self._add_footnotes(text)
  205. text = self._unescape_special_chars(text)
  206. if self.safe_mode:
  207. text = self._unhash_html_spans(text)
  208. text += "\n"
  209. return text
  210. _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE)
  211. # This regular expression is intended to match blocks like this:
  212. # PREFIX Local Variables: SUFFIX
  213. # PREFIX mode: Tcl SUFFIX
  214. # PREFIX End: SUFFIX
  215. # Some notes:
  216. # - "[ \t]" is used instead of "\s" to specifically exclude newlines
  217. # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does
  218. # not like anything other than Unix-style line terminators.
  219. _emacs_local_vars_pat = re.compile(r"""^
  220. (?P<prefix>(?:[^\r\n|\n|\r])*?)
  221. [\ \t]*Local\ Variables:[\ \t]*
  222. (?P<suffix>.*?)(?:\r\n|\n|\r)
  223. (?P<content>.*?\1End:)
  224. """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)
  225. def _get_emacs_vars(self, text):
  226. """Return a dictionary of emacs-style local variables.
  227. Parsing is done loosely according to this spec (and according to
  228. some in-practice deviations from this):
  229. http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables
  230. """
  231. emacs_vars = {}
  232. SIZE = pow(2, 13) # 8kB
  233. # Search near the start for a '-*-'-style one-liner of variables.
  234. head = text[:SIZE]
  235. if "-*-" in head:
  236. match = self._emacs_oneliner_vars_pat.search(head)
  237. if match:
  238. emacs_vars_str = match.group(1)
  239. assert '\n' not in emacs_vars_str
  240. emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';')
  241. if s.strip()]
  242. if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]:
  243. # While not in the spec, this form is allowed by emacs:
  244. # -*- Tcl -*-
  245. # where the implied "variable" is "mode". This form
  246. # is only allowed if there are no other variables.
  247. emacs_vars["mode"] = emacs_var_strs[0].strip()
  248. else:
  249. for emacs_var_str in emacs_var_strs:
  250. try:
  251. variable, value = emacs_var_str.strip().split(':', 1)
  252. except ValueError:
  253. log.debug("emacs variables error: malformed -*- "
  254. "line: %r", emacs_var_str)
  255. continue
  256. # Lowercase the variable name because Emacs allows "Mode"
  257. # or "mode" or "MoDe", etc.
  258. emacs_vars[variable.lower()] = value.strip()
  259. tail = text[-SIZE:]
  260. if "Local Variables" in tail:
  261. match = self._emacs_local_vars_pat.search(tail)
  262. if match:
  263. prefix = match.group("prefix")
  264. suffix = match.group("suffix")
  265. lines = match.group("content").splitlines(0)
  266. #print "prefix=%r, suffix=%r, content=%r, lines: %s"\
  267. # % (prefix, suffix, match.group("content"), lines)
  268. # Validate the Local Variables block: proper prefix and suffix
  269. # usage.
  270. for i, line in enumerate(lines):
  271. if not line.startswith(prefix):
  272. log.debug("emacs variables error: line '%s' "
  273. "does not use proper prefix '%s'"
  274. % (line, prefix))
  275. return {}
  276. # Don't validate suffix on last line. Emacs doesn't care,
  277. # neither should we.
  278. if i != len(lines)-1 and not line.endswith(suffix):
  279. log.debug("emacs variables error: line '%s' "
  280. "does not use proper suffix '%s'"
  281. % (line, suffix))
  282. return {}
  283. # Parse out one emacs var per line.
  284. continued_for = None
  285. for line in lines[:-1]: # no var on the last line ("PREFIX End:")
  286. if prefix: line = line[len(prefix):] # strip prefix
  287. if suffix: line = line[:-len(suffix)] # strip suffix
  288. line = line.strip()
  289. if continued_for:
  290. variable = continued_for
  291. if line.endswith('\\'):
  292. line = line[:-1].rstrip()
  293. else:
  294. continued_for = None
  295. emacs_vars[variable] += ' ' + line
  296. else:
  297. try:
  298. variable, value = line.split(':', 1)
  299. except ValueError:
  300. log.debug("local variables error: missing colon "
  301. "in local variables entry: '%s'" % line)
  302. continue
  303. # Do NOT lowercase the variable name, because Emacs only
  304. # allows "mode" (and not "Mode", "MoDe", etc.) in this block.
  305. value = value.strip()
  306. if value.endswith('\\'):
  307. value = value[:-1].rstrip()
  308. continued_for = variable
  309. else:
  310. continued_for = None
  311. emacs_vars[variable] = value
  312. # Unquote values.
  313. for var, val in emacs_vars.items():
  314. if len(val) > 1 and (val.startswith('"') and val.endswith('"')
  315. or val.startswith('"') and val.endswith('"')):
  316. emacs_vars[var] = val[1:-1]
  317. return emacs_vars
  318. # Cribbed from a post by Bart Lateur:
  319. # <http://www.nntp.perl.org/group/perl.macperl.anyperl/154>
  320. _detab_re = re.compile(r'(.*?)\t', re.M)
  321. def _detab_sub(self, match):
  322. g1 = match.group(1)
  323. return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width))
  324. def _detab(self, text):
  325. r"""Remove (leading?) tabs from a file.
  326. >>> m = Markdown()
  327. >>> m._detab("\tfoo")
  328. ' foo'
  329. >>> m._detab(" \tfoo")
  330. ' foo'
  331. >>> m._detab("\t foo")
  332. ' foo'
  333. >>> m._detab(" foo")
  334. ' foo'
  335. >>> m._detab(" foo\n\tbar\tblam")
  336. ' foo\n bar blam'
  337. """
  338. if '\t' not in text:
  339. return text
  340. return self._detab_re.subn(self._detab_sub, text)[0]
  341. _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del'
  342. _strict_tag_block_re = re.compile(r"""
  343. ( # save in \1
  344. ^ # start of line (with re.M)
  345. <(%s) # start tag = \2
  346. \b # word break
  347. (.*\n)*? # any number of lines, minimally matching
  348. </\2> # the matching end tag
  349. [ \t]* # trailing spaces/tabs
  350. (?=\n+|\Z) # followed by a newline or end of document
  351. )
  352. """ % _block_tags_a,
  353. re.X | re.M)
  354. _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math'
  355. _liberal_tag_block_re = re.compile(r"""
  356. ( # save in \1
  357. ^ # start of line (with re.M)
  358. <(%s) # start tag = \2
  359. \b # word break
  360. (.*\n)*? # any number of lines, minimally matching
  361. .*</\2> # the matching end tag
  362. [ \t]* # trailing spaces/tabs
  363. (?=\n+|\Z) # followed by a newline or end of document
  364. )
  365. """ % _block_tags_b,
  366. re.X | re.M)
  367. def _hash_html_block_sub(self, match, raw=False):
  368. html = match.group(1)
  369. if raw and self.safe_mode:
  370. html = self._sanitize_html(html)
  371. key = _hash_text(html)
  372. self.html_blocks[key] = html
  373. return "\n\n" + key + "\n\n"
  374. def _hash_html_blocks(self, text, raw=False):
  375. """Hashify HTML blocks
  376. We only want to do this for block-level HTML tags, such as headers,
  377. lists, and tables. That's because we still want to wrap <p>s around
  378. "paragraphs" that are wrapped in non-block-level tags, such as anchors,
  379. phrase emphasis, and spans. The list of tags we're looking for is
  380. hard-coded.
  381. @param raw {boolean} indicates if these are raw HTML blocks in
  382. the original source. It makes a difference in "safe" mode.
  383. """
  384. if '<' not in text:
  385. return text
  386. # Pass `raw` value into our calls to self._hash_html_block_sub.
  387. hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw)
  388. # First, look for nested blocks, e.g.:
  389. # <div>
  390. # <div>
  391. # tags for inner block must be indented.
  392. # </div>
  393. # </div>
  394. #
  395. # The outermost tags must start at the left margin for this to match, and
  396. # the inner nested divs must be indented.
  397. # We need to do this before the next, more liberal match, because the next
  398. # match will start at the first `<div>` and stop at the first `</div>`.
  399. text = self._strict_tag_block_re.sub(hash_html_block_sub, text)
  400. # Now match more liberally, simply from `\n<tag>` to `</tag>\n`
  401. text = self._liberal_tag_block_re.sub(hash_html_block_sub, text)
  402. # Special case just for <hr />. It was easier to make a special
  403. # case than to make the other regex more complicated.
  404. if "<hr" in text:
  405. _hr_tag_re = _hr_tag_re_from_tab_width(self.tab_width)
  406. text = _hr_tag_re.sub(hash_html_block_sub, text)
  407. # Special case for standalone HTML comments:
  408. if "<!--" in text:
  409. start = 0
  410. while True:
  411. # Delimiters for next comment block.
  412. try:
  413. start_idx = text.index("<!--", start)
  414. except ValueError, ex:
  415. break
  416. try:
  417. end_idx = text.index("-->", start_idx) + 3
  418. except ValueError, ex:
  419. break
  420. # Start position for next comment block search.
  421. start = end_idx
  422. # Validate whitespace before comment.
  423. if start_idx:
  424. # - Up to `tab_width - 1` spaces before start_idx.
  425. for i in range(self.tab_width - 1):
  426. if text[start_idx - 1] != ' ':
  427. break
  428. start_idx -= 1
  429. if start_idx == 0:
  430. break
  431. # - Must be preceded by 2 newlines or hit the start of
  432. # the document.
  433. if start_idx == 0:
  434. pass
  435. elif start_idx == 1 and text[0] == '\n':
  436. start_idx = 0 # to match minute detail of Markdown.pl regex
  437. elif text[start_idx-2:start_idx] == '\n\n':
  438. pass
  439. else:
  440. break
  441. # Validate whitespace after comment.
  442. # - Any number of spaces and tabs.
  443. while end_idx < len(text):
  444. if text[end_idx] not in ' \t':
  445. break
  446. end_idx += 1
  447. # - Must be following by 2 newlines or hit end of text.
  448. if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'):
  449. continue
  450. # Escape and hash (must match `_hash_html_block_sub`).
  451. html = text[start_idx:end_idx]
  452. if raw and self.safe_mode:
  453. html = self._sanitize_html(html)
  454. key = _hash_text(html)
  455. self.html_blocks[key] = html
  456. text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:]
  457. if "xml" in self.extras:
  458. # Treat XML processing instructions and namespaced one-liner
  459. # tags as if they were block HTML tags. E.g., if standalone
  460. # (i.e. are their own paragraph), the following do not get
  461. # wrapped in a <p> tag:
  462. # <?foo bar?>
  463. #
  464. # <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="chapter_1.md"/>
  465. _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width)
  466. text = _xml_oneliner_re.sub(hash_html_block_sub, text)
  467. return text
  468. def _strip_link_definitions(self, text):
  469. # Strips link definitions from text, stores the URLs and titles in
  470. # hash references.
  471. less_than_tab = self.tab_width - 1
  472. # Link defs are in the form:
  473. # [id]: url "optional title"
  474. _link_def_re = re.compile(r"""
  475. ^[ ]{0,%d}\[(.+)\]: # id = \1
  476. [ \t]*
  477. \n? # maybe *one* newline
  478. [ \t]*
  479. <?(.+?)>? # url = \2
  480. [ \t]*
  481. (?:
  482. \n? # maybe one newline
  483. [ \t]*
  484. (?<=\s) # lookbehind for whitespace
  485. ['"(]
  486. ([^\n]*) # title = \3
  487. ['")]
  488. [ \t]*
  489. )? # title is optional
  490. (?:\n+|\Z)
  491. """ % less_than_tab, re.X | re.M | re.U)
  492. return _link_def_re.sub(self._extract_link_def_sub, text)
  493. def _extract_link_def_sub(self, match):
  494. id, url, title = match.groups()
  495. key = id.lower() # Link IDs are case-insensitive
  496. self.urls[key] = self._encode_amps_and_angles(url)
  497. if title:
  498. self.titles[key] = title.replace('"', '&quot;')
  499. return ""
  500. def _extract_footnote_def_sub(self, match):
  501. id, text = match.groups()
  502. text = _dedent(text, skip_first_line=not text.startswith('\n')).strip()
  503. normed_id = re.sub(r'\W', '-', id)
  504. # Ensure footnote text ends with a couple newlines (for some
  505. # block gamut matches).
  506. self.footnotes[normed_id] = text + "\n\n"
  507. return ""
  508. def _strip_footnote_definitions(self, text):
  509. """A footnote definition looks like this:
  510. [^note-id]: Text of the note.
  511. May include one or more indented paragraphs.
  512. Where,
  513. - The 'note-id' can be pretty much anything, though typically it
  514. is the number of the footnote.
  515. - The first paragraph may start on the next line, like so:
  516. [^note-id]:
  517. Text of the note.
  518. """
  519. less_than_tab = self.tab_width - 1
  520. footnote_def_re = re.compile(r'''
  521. ^[ ]{0,%d}\[\^(.+)\]: # id = \1
  522. [ \t]*
  523. ( # footnote text = \2
  524. # First line need not start with the spaces.
  525. (?:\s*.*\n+)
  526. (?:
  527. (?:[ ]{%d} | \t) # Subsequent lines must be indented.
  528. .*\n+
  529. )*
  530. )
  531. # Lookahead for non-space at line-start, or end of doc.
  532. (?:(?=^[ ]{0,%d}\S)|\Z)
  533. ''' % (less_than_tab, self.tab_width, self.tab_width),
  534. re.X | re.M)
  535. return footnote_def_re.sub(self._extract_footnote_def_sub, text)
  536. _hr_res = [
  537. re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M),
  538. re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M),
  539. re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M),
  540. ]
  541. def _run_block_gamut(self, text):
  542. # These are all the transformations that form block-level
  543. # tags like paragraphs, headers, and list items.
  544. text = self._do_headers(text)
  545. # Do Horizontal Rules:
  546. hr = "\n<hr"+self.empty_element_suffix+"\n"
  547. for hr_re in self._hr_res:
  548. text = hr_re.sub(hr, text)
  549. text = self._do_lists(text)
  550. if "pyshell" in self.extras:
  551. text = self._prepare_pyshell_blocks(text)
  552. text = self._do_code_blocks(text)
  553. text = self._do_block_quotes(text)
  554. # We already ran _HashHTMLBlocks() before, in Markdown(), but that
  555. # was to escape raw HTML in the original Markdown source. This time,
  556. # we're escaping the markup we've just created, so that we don't wrap
  557. # <p> tags around block-level tags.
  558. text = self._hash_html_blocks(text)
  559. text = self._form_paragraphs(text)
  560. return text
  561. def _pyshell_block_sub(self, match):
  562. lines = match.group(0).splitlines(0)
  563. _dedentlines(lines)
  564. indent = ' ' * self.tab_width
  565. s = ('\n' # separate from possible cuddled paragraph
  566. + indent + ('\n'+indent).join(lines)
  567. + '\n\n')
  568. return s
  569. def _prepare_pyshell_blocks(self, text):
  570. """Ensure that Python interactive shell sessions are put in
  571. code blocks -- even if not properly indented.
  572. """
  573. if ">>>" not in text:
  574. return text
  575. less_than_tab = self.tab_width - 1
  576. _pyshell_block_re = re.compile(r"""
  577. ^([ ]{0,%d})>>>[ ].*\n # first line
  578. ^(\1.*\S+.*\n)* # any number of subsequent lines
  579. ^\n # ends with a blank line
  580. """ % less_than_tab, re.M | re.X)
  581. return _pyshell_block_re.sub(self._pyshell_block_sub, text)
  582. def _run_span_gamut(self, text):
  583. # These are all the transformations that occur *within* block-level
  584. # tags like paragraphs, headers, and list items.
  585. text = self._do_code_spans(text)
  586. text = self._escape_special_chars(text)
  587. # Process anchor and image tags.
  588. text = self._do_links(text)
  589. # Make links out of things like `<http://example.com/>`
  590. # Must come after _do_links(), because you can use < and >
  591. # delimiters in inline links like [this](<url>).
  592. text = self._do_auto_links(text)
  593. if "link-patterns" in self.extras:
  594. text = self._do_link_patterns(text)
  595. text = self._encode_amps_and_angles(text)
  596. text = self._do_italics_and_bold(text)
  597. # Do hard breaks:
  598. text = re.sub(r" {2,}\n", " <br%s\n" % self.empty_element_suffix, text)
  599. return text
  600. # "Sorta" because auto-links are identified as "tag" tokens.
  601. _sorta_html_tokenize_re = re.compile(r"""
  602. (
  603. # tag
  604. </?
  605. (?:\w+) # tag name
  606. (?:\s+(?:[\w-]+:)?[\w-]+=(?:".*?"|'.*?'))* # attributes
  607. \s*/?>
  608. |
  609. # auto-link (e.g., <http://www.activestate.com/>)
  610. <\w+[^>]*>
  611. |
  612. <!--.*?--> # comment
  613. |
  614. <\?.*?\?> # processing instruction
  615. )
  616. """, re.X)
  617. def _escape_special_chars(self, text):
  618. # Python markdown note: the HTML tokenization here differs from
  619. # that in Markdown.pl, hence the behaviour for subtle cases can
  620. # differ (I believe the tokenizer here does a better job because
  621. # it isn't susceptible to unmatched '<' and '>' in HTML tags).
  622. # Note, however, that '>' is not allowed in an auto-link URL
  623. # here.
  624. escaped = []
  625. is_html_markup = False
  626. for token in self._sorta_html_tokenize_re.split(text):
  627. if is_html_markup:
  628. # Within tags/HTML-comments/auto-links, encode * and _
  629. # so they don't conflict with their use in Markdown for
  630. # italics and strong. We're replacing each such
  631. # character with its corresponding MD5 checksum value;
  632. # this is likely overkill, but it should prevent us from
  633. # colliding with the escape values by accident.
  634. escaped.append(token.replace('*', g_escape_table['*'])
  635. .replace('_', g_escape_table['_']))
  636. else:
  637. escaped.append(self._encode_backslash_escapes(token))
  638. is_html_markup = not is_html_markup
  639. return ''.join(escaped)
  640. def _hash_html_spans(self, text):
  641. # Used for safe_mode.
  642. def _is_auto_link(s):
  643. if ':' in s and self._auto_link_re.match(s):
  644. return True
  645. elif '@' in s and self._auto_email_link_re.match(s):
  646. return True
  647. return False
  648. tokens = []
  649. is_html_markup = False
  650. for token in self._sorta_html_tokenize_re.split(text):
  651. if is_html_markup and not _is_auto_link(token):
  652. sanitized = self._sanitize_html(token)
  653. key = _hash_text(sanitized)
  654. self.html_spans[key] = sanitized
  655. tokens.append(key)
  656. else:
  657. tokens.append(token)
  658. is_html_markup = not is_html_markup
  659. return ''.join(tokens)
  660. def _unhash_html_spans(self, text):
  661. for key, sanitized in self.html_spans.items():
  662. text = text.replace(key, sanitized)
  663. return text
  664. def _sanitize_html(self, s):
  665. if self.safe_mode == "replace":
  666. return self.html_removed_text
  667. elif self.safe_mode == "escape":
  668. replacements = [
  669. ('&', '&amp;'),
  670. ('<', '&lt;'),
  671. ('>', '&gt;'),
  672. ]
  673. for before, after in replacements:
  674. s = s.replace(before, after)
  675. return s
  676. else:
  677. raise MarkdownError("invalid value for 'safe_mode': %r (must be "
  678. "'escape' or 'replace')" % self.safe_mode)
  679. _tail_of_inline_link_re = re.compile(r'''
  680. # Match tail of: [text](/url/) or [text](/url/ "title")
  681. \( # literal paren
  682. [ \t]*
  683. (?P<url> # \1
  684. <.*?>
  685. |
  686. .*?
  687. )
  688. [ \t]*
  689. ( # \2
  690. (['"]) # quote char = \3
  691. (?P<title>.*?)
  692. \3 # matching quote
  693. )? # title is optional
  694. \)
  695. ''', re.X | re.S)
  696. _tail_of_reference_link_re = re.compile(r'''
  697. # Match tail of: [text][id]
  698. [ ]? # one optional space
  699. (?:\n[ ]*)? # one optional newline followed by spaces
  700. \[
  701. (?P<id>.*?)
  702. \]
  703. ''', re.X | re.S)
  704. def _do_links(self, text):
  705. """Turn Markdown link shortcuts into XHTML <a> and <img> tags.
  706. This is a combination of Markdown.pl's _DoAnchors() and
  707. _DoImages(). They are done together because that simplified the
  708. approach. It was necessary to use a different approach than
  709. Markdown.pl because of the lack of atomic matching support in
  710. Python's regex engine used in $g_nested_brackets.
  711. """
  712. MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24
  713. # `anchor_allowed_pos` is used to support img links inside
  714. # anchors, but not anchors inside anchors. An anchor's start
  715. # pos must be `>= anchor_allowed_pos`.
  716. anchor_allowed_pos = 0
  717. curr_pos = 0
  718. while True: # Handle the next link.
  719. # The next '[' is the start of:
  720. # - an inline anchor: [text](url "title")
  721. # - a reference anchor: [text][id]
  722. # - an inline img: ![text](url "title")
  723. # - a reference img: ![text][id]
  724. # - a footnote ref: [^id]
  725. # (Only if 'footnotes' extra enabled)
  726. # - a footnote defn: [^id]: ...
  727. # (Only if 'footnotes' extra enabled) These have already
  728. # been stripped in _strip_footnote_definitions() so no
  729. # need to watch for them.
  730. # - a link definition: [id]: url "title"
  731. # These have already been stripped in
  732. # _strip_link_definitions() so no need to watch for them.
  733. # - not markup: [...anything else...
  734. try:
  735. start_idx = text.index('[', curr_pos)
  736. except ValueError:
  737. break
  738. text_length = len(text)
  739. # Find the matching closing ']'.
  740. # Markdown.pl allows *matching* brackets in link text so we
  741. # will here too. Markdown.pl *doesn't* currently allow
  742. # matching brackets in img alt text -- we'll differ in that
  743. # regard.
  744. bracket_depth = 0
  745. for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL,
  746. text_length)):
  747. ch = text[p]
  748. if ch == ']':
  749. bracket_depth -= 1
  750. if bracket_depth < 0:
  751. break
  752. elif ch == '[':
  753. bracket_depth += 1
  754. else:
  755. # Closing bracket not found within sentinel length.
  756. # This isn't markup.
  757. curr_pos = start_idx + 1
  758. continue
  759. link_text = text[start_idx+1:p]
  760. # Possibly a footnote ref?
  761. if "footnotes" in self.extras and link_text.startswith("^"):
  762. normed_id = re.sub(r'\W', '-', link_text[1:])
  763. if normed_id in self.footnotes:
  764. self.footnote_ids.append(normed_id)
  765. result = '<sup class="footnote-ref" id="fnref-%s">' \
  766. '<a href="#fn-%s">%s</a></sup>' \
  767. % (normed_id, normed_id, len(self.footnote_ids))
  768. text = text[:start_idx] + result + text[p+1:]
  769. else:
  770. # This id isn't defined, leave the markup alone.
  771. curr_pos = p+1
  772. continue
  773. # Now determine what this is by the remainder.
  774. p += 1
  775. if p == text_length:
  776. return text
  777. # Inline anchor or img?
  778. if text[p] == '(': # attempt at perf improvement
  779. match = self._tail_of_inline_link_re.match(text, p)
  780. if match:
  781. # Handle an inline anchor or img.
  782. is_img = start_idx > 0 and text[start_idx-1] == "!"
  783. if is_img:
  784. start_idx -= 1
  785. url, title = match.group("url"), match.group("title")
  786. if url and url[0] == '<':
  787. url = url[1:-1] # '<url>' -> 'url'
  788. # We've got to encode these to avoid conflicting
  789. # with italics/bold.
  790. url = url.replace('*', g_escape_table['*']) \
  791. .replace('_', g_escape_table['_'])
  792. if title:
  793. title_str = ' title="%s"' \
  794. % title.replace('*', g_escape_table['*']) \
  795. .replace('_', g_escape_table['_']) \
  796. .replace('"', '&quot;')
  797. else:
  798. title_str = ''
  799. if is_img:
  800. result = '<img src="%s" alt="%s"%s%s' \
  801. % (url, link_text.replace('"', '&quot;'),
  802. title_str, self.empty_element_suffix)
  803. curr_pos = start_idx + len(result)
  804. text = text[:start_idx] + result + text[match.end():]
  805. elif start_idx >= anchor_allowed_pos:
  806. result_head = '<a href="%s"%s>' % (url, title_str)
  807. result = '%s%s</a>' % (result_head, link_text)
  808. # <img> allowed from curr_pos on, <a> from
  809. # anchor_allowed_pos on.
  810. curr_pos = start_idx + len(result_head)
  811. anchor_allowed_pos = start_idx + len(result)
  812. text = text[:start_idx] + result + text[match.end():]
  813. else:
  814. # Anchor not allowed here.
  815. curr_pos = start_idx + 1
  816. continue
  817. # Reference anchor or img?
  818. else:
  819. match = self._tail_of_reference_link_re.match(text, p)
  820. if match:
  821. # Handle a reference-style anchor or img.
  822. is_img = start_idx > 0 and text[start_idx-1] == "!"
  823. if is_img:
  824. start_idx -= 1
  825. link_id = match.group("id").lower()
  826. if not link_id:
  827. link_id = link_text.lower() # for links like [this][]
  828. if link_id in self.urls:
  829. url = self.urls[link_id]
  830. # We've got to encode these to avoid conflicting
  831. # with italics/bold.
  832. url = url.replace('*', g_escape_table['*']) \
  833. .replace('_', g_escape_table['_'])
  834. title = self.titles.get(link_id)
  835. if title:
  836. title = title.replace('*', g_escape_table['*']) \
  837. .replace('_', g_escape_table['_'])
  838. title_str = ' title="%s"' % title
  839. else:
  840. title_str = ''
  841. if is_img:
  842. result = '<img src="%s" alt="%s"%s%s' \
  843. % (url, link_text.replace('"', '&quot;'),
  844. title_str, self.empty_element_suffix)
  845. curr_pos = start_idx + len(result)
  846. text = text[:start_idx] + result + text[match.end():]
  847. elif start_idx >= anchor_allowed_pos:
  848. result = '<a href="%s"%s>%s</a>' \
  849. % (url, title_str, link_text)
  850. result_head = '<a href="%s"%s>' % (url, title_str)
  851. result = '%s%s</a>' % (result_head, link_text)
  852. # <img> allowed from curr_pos on, <a> from
  853. # anchor_allowed_pos on.
  854. curr_pos = start_idx + len(result_head)
  855. anchor_allowed_pos = start_idx + len(result)
  856. text = text[:start_idx] + result + text[match.end():]
  857. else:
  858. # Anchor not allowed here.
  859. curr_pos = start_idx + 1
  860. else:
  861. # This id isn't defined, leave the markup alone.
  862. curr_pos = match.end()
  863. continue
  864. # Otherwise, it isn't markup.
  865. curr_pos = start_idx + 1
  866. return text
  867. _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M)
  868. def _setext_h_sub(self, match):
  869. n = {"=": 1, "-": 2}[match.group(2)[0]]
  870. demote_headers = self.extras.get("demote-headers")
  871. if demote_headers:
  872. n = min(n + demote_headers, 6)
  873. return "<h%d>%s</h%d>\n\n" \
  874. % (n, self._run_span_gamut(match.group(1)), n)
  875. _atx_h_re = re.compile(r'''
  876. ^(\#{1,6}) # \1 = string of #'s
  877. [ \t]*
  878. (.+?) # \2 = Header text
  879. [ \t]*
  880. (?<!\\) # ensure not an escaped trailing '#'
  881. \#* # optional closing #'s (not counted)
  882. \n+
  883. ''', re.X | re.M)
  884. def _atx_h_sub(self, match):
  885. n = len(match.group(1))
  886. demote_headers = self.extras.get("demote-headers")
  887. if demote_headers:
  888. n = min(n + demote_headers, 6)
  889. return "<h%d>%s</h%d>\n\n" \
  890. % (n, self._run_span_gamut(match.group(2)), n)
  891. def _do_headers(self, text):
  892. # Setext-style headers:
  893. # Header 1
  894. # ========
  895. #
  896. # Header 2
  897. # --------
  898. text = self._setext_h_re.sub(self._setext_h_sub, text)
  899. # atx-style headers:
  900. # # Header 1
  901. # ## Header 2
  902. # ## Header 2 with closing hashes ##
  903. # ...
  904. # ###### Header 6
  905. text = self._atx_h_re.sub(self._atx_h_sub, text)
  906. return text
  907. _marker_ul_chars = '*+-'
  908. _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars
  909. _marker_ul = '(?:[%s])' % _marker_ul_chars
  910. _marker_ol = r'(?:\d+\.)'
  911. def _list_sub(self, match):
  912. lst = match.group(1)
  913. lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol"
  914. result = self._process_list_items(lst)
  915. if self.list_level:
  916. return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type)
  917. else:
  918. return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type)
  919. def _do_lists(self, text):
  920. # Form HTML ordered (numbered) and unordered (bulleted) lists.
  921. for marker_pat in (self._marker_ul, self._marker_ol):
  922. # Re-usable pattern to match any entire ul or ol list:
  923. less_than_tab = self.tab_width - 1
  924. whole_list = r'''
  925. ( # \1 = whole list
  926. ( # \2
  927. [ ]{0,%d}
  928. (%s) # \3 = first list item marker
  929. [ \t]+
  930. )
  931. (?:.+?)
  932. ( # \4
  933. \Z
  934. |
  935. \n{2,}
  936. (?=\S)
  937. (?! # Negative lookahead for another list item marker
  938. [ \t]*
  939. %s[ \t]+
  940. )
  941. )
  942. )
  943. ''' % (less_than_tab, marker_pat, marker_pat)
  944. # We use a different prefix before nested lists than top-level lists.
  945. # See extended comment in _process_list_items().
  946. #
  947. # Note: There's a bit of duplication here. My original implementation
  948. # created a scalar regex pattern as the conditional result of the test on
  949. # $g_list_level, and then only ran the $text =~ s{...}{...}egmx
  950. # substitution once, using the scalar as the pattern. This worked,
  951. # everywhere except when running under MT on my hosting account at Pair
  952. # Networks. There, this caused all rebuilds to be killed by the reaper (or
  953. # perhaps they crashed, but that seems incredibly unlikely given that the
  954. # same script on the same server ran fine *except* under MT. I've spent
  955. # more time trying to figure out why this is happening than I'd like to
  956. # admit. My only guess, backed up by the fact that this workaround works,
  957. # is that Perl optimizes the substition when it can figure out that the
  958. # pattern will never change, and when this optimization isn't on, we run
  959. # afoul of the reaper. Thus, the slightly redundant code to that uses two
  960. # static s/// patterns rather than one conditional pattern.
  961. if self.list_level:
  962. sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S)
  963. text = sub_list_re.sub(self._list_sub, text)
  964. else:
  965. list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list,
  966. re.X | re.M | re.S)
  967. text = list_re.sub(self._list_sub, text)
  968. return text
  969. _list_item_re = re.compile(r'''
  970. (\n)? # leading line = \1
  971. (^[ \t]*) # leading whitespace = \2
  972. (%s) [ \t]+ # list marker = \3
  973. ((?:.+?) # list item text = \4
  974. (\n{1,2})) # eols = \5
  975. (?= \n* (\Z | \2 (%s) [ \t]+))
  976. ''' % (_marker_any, _marker_any),
  977. re.M | re.X | re.S)
  978. _last_li_endswith_two_eols = False
  979. def _list_item_sub(self, match):
  980. item = match.group(4)
  981. leading_line = match.group(1)
  982. leading_space = match.group(2)
  983. if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
  984. item = self._run_block_gamut(self._outdent(item))
  985. else:
  986. # Recursion for sub-lists:
  987. item = self._do_lists(self._outdent(item))
  988. if item.endswith('\n'):
  989. item = item[:-1]
  990. item = self._run_span_gamut(item)
  991. self._last_li_endswith_two_eols = (len(match.group(5)) == 2)
  992. return "<li>%s</li>\n" % item
  993. def _process_list_items(self, list_str):
  994. # Process the contents of a single ordered or unordered list,
  995. # splitting it into individual list items.
  996. # The $g_list_level global keeps track of when we're inside a list.
  997. # Each time we enter a list, we increment it; when we leave a list,
  998. # we decrement. If it's zero, we're not in a list anymore.
  999. #
  1000. # We do this because when we're not inside a list, we want to treat
  1001. # something like this:
  1002. #
  1003. # I recommend upgrading to version
  1004. # 8. Oops, now this line is treated
  1005. # as a sub-list.
  1006. #
  1007. # As a single paragraph, despite the fact that the second line starts
  1008. # with a digit-period-space sequence.
  1009. #
  1010. # Whereas when we're inside a list (or sub-list), that line will be
  1011. # treated as the start of a sub-list. What a kludge, huh? This is
  1012. # an aspect of Markdown's syntax that's hard to parse perfectly
  1013. # without resorting to mind-reading. Perhaps the solution is to
  1014. # change the syntax rules such that sub-lists must start with a
  1015. # starting cardinal number; e.g. "1." or "a.".
  1016. self.list_level += 1
  1017. self._last_li_endswith_two_eols = False
  1018. list_str = list_str.rstrip('\n') + '\n'
  1019. list_str = self._list_item_re.sub(self._list_item_sub, list_str)
  1020. self.list_level -= 1
  1021. return list_str
  1022. def _get_pygments_lexer(self, lexer_name):
  1023. try:
  1024. from pygments import lexers, util
  1025. except ImportError:
  1026. return None
  1027. try:
  1028. return lexers.get_lexer_by_name(lexer_name)
  1029. except util.ClassNotFound:
  1030. return None
  1031. def _color_with_pygments(self, codeblock, lexer, **formatter_opts):
  1032. import pygments
  1033. import pygments.formatters
  1034. class HtmlCodeFormatter(pygments.formatters.HtmlFormatter):
  1035. def _wrap_code(self, inner):
  1036. """A function for use in a Pygments Formatter which
  1037. wraps in <code> tags.
  1038. """
  1039. yield 0, "<code>"
  1040. for tup in inner:
  1041. yield tup
  1042. yield 0, "</code>"
  1043. def wrap(self, source, outfile):
  1044. """Return the source with a code, pre, and div."""
  1045. return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
  1046. formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts)
  1047. return pygments.highlight(codeblock, lexer, formatter)
  1048. def _code_block_sub(self, match):
  1049. codeblock = match.group(1)
  1050. codeblock = self._outdent(codeblock)
  1051. codeblock = self._detab(codeblock)
  1052. codeblock = codeblock.lstrip('\n') # trim leading newlines
  1053. codeblock = codeblock.rstrip() # trim trailing whitespace
  1054. if "code-color" in self.extras and codeblock.startswith(":::"):
  1055. lexer_name, rest = codeblock.split('\n', 1)
  1056. lexer_name = lexer_name[3:].strip()
  1057. lexer = self._get_pygments_lexer(lexer_name)
  1058. codeblock = rest.lstrip("\n") # Remove lexer declaration line.
  1059. if lexer:
  1060. formatter_opts = self.extras['code-color'] or {}
  1061. colored = self._color_with_pygments(codeblock, lexer,
  1062. **formatter_opts)
  1063. return "\n\n%s\n\n" % colored
  1064. codeblock = self._encode_code(codeblock)
  1065. return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock
  1066. def _do_code_blocks(self, text):
  1067. """Process Markdown `<pre><code>` blocks."""
  1068. code_block_re = re.compile(r'''
  1069. (?:\n\n|\A)
  1070. ( # $1 = the code block -- one or more lines, starting with a space/tab
  1071. (?:
  1072. (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces
  1073. .*\n