PageRenderTime 63ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 0ms

/markdown2.py

https://github.com/jrk/md2html
Python | 1235 lines | 1126 code | 35 blank | 74 comment | 49 complexity | ebd0d5cf2b3af5ed3cf72423b0b771d2 MD5 | raw file
  1. #!/usr/bin/env python
  2. # Copyright (c) 2007-2008 ActiveState Corp.
  3. # License: MIT (http://www.opensource.org/licenses/mit-license.php)
  4. r"""A fast and complete Python implementation of Markdown.
  5. [from http://daringfireball.net/projects/markdown/]
  6. > Markdown is a text-to-HTML filter; it translates an easy-to-read /
  7. > easy-to-write structured text format into HTML. Markdown's text
  8. > format is most similar to that of plain text email, and supports
  9. > features such as headers, *emphasis*, code blocks, blockquotes, and
  10. > links.
  11. >
  12. > Markdown's syntax is designed not as a generic markup language, but
  13. > specifically to serve as a front-end to (X)HTML. You can use span-level
  14. > HTML tags anywhere in a Markdown document, and you can use block level
  15. > HTML tags (like <div> and <table> as well).
  16. Module usage:
  17. >>> import markdown2
  18. >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)`
  19. u'<p><em>boo!</em></p>\n'
  20. >>> markdowner = Markdown()
  21. >>> markdowner.convert("*boo!*")
  22. u'<p><em>boo!</em></p>\n'
  23. >>> markdowner.convert("**boom!**")
  24. u'<p><strong>boom!</strong></p>\n'
  25. This implementation of Markdown implements the full "core" syntax plus a
  26. number of extras (e.g., code syntax coloring, footnotes) as described on
  27. <http://code.google.com/p/python-markdown2/wiki/Extras>.
  28. """
  29. cmdln_desc = """A fast and complete Python implementation of Markdown, a
  30. text-to-HTML conversion tool for web writers.
  31. Supported extras (see -x|--extras option below):
  32. * code-friendly: Disable _ and __ for em and strong.
  33. * code-color: Pygments-based syntax coloring of <code> sections.
  34. * cuddled-lists: Allow lists to be cuddled to the preceding paragraph.
  35. * footnotes: Support footnotes as in use on daringfireball.net and
  36. implemented in other Markdown processors (tho not in Markdown.pl v1.0.1).
  37. * html-classes: Takes a dict mapping html tag names (lowercase) to a
  38. string to use for a "class" tag attribute. Currently only supports
  39. "pre" and "code" tags. Add an issue if you require this for other tags.
  40. * pyshell: Treats unindented Python interactive shell sessions as <code>
  41. blocks.
  42. * link-patterns: Auto-link given regex patterns in text (e.g. bug number
  43. references, revision number references).
  44. * xml: Passes one-liner processing instructions and namespaced XML tags.
  45. """
  46. # Dev Notes:
  47. # - There is already a Python markdown processor
  48. # (http://www.freewisdom.org/projects/python-markdown/).
  49. # - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm
  50. # not yet sure if there implications with this. Compare 'pydoc sre'
  51. # and 'perldoc perlre'.
  52. __version_info__ = (1, 0, 1, 17) # first three nums match Markdown.pl
  53. __version__ = '1.0.1.17'
  54. __author__ = "Trent Mick"
  55. import os
  56. import sys
  57. from pprint import pprint
  58. import re
  59. import logging
  60. try:
  61. from hashlib import md5
  62. except ImportError:
  63. from md5 import md5
  64. import optparse
  65. from random import random, randint
  66. import codecs
  67. from urllib import quote
  68. #---- Python version compat
  69. if sys.version_info[:2] < (2,4):
  70. from sets import Set as set
  71. def reversed(sequence):
  72. for i in sequence[::-1]:
  73. yield i
  74. def _unicode_decode(s, encoding, errors='xmlcharrefreplace'):
  75. return unicode(s, encoding, errors)
  76. else:
  77. def _unicode_decode(s, encoding, errors='strict'):
  78. return s.decode(encoding, errors)
  79. #---- globals
  80. DEBUG = False
  81. log = logging.getLogger("markdown")
  82. DEFAULT_TAB_WIDTH = 4
  83. try:
  84. import uuid
  85. except ImportError:
  86. SECRET_SALT = str(randint(0, 1000000))
  87. else:
  88. SECRET_SALT = str(uuid.uuid4())
  89. def _hash_ascii(s):
  90. #return md5(s).hexdigest() # Markdown.pl effectively does this.
  91. return 'md5-' + md5(SECRET_SALT + s).hexdigest()
  92. def _hash_text(s):
  93. return 'md5-' + md5(SECRET_SALT + s.encode("utf-8")).hexdigest()
  94. # Table of hash values for escaped characters:
  95. g_escape_table = dict([(ch, _hash_ascii(ch))
  96. for ch in '\\`*_{}[]()>#+-.!'])
  97. #---- exceptions
  98. class MarkdownError(Exception):
  99. pass
  100. #---- public api
  101. def markdown_path(path, encoding="utf-8",
  102. html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
  103. safe_mode=None, extras=None, link_patterns=None,
  104. use_file_vars=False):
  105. fp = codecs.open(path, 'r', encoding)
  106. text = fp.read()
  107. fp.close()
  108. return Markdown(html4tags=html4tags, tab_width=tab_width,
  109. safe_mode=safe_mode, extras=extras,
  110. link_patterns=link_patterns,
  111. use_file_vars=use_file_vars).convert(text)
  112. def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH,
  113. safe_mode=None, extras=None, link_patterns=None,
  114. use_file_vars=False):
  115. return Markdown(html4tags=html4tags, tab_width=tab_width,
  116. safe_mode=safe_mode, extras=extras,
  117. link_patterns=link_patterns,
  118. use_file_vars=use_file_vars).convert(text)
  119. class Markdown(object):
  120. # The dict of "extras" to enable in processing -- a mapping of
  121. # extra name to argument for the extra. Most extras do not have an
  122. # argument, in which case the value is None.
  123. #
  124. # This can be set via (a) subclassing and (b) the constructor
  125. # "extras" argument.
  126. extras = None
  127. urls = None
  128. titles = None
  129. html_blocks = None
  130. html_spans = None
  131. html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py
  132. # Used to track when we're inside an ordered or unordered list
  133. # (see _ProcessListItems() for details):
  134. list_level = 0
  135. _ws_only_line_re = re.compile(r"^[ \t]+$", re.M)
  136. def __init__(self, html4tags=False, tab_width=4, safe_mode=None,
  137. extras=None, link_patterns=None, use_file_vars=False):
  138. if html4tags:
  139. self.empty_element_suffix = ">"
  140. else:
  141. self.empty_element_suffix = " />"
  142. self.tab_width = tab_width
  143. # For compatibility with earlier markdown2.py and with
  144. # markdown.py's safe_mode being a boolean,
  145. # safe_mode == True -> "replace"
  146. if safe_mode is True:
  147. self.safe_mode = "replace"
  148. else:
  149. self.safe_mode = safe_mode
  150. if self.extras is None:
  151. self.extras = {}
  152. elif not isinstance(self.extras, dict):
  153. self.extras = dict([(e, None) for e in self.extras])
  154. if extras:
  155. if not isinstance(extras, dict):
  156. extras = dict([(e, None) for e in extras])
  157. self.extras.update(extras)
  158. assert isinstance(self.extras, dict)
  159. if "toc" in self.extras and not "header-ids" in self.extras:
  160. self.extras["header-ids"] = None # "toc" implies "header-ids"
  161. self._instance_extras = self.extras.copy()
  162. self.link_patterns = link_patterns
  163. self.use_file_vars = use_file_vars
  164. self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M)
  165. def reset(self):
  166. self.urls = {}
  167. self.titles = {}
  168. self.html_blocks = {}
  169. self.html_spans = {}
  170. self.list_level = 0
  171. self.extras = self._instance_extras.copy()
  172. if "footnotes" in self.extras:
  173. self.footnotes = {}
  174. self.footnote_ids = []
  175. if "header-ids" in self.extras:
  176. self._count_from_header_id = {} # no `defaultdict` in Python 2.4
  177. def convert(self, text):
  178. """Convert the given text."""
  179. # Main function. The order in which other subs are called here is
  180. # essential. Link and image substitutions need to happen before
  181. # _EscapeSpecialChars(), so that any *'s or _'s in the <a>
  182. # and <img> tags get encoded.
  183. # Clear the global hashes. If we don't clear these, you get conflicts
  184. # from other articles when generating a page which contains more than
  185. # one article (e.g. an index page that shows the N most recent
  186. # articles):
  187. self.reset()
  188. if not isinstance(text, unicode):
  189. #TODO: perhaps shouldn't presume UTF-8 for string input?
  190. text = unicode(text, 'utf-8')
  191. if self.use_file_vars:
  192. # Look for emacs-style file variable hints.
  193. emacs_vars = self._get_emacs_vars(text)
  194. if "markdown-extras" in emacs_vars:
  195. splitter = re.compile("[ ,]+")
  196. for e in splitter.split(emacs_vars["markdown-extras"]):
  197. if '=' in e:
  198. ename, earg = e.split('=', 1)
  199. try:
  200. earg = int(earg)
  201. except ValueError:
  202. pass
  203. else:
  204. ename, earg = e, None
  205. self.extras[ename] = earg
  206. # Standardize line endings:
  207. text = re.sub("\r\n|\r", "\n", text)
  208. # Make sure $text ends with a couple of newlines:
  209. text += "\n\n"
  210. # Convert all tabs to spaces.
  211. text = self._detab(text)
  212. # Strip any lines consisting only of spaces and tabs.
  213. # This makes subsequent regexen easier to write, because we can
  214. # match consecutive blank lines with /\n+/ instead of something
  215. # contorted like /[ \t]*\n+/ .
  216. text = self._ws_only_line_re.sub("", text)
  217. if self.safe_mode:
  218. text = self._hash_html_spans(text)
  219. # Turn block-level HTML blocks into hash entries
  220. text = self._hash_html_blocks(text, raw=True)
  221. # Strip link definitions, store in hashes.
  222. if "footnotes" in self.extras:
  223. # Must do footnotes first because an unlucky footnote defn
  224. # looks like a link defn:
  225. # [^4]: this "looks like a link defn"
  226. text = self._strip_footnote_definitions(text)
  227. text = self._strip_link_definitions(text)
  228. text = self._run_block_gamut(text)
  229. if "footnotes" in self.extras:
  230. text = self._add_footnotes(text)
  231. text = self._unescape_special_chars(text)
  232. if self.safe_mode:
  233. text = self._unhash_html_spans(text)
  234. text += "\n"
  235. rv = UnicodeWithAttrs(text)
  236. if "toc" in self.extras:
  237. rv._toc = self._toc
  238. return rv
  239. _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE)
  240. # This regular expression is intended to match blocks like this:
  241. # PREFIX Local Variables: SUFFIX
  242. # PREFIX mode: Tcl SUFFIX
  243. # PREFIX End: SUFFIX
  244. # Some notes:
  245. # - "[ \t]" is used instead of "\s" to specifically exclude newlines
  246. # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does
  247. # not like anything other than Unix-style line terminators.
  248. _emacs_local_vars_pat = re.compile(r"""^
  249. (?P<prefix>(?:[^\r\n|\n|\r])*?)
  250. [\ \t]*Local\ Variables:[\ \t]*
  251. (?P<suffix>.*?)(?:\r\n|\n|\r)
  252. (?P<content>.*?\1End:)
  253. """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE)
  254. def _get_emacs_vars(self, text):
  255. """Return a dictionary of emacs-style local variables.
  256. Parsing is done loosely according to this spec (and according to
  257. some in-practice deviations from this):
  258. http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables
  259. """
  260. emacs_vars = {}
  261. SIZE = pow(2, 13) # 8kB
  262. # Search near the start for a '-*-'-style one-liner of variables.
  263. head = text[:SIZE]
  264. if "-*-" in head:
  265. match = self._emacs_oneliner_vars_pat.search(head)
  266. if match:
  267. emacs_vars_str = match.group(1)
  268. assert '\n' not in emacs_vars_str
  269. emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';')
  270. if s.strip()]
  271. if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]:
  272. # While not in the spec, this form is allowed by emacs:
  273. # -*- Tcl -*-
  274. # where the implied "variable" is "mode". This form
  275. # is only allowed if there are no other variables.
  276. emacs_vars["mode"] = emacs_var_strs[0].strip()
  277. else:
  278. for emacs_var_str in emacs_var_strs:
  279. try:
  280. variable, value = emacs_var_str.strip().split(':', 1)
  281. except ValueError:
  282. log.debug("emacs variables error: malformed -*- "
  283. "line: %r", emacs_var_str)
  284. continue
  285. # Lowercase the variable name because Emacs allows "Mode"
  286. # or "mode" or "MoDe", etc.
  287. emacs_vars[variable.lower()] = value.strip()
  288. tail = text[-SIZE:]
  289. if "Local Variables" in tail:
  290. match = self._emacs_local_vars_pat.search(tail)
  291. if match:
  292. prefix = match.group("prefix")
  293. suffix = match.group("suffix")
  294. lines = match.group("content").splitlines(0)
  295. #print "prefix=%r, suffix=%r, content=%r, lines: %s"\
  296. # % (prefix, suffix, match.group("content"), lines)
  297. # Validate the Local Variables block: proper prefix and suffix
  298. # usage.
  299. for i, line in enumerate(lines):
  300. if not line.startswith(prefix):
  301. log.debug("emacs variables error: line '%s' "
  302. "does not use proper prefix '%s'"
  303. % (line, prefix))
  304. return {}
  305. # Don't validate suffix on last line. Emacs doesn't care,
  306. # neither should we.
  307. if i != len(lines)-1 and not line.endswith(suffix):
  308. log.debug("emacs variables error: line '%s' "
  309. "does not use proper suffix '%s'"
  310. % (line, suffix))
  311. return {}
  312. # Parse out one emacs var per line.
  313. continued_for = None
  314. for line in lines[:-1]: # no var on the last line ("PREFIX End:")
  315. if prefix: line = line[len(prefix):] # strip prefix
  316. if suffix: line = line[:-len(suffix)] # strip suffix
  317. line = line.strip()
  318. if continued_for:
  319. variable = continued_for
  320. if line.endswith('\\'):
  321. line = line[:-1].rstrip()
  322. else:
  323. continued_for = None
  324. emacs_vars[variable] += ' ' + line
  325. else:
  326. try:
  327. variable, value = line.split(':', 1)
  328. except ValueError:
  329. log.debug("local variables error: missing colon "
  330. "in local variables entry: '%s'" % line)
  331. continue
  332. # Do NOT lowercase the variable name, because Emacs only
  333. # allows "mode" (and not "Mode", "MoDe", etc.) in this block.
  334. value = value.strip()
  335. if value.endswith('\\'):
  336. value = value[:-1].rstrip()
  337. continued_for = variable
  338. else:
  339. continued_for = None
  340. emacs_vars[variable] = value
  341. # Unquote values.
  342. for var, val in emacs_vars.items():
  343. if len(val) > 1 and (val.startswith('"') and val.endswith('"')
  344. or val.startswith('"') and val.endswith('"')):
  345. emacs_vars[var] = val[1:-1]
  346. return emacs_vars
  347. # Cribbed from a post by Bart Lateur:
  348. # <http://www.nntp.perl.org/group/perl.macperl.anyperl/154>
  349. _detab_re = re.compile(r'(.*?)\t', re.M)
  350. def _detab_sub(self, match):
  351. g1 = match.group(1)
  352. return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width))
  353. def _detab(self, text):
  354. r"""Remove (leading?) tabs from a file.
  355. >>> m = Markdown()
  356. >>> m._detab("\tfoo")
  357. ' foo'
  358. >>> m._detab(" \tfoo")
  359. ' foo'
  360. >>> m._detab("\t foo")
  361. ' foo'
  362. >>> m._detab(" foo")
  363. ' foo'
  364. >>> m._detab(" foo\n\tbar\tblam")
  365. ' foo\n bar blam'
  366. """
  367. if '\t' not in text:
  368. return text
  369. return self._detab_re.subn(self._detab_sub, text)[0]
  370. _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del'
  371. _strict_tag_block_re = re.compile(r"""
  372. ( # save in \1
  373. ^ # start of line (with re.M)
  374. <(%s) # start tag = \2
  375. \b # word break
  376. (.*\n)*? # any number of lines, minimally matching
  377. </\2> # the matching end tag
  378. [ \t]* # trailing spaces/tabs
  379. (?=\n+|\Z) # followed by a newline or end of document
  380. )
  381. """ % _block_tags_a,
  382. re.X | re.M)
  383. _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math'
  384. _liberal_tag_block_re = re.compile(r"""
  385. ( # save in \1
  386. ^ # start of line (with re.M)
  387. <(%s) # start tag = \2
  388. \b # word break
  389. (.*\n)*? # any number of lines, minimally matching
  390. .*</\2> # the matching end tag
  391. [ \t]* # trailing spaces/tabs
  392. (?=\n+|\Z) # followed by a newline or end of document
  393. )
  394. """ % _block_tags_b,
  395. re.X | re.M)
  396. def _hash_html_block_sub(self, match, raw=False):
  397. html = match.group(1)
  398. if raw and self.safe_mode:
  399. html = self._sanitize_html(html)
  400. key = _hash_text(html)
  401. self.html_blocks[key] = html
  402. return "\n\n" + key + "\n\n"
  403. def _hash_html_blocks(self, text, raw=False):
  404. """Hashify HTML blocks
  405. We only want to do this for block-level HTML tags, such as headers,
  406. lists, and tables. That's because we still want to wrap <p>s around
  407. "paragraphs" that are wrapped in non-block-level tags, such as anchors,
  408. phrase emphasis, and spans. The list of tags we're looking for is
  409. hard-coded.
  410. @param raw {boolean} indicates if these are raw HTML blocks in
  411. the original source. It makes a difference in "safe" mode.
  412. """
  413. if '<' not in text:
  414. return text
  415. # Pass `raw` value into our calls to self._hash_html_block_sub.
  416. hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw)
  417. # First, look for nested blocks, e.g.:
  418. # <div>
  419. # <div>
  420. # tags for inner block must be indented.
  421. # </div>
  422. # </div>
  423. #
  424. # The outermost tags must start at the left margin for this to match, and
  425. # the inner nested divs must be indented.
  426. # We need to do this before the next, more liberal match, because the next
  427. # match will start at the first `<div>` and stop at the first `</div>`.
  428. text = self._strict_tag_block_re.sub(hash_html_block_sub, text)
  429. # Now match more liberally, simply from `\n<tag>` to `</tag>\n`
  430. text = self._liberal_tag_block_re.sub(hash_html_block_sub, text)
  431. # Special case just for <hr />. It was easier to make a special
  432. # case than to make the other regex more complicated.
  433. if "<hr" in text:
  434. _hr_tag_re = _hr_tag_re_from_tab_width(self.tab_width)
  435. text = _hr_tag_re.sub(hash_html_block_sub, text)
  436. # Special case for standalone HTML comments:
  437. if "<!--" in text:
  438. start = 0
  439. while True:
  440. # Delimiters for next comment block.
  441. try:
  442. start_idx = text.index("<!--", start)
  443. except ValueError, ex:
  444. break
  445. try:
  446. end_idx = text.index("-->", start_idx) + 3
  447. except ValueError, ex:
  448. break
  449. # Start position for next comment block search.
  450. start = end_idx
  451. # Validate whitespace before comment.
  452. if start_idx:
  453. # - Up to `tab_width - 1` spaces before start_idx.
  454. for i in range(self.tab_width - 1):
  455. if text[start_idx - 1] != ' ':
  456. break
  457. start_idx -= 1
  458. if start_idx == 0:
  459. break
  460. # - Must be preceded by 2 newlines or hit the start of
  461. # the document.
  462. if start_idx == 0:
  463. pass
  464. elif start_idx == 1 and text[0] == '\n':
  465. start_idx = 0 # to match minute detail of Markdown.pl regex
  466. elif text[start_idx-2:start_idx] == '\n\n':
  467. pass
  468. else:
  469. break
  470. # Validate whitespace after comment.
  471. # - Any number of spaces and tabs.
  472. while end_idx < len(text):
  473. if text[end_idx] not in ' \t':
  474. break
  475. end_idx += 1
  476. # - Must be following by 2 newlines or hit end of text.
  477. if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'):
  478. continue
  479. # Escape and hash (must match `_hash_html_block_sub`).
  480. html = text[start_idx:end_idx]
  481. if raw and self.safe_mode:
  482. html = self._sanitize_html(html)
  483. key = _hash_text(html)
  484. self.html_blocks[key] = html
  485. text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:]
  486. if "xml" in self.extras:
  487. # Treat XML processing instructions and namespaced one-liner
  488. # tags as if they were block HTML tags. E.g., if standalone
  489. # (i.e. are their own paragraph), the following do not get
  490. # wrapped in a <p> tag:
  491. # <?foo bar?>
  492. #
  493. # <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="chapter_1.md"/>
  494. _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width)
  495. text = _xml_oneliner_re.sub(hash_html_block_sub, text)
  496. return text
  497. def _strip_link_definitions(self, text):
  498. # Strips link definitions from text, stores the URLs and titles in
  499. # hash references.
  500. less_than_tab = self.tab_width - 1
  501. # Link defs are in the form:
  502. # [id]: url "optional title"
  503. _link_def_re = re.compile(r"""
  504. ^[ ]{0,%d}\[(.+)\]: # id = \1
  505. [ \t]*
  506. \n? # maybe *one* newline
  507. [ \t]*
  508. <?(.+?)>? # url = \2
  509. [ \t]*
  510. (?:
  511. \n? # maybe one newline
  512. [ \t]*
  513. (?<=\s) # lookbehind for whitespace
  514. ['"(]
  515. ([^\n]*) # title = \3
  516. ['")]
  517. [ \t]*
  518. )? # title is optional
  519. (?:\n+|\Z)
  520. """ % less_than_tab, re.X | re.M | re.U)
  521. return _link_def_re.sub(self._extract_link_def_sub, text)
  522. def _extract_link_def_sub(self, match):
  523. id, url, title = match.groups()
  524. key = id.lower() # Link IDs are case-insensitive
  525. self.urls[key] = self._encode_amps_and_angles(url)
  526. if title:
  527. self.titles[key] = title.replace('"', '&quot;')
  528. return ""
  529. def _extract_footnote_def_sub(self, match):
  530. id, text = match.groups()
  531. text = _dedent(text, skip_first_line=not text.startswith('\n')).strip()
  532. normed_id = re.sub(r'\W', '-', id)
  533. # Ensure footnote text ends with a couple newlines (for some
  534. # block gamut matches).
  535. self.footnotes[normed_id] = text + "\n\n"
  536. return ""
  537. def _strip_footnote_definitions(self, text):
  538. """A footnote definition looks like this:
  539. [^note-id]: Text of the note.
  540. May include one or more indented paragraphs.
  541. Where,
  542. - The 'note-id' can be pretty much anything, though typically it
  543. is the number of the footnote.
  544. - The first paragraph may start on the next line, like so:
  545. [^note-id]:
  546. Text of the note.
  547. """
  548. less_than_tab = self.tab_width - 1
  549. footnote_def_re = re.compile(r'''
  550. ^[ ]{0,%d}\[\^(.+)\]: # id = \1
  551. [ \t]*
  552. ( # footnote text = \2
  553. # First line need not start with the spaces.
  554. (?:\s*.*\n+)
  555. (?:
  556. (?:[ ]{%d} | \t) # Subsequent lines must be indented.
  557. .*\n+
  558. )*
  559. )
  560. # Lookahead for non-space at line-start, or end of doc.
  561. (?:(?=^[ ]{0,%d}\S)|\Z)
  562. ''' % (less_than_tab, self.tab_width, self.tab_width),
  563. re.X | re.M)
  564. return footnote_def_re.sub(self._extract_footnote_def_sub, text)
  565. _hr_res = [
  566. re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M),
  567. re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M),
  568. re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M),
  569. ]
  570. def _run_block_gamut(self, text):
  571. # These are all the transformations that form block-level
  572. # tags like paragraphs, headers, and list items.
  573. text = self._do_headers(text)
  574. # Do Horizontal Rules:
  575. hr = "\n<hr"+self.empty_element_suffix+"\n"
  576. for hr_re in self._hr_res:
  577. text = hr_re.sub(hr, text)
  578. text = self._do_lists(text)
  579. if "pyshell" in self.extras:
  580. text = self._prepare_pyshell_blocks(text)
  581. text = self._do_code_blocks(text)
  582. text = self._do_block_quotes(text)
  583. # We already ran _HashHTMLBlocks() before, in Markdown(), but that
  584. # was to escape raw HTML in the original Markdown source. This time,
  585. # we're escaping the markup we've just created, so that we don't wrap
  586. # <p> tags around block-level tags.
  587. text = self._hash_html_blocks(text)
  588. text = self._form_paragraphs(text)
  589. return text
  590. def _pyshell_block_sub(self, match):
  591. lines = match.group(0).splitlines(0)
  592. _dedentlines(lines)
  593. indent = ' ' * self.tab_width
  594. s = ('\n' # separate from possible cuddled paragraph
  595. + indent + ('\n'+indent).join(lines)
  596. + '\n\n')
  597. return s
  598. def _prepare_pyshell_blocks(self, text):
  599. """Ensure that Python interactive shell sessions are put in
  600. code blocks -- even if not properly indented.
  601. """
  602. if ">>>" not in text:
  603. return text
  604. less_than_tab = self.tab_width - 1
  605. _pyshell_block_re = re.compile(r"""
  606. ^([ ]{0,%d})>>>[ ].*\n # first line
  607. ^(\1.*\S+.*\n)* # any number of subsequent lines
  608. ^\n # ends with a blank line
  609. """ % less_than_tab, re.M | re.X)
  610. return _pyshell_block_re.sub(self._pyshell_block_sub, text)
  611. def _run_span_gamut(self, text):
  612. # These are all the transformations that occur *within* block-level
  613. # tags like paragraphs, headers, and list items.
  614. text = self._do_code_spans(text)
  615. text = self._escape_special_chars(text)
  616. # Process anchor and image tags.
  617. text = self._do_links(text)
  618. # Make links out of things like `<http://example.com/>`
  619. # Must come after _do_links(), because you can use < and >
  620. # delimiters in inline links like [this](<url>).
  621. text = self._do_auto_links(text)
  622. if "link-patterns" in self.extras:
  623. text = self._do_link_patterns(text)
  624. text = self._encode_amps_and_angles(text)
  625. text = self._do_italics_and_bold(text)
  626. # Do hard breaks:
  627. text = re.sub(r" {2,}\n", " <br%s\n" % self.empty_element_suffix, text)
  628. return text
  629. # "Sorta" because auto-links are identified as "tag" tokens.
  630. _sorta_html_tokenize_re = re.compile(r"""
  631. (
  632. # tag
  633. </?
  634. (?:\w+) # tag name
  635. (?:\s+(?:[\w-]+:)?[\w-]+=(?:".*?"|'.*?'))* # attributes
  636. \s*/?>
  637. |
  638. # auto-link (e.g., <http://www.activestate.com/>)
  639. <\w+[^>]*>
  640. |
  641. <!--.*?--> # comment
  642. |
  643. <\?.*?\?> # processing instruction
  644. )
  645. """, re.X)
  646. def _escape_special_chars(self, text):
  647. # Python markdown note: the HTML tokenization here differs from
  648. # that in Markdown.pl, hence the behaviour for subtle cases can
  649. # differ (I believe the tokenizer here does a better job because
  650. # it isn't susceptible to unmatched '<' and '>' in HTML tags).
  651. # Note, however, that '>' is not allowed in an auto-link URL
  652. # here.
  653. escaped = []
  654. is_html_markup = False
  655. for token in self._sorta_html_tokenize_re.split(text):
  656. if is_html_markup:
  657. # Within tags/HTML-comments/auto-links, encode * and _
  658. # so they don't conflict with their use in Markdown for
  659. # italics and strong. We're replacing each such
  660. # character with its corresponding MD5 checksum value;
  661. # this is likely overkill, but it should prevent us from
  662. # colliding with the escape values by accident.
  663. escaped.append(token.replace('*', g_escape_table['*'])
  664. .replace('_', g_escape_table['_']))
  665. else:
  666. escaped.append(self._encode_backslash_escapes(token))
  667. is_html_markup = not is_html_markup
  668. return ''.join(escaped)
  669. def _hash_html_spans(self, text):
  670. # Used for safe_mode.
  671. def _is_auto_link(s):
  672. if ':' in s and self._auto_link_re.match(s):
  673. return True
  674. elif '@' in s and self._auto_email_link_re.match(s):
  675. return True
  676. return False
  677. tokens = []
  678. is_html_markup = False
  679. for token in self._sorta_html_tokenize_re.split(text):
  680. if is_html_markup and not _is_auto_link(token):
  681. sanitized = self._sanitize_html(token)
  682. key = _hash_text(sanitized)
  683. self.html_spans[key] = sanitized
  684. tokens.append(key)
  685. else:
  686. tokens.append(token)
  687. is_html_markup = not is_html_markup
  688. return ''.join(tokens)
  689. def _unhash_html_spans(self, text):
  690. for key, sanitized in self.html_spans.items():
  691. text = text.replace(key, sanitized)
  692. return text
  693. def _sanitize_html(self, s):
  694. if self.safe_mode == "replace":
  695. return self.html_removed_text
  696. elif self.safe_mode == "escape":
  697. replacements = [
  698. ('&', '&amp;'),
  699. ('<', '&lt;'),
  700. ('>', '&gt;'),
  701. ]
  702. for before, after in replacements:
  703. s = s.replace(before, after)
  704. return s
  705. else:
  706. raise MarkdownError("invalid value for 'safe_mode': %r (must be "
  707. "'escape' or 'replace')" % self.safe_mode)
  708. _tail_of_inline_link_re = re.compile(r'''
  709. # Match tail of: [text](/url/) or [text](/url/ "title")
  710. \( # literal paren
  711. [ \t]*
  712. (?P<url> # \1
  713. <.*?>
  714. |
  715. .*?
  716. )
  717. [ \t]*
  718. ( # \2
  719. (['"]) # quote char = \3
  720. (?P<title>.*?)
  721. \3 # matching quote
  722. )? # title is optional
  723. \)
  724. ''', re.X | re.S)
  725. _tail_of_reference_link_re = re.compile(r'''
  726. # Match tail of: [text][id]
  727. [ ]? # one optional space
  728. (?:\n[ ]*)? # one optional newline followed by spaces
  729. \[
  730. (?P<id>.*?)
  731. \]
  732. ''', re.X | re.S)
  733. def _do_links(self, text):
  734. """Turn Markdown link shortcuts into XHTML <a> and <img> tags.
  735. This is a combination of Markdown.pl's _DoAnchors() and
  736. _DoImages(). They are done together because that simplified the
  737. approach. It was necessary to use a different approach than
  738. Markdown.pl because of the lack of atomic matching support in
  739. Python's regex engine used in $g_nested_brackets.
  740. """
  741. MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24
  742. # `anchor_allowed_pos` is used to support img links inside
  743. # anchors, but not anchors inside anchors. An anchor's start
  744. # pos must be `>= anchor_allowed_pos`.
  745. anchor_allowed_pos = 0
  746. curr_pos = 0
  747. while True: # Handle the next link.
  748. # The next '[' is the start of:
  749. # - an inline anchor: [text](url "title")
  750. # - a reference anchor: [text][id]
  751. # - an inline img: ![text](url "title")
  752. # - a reference img: ![text][id]
  753. # - a footnote ref: [^id]
  754. # (Only if 'footnotes' extra enabled)
  755. # - a footnote defn: [^id]: ...
  756. # (Only if 'footnotes' extra enabled) These have already
  757. # been stripped in _strip_footnote_definitions() so no
  758. # need to watch for them.
  759. # - a link definition: [id]: url "title"
  760. # These have already been stripped in
  761. # _strip_link_definitions() so no need to watch for them.
  762. # - not markup: [...anything else...
  763. try:
  764. start_idx = text.index('[', curr_pos)
  765. except ValueError:
  766. break
  767. text_length = len(text)
  768. # Find the matching closing ']'.
  769. # Markdown.pl allows *matching* brackets in link text so we
  770. # will here too. Markdown.pl *doesn't* currently allow
  771. # matching brackets in img alt text -- we'll differ in that
  772. # regard.
  773. bracket_depth = 0
  774. for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL,
  775. text_length)):
  776. ch = text[p]
  777. if ch == ']':
  778. bracket_depth -= 1
  779. if bracket_depth < 0:
  780. break
  781. elif ch == '[':
  782. bracket_depth += 1
  783. else:
  784. # Closing bracket not found within sentinel length.
  785. # This isn't markup.
  786. curr_pos = start_idx + 1
  787. continue
  788. link_text = text[start_idx+1:p]
  789. # Possibly a footnote ref?
  790. if "footnotes" in self.extras and link_text.startswith("^"):
  791. normed_id = re.sub(r'\W', '-', link_text[1:])
  792. if normed_id in self.footnotes:
  793. self.footnote_ids.append(normed_id)
  794. result = '<sup class="footnote-ref" id="fnref-%s">' \
  795. '<a href="#fn-%s">%s</a></sup>' \
  796. % (normed_id, normed_id, len(self.footnote_ids))
  797. text = text[:start_idx] + result + text[p+1:]
  798. else:
  799. # This id isn't defined, leave the markup alone.
  800. curr_pos = p+1
  801. continue
  802. # Now determine what this is by the remainder.
  803. p += 1
  804. if p == text_length:
  805. return text
  806. # Inline anchor or img?
  807. if text[p] == '(': # attempt at perf improvement
  808. match = self._tail_of_inline_link_re.match(text, p)
  809. if match:
  810. # Handle an inline anchor or img.
  811. is_img = start_idx > 0 and text[start_idx-1] == "!"
  812. if is_img:
  813. start_idx -= 1
  814. url, title = match.group("url"), match.group("title")
  815. if url and url[0] == '<':
  816. url = url[1:-1] # '<url>' -> 'url'
  817. # We've got to encode these to avoid conflicting
  818. # with italics/bold.
  819. url = url.replace('*', g_escape_table['*']) \
  820. .replace('_', g_escape_table['_'])
  821. if title:
  822. title_str = ' title="%s"' \
  823. % title.replace('*', g_escape_table['*']) \
  824. .replace('_', g_escape_table['_']) \
  825. .replace('"', '&quot;')
  826. else:
  827. title_str = ''
  828. if is_img:
  829. result = '<img src="%s" alt="%s"%s%s' \
  830. % (url.replace('"', '&quot;'),
  831. link_text.replace('"', '&quot;'),
  832. title_str, self.empty_element_suffix)
  833. curr_pos = start_idx + len(result)
  834. text = text[:start_idx] + result + text[match.end():]
  835. elif start_idx >= anchor_allowed_pos:
  836. result_head = '<a href="%s"%s>' % (url, title_str)
  837. result = '%s%s</a>' % (result_head, link_text)
  838. # <img> allowed from curr_pos on, <a> from
  839. # anchor_allowed_pos on.
  840. curr_pos = start_idx + len(result_head)
  841. anchor_allowed_pos = start_idx + len(result)
  842. text = text[:start_idx] + result + text[match.end():]
  843. else:
  844. # Anchor not allowed here.
  845. curr_pos = start_idx + 1
  846. continue
  847. # Reference anchor or img?
  848. else:
  849. match = self._tail_of_reference_link_re.match(text, p)
  850. if match:
  851. # Handle a reference-style anchor or img.
  852. is_img = start_idx > 0 and text[start_idx-1] == "!"
  853. if is_img:
  854. start_idx -= 1
  855. link_id = match.group("id").lower()
  856. if not link_id:
  857. link_id = link_text.lower() # for links like [this][]
  858. if link_id in self.urls:
  859. url = self.urls[link_id]
  860. # We've got to encode these to avoid conflicting
  861. # with italics/bold.
  862. url = url.replace('*', g_escape_table['*']) \
  863. .replace('_', g_escape_table['_'])
  864. title = self.titles.get(link_id)
  865. if title:
  866. title = title.replace('*', g_escape_table['*']) \
  867. .replace('_', g_escape_table['_'])
  868. title_str = ' title="%s"' % title
  869. else:
  870. title_str = ''
  871. if is_img:
  872. result = '<img src="%s" alt="%s"%s%s' \
  873. % (url.replace('"', '&quot;'),
  874. link_text.replace('"', '&quot;'),
  875. title_str, self.empty_element_suffix)
  876. curr_pos = start_idx + len(result)
  877. text = text[:start_idx] + result + text[match.end():]
  878. elif start_idx >= anchor_allowed_pos:
  879. result = '<a href="%s"%s>%s</a>' \
  880. % (url, title_str, link_text)
  881. result_head = '<a href="%s"%s>' % (url, title_str)
  882. result = '%s%s</a>' % (result_head, link_text)
  883. # <img> allowed from curr_pos on, <a> from
  884. # anchor_allowed_pos on.
  885. curr_pos = start_idx + len(result_head)
  886. anchor_allowed_pos = start_idx + len(result)
  887. text = text[:start_idx] + result + text[match.end():]
  888. else:
  889. # Anchor not allowed here.
  890. curr_pos = start_idx + 1
  891. else:
  892. # This id isn't defined, leave the markup alone.
  893. curr_pos = match.end()
  894. continue
  895. # Otherwise, it isn't markup.
  896. curr_pos = start_idx + 1
  897. return text
  898. def header_id_from_text(self, text, prefix):
  899. """Generate a header id attribute value from the given header
  900. HTML content.
  901. This is only called if the "header-ids" extra is enabled.
  902. Subclasses may override this for different header ids.
  903. """
  904. header_id = _slugify(text)
  905. if prefix:
  906. header_id = prefix + '-' + header_id
  907. if header_id in self._count_from_header_id:
  908. self._count_from_header_id[header_id] += 1
  909. header_id += '-%s' % self._count_from_header_id[header_id]
  910. else:
  911. self._count_from_header_id[header_id] = 1
  912. return header_id
  913. _toc = None
  914. def _toc_add_entry(self, level, id, name):
  915. if self._toc is None:
  916. self._toc = []
  917. self._toc.append((level, id, name))
  918. _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M)
  919. def _setext_h_sub(self, match):
  920. n = {"=": 1, "-": 2}[match.group(2)[0]]
  921. demote_headers = self.extras.get("demote-headers")
  922. if demote_headers:
  923. n = min(n + demote_headers, 6)
  924. header_id_attr = ""
  925. if "header-ids" in self.extras:
  926. header_id = self.header_id_from_text(match.group(1),
  927. prefix=self.extras["header-ids"])
  928. header_id_attr = ' id="%s"' % header_id
  929. html = self._run_span_gamut(match.group(1))
  930. if "toc" in self.extras:
  931. self._toc_add_entry(n, header_id, html)
  932. return "<h%d%s>%s</h%d>\n\n" % (n, header_id_attr, html, n)
  933. _atx_h_re = re.compile(r'''
  934. ^(\#{1,6}) # \1 = string of #'s
  935. [ \t]*
  936. (.+?) # \2 = Header text
  937. [ \t]*
  938. (?<!\\) # ensure not an escaped trailing '#'
  939. \#* # optional closing #'s (not counted)
  940. \n+
  941. ''', re.X | re.M)
  942. def _atx_h_sub(self, match):
  943. n = len(match.group(1))
  944. demote_headers = self.extras.get("demote-headers")
  945. if demote_headers:
  946. n = min(n + demote_headers, 6)
  947. header_id_attr = ""
  948. if "header-ids" in self.extras:
  949. header_id = self.header_id_from_text(match.group(2),
  950. prefix=self.extras["header-ids"])
  951. header_id_attr = ' id="%s"' % header_id
  952. html = self._run_span_gamut(match.group(2))
  953. if "toc" in self.extras:
  954. self._toc_add_entry(n, header_id, html)
  955. return "<h%d%s>%s</h%d>\n\n" % (n, header_id_attr, html, n)
  956. def _do_headers(self, text):
  957. # Setext-style headers:
  958. # Header 1
  959. # ========
  960. #
  961. # Header 2
  962. # --------
  963. text = self._setext_h_re.sub(self._setext_h_sub, text)
  964. # atx-style headers:
  965. # # Header 1
  966. # ## Header 2
  967. # ## Header 2 with closing hashes ##
  968. # ...
  969. # ###### Header 6
  970. text = self._atx_h_re.sub(self._atx_h_sub, text)
  971. return text
  972. _marker_ul_chars = '*+-'
  973. _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars
  974. _marker_ul = '(?:[%s])' % _marker_ul_chars
  975. _marker_ol = r'(?:\d+\.)'
  976. def _list_sub(self, match):
  977. lst = match.group(1)
  978. lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol"
  979. result = self._process_list_items(lst)
  980. if self.list_level:
  981. return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type)
  982. else:
  983. return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type)
  984. def _do_lists(self, text):
  985. # Form HTML ordered (numbered) and unordered (bulleted) lists.
  986. for marker_pat in (self._marker_ul, self._marker_ol):
  987. # Re-usable pattern to match any entire ul or ol list:
  988. less_than_tab = self.tab_width - 1
  989. whole_list = r'''
  990. ( # \1 = whole list
  991. ( # \2
  992. [ ]{0,%d}
  993. (%s) # \3 = first list item marker
  994. [ \t]+
  995. )
  996. (?:.+?)
  997. ( # \4
  998. \Z
  999. |
  1000. \n{2,}
  1001. (?=\S)
  1002. (?! # Negative lookahead for another list item marker
  1003. [ \t]*
  1004. %s[ \t]+
  1005. )
  1006. )
  1007. )
  1008. ''' % (less_than_tab, marker_pat, marker_pat)
  1009. # We use a different prefix before nested lists than top-level lists.
  1010. # See extended comment in _process_list_items().
  1011. #
  1012. # Note: There's a bit of duplication here. My original implementation
  1013. # created a scalar regex pattern as the conditional result of the test on
  1014. # $g_list_level, and then only ran the $text =~ s{...}{...}egmx
  1015. # substitution once, using the scalar as the pattern. This worked,
  1016. # everywhere except when running under MT on my hosting account at Pair
  1017. # Networks. There, this caused all rebuilds to be killed by the reaper (or
  1018. # perhaps they crashed, but that seems incredibly unlikely given that the
  1019. # same script on the same server ran fine *except* under MT. I've spent
  1020. # more time trying to figure out why this is happening than I'd like to
  1021. # admit. My only guess, backed up by the fact that this workaround works,
  1022. # is that Perl optimizes the substition when it can figure out that the
  1023. # pattern will never change, and when this optimization isn't on, we run
  1024. # afoul of the reaper. Thus, the slightly redundant code to that uses two
  1025. # static s/// patterns rather than one conditional pattern.
  1026. if self.list_level:
  1027. sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S)
  1028. text = sub_list_re.sub(self._list_sub, text)
  1029. else:
  1030. list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list,
  1031. re.X | re.M | re.S)
  1032. text = list_re.sub(self._list_sub, text)
  1033. return text
  1034. _list_item_re = re.compile(r'''
  1035. (\n)? # leading line = \1
  1036. (^[ \t]*) # leading whitespace = \2
  1037. (?P<marker>%s) [ \t]+ # list marker = \3
  1038. ((?:.+?) # list item text = \4
  1039. (\n{1,2})) # eols = \5
  1040. (?= \n* (\Z | \2 (?P<next_marker>%s) [ \t]+))
  1041. ''' % (_marker_any, _marker_any),
  1042. re.M | re.X | re.S)
  1043. _last_li_endswith_two_eols = False
  1044. def _list_item_sub(self, match):
  1045. item = match.group(4)
  1046. leading_line = match.group(1)
  1047. leading_space = match.group(2)
  1048. if leading_line or "\n\n" in item or self._last_li_endswith_two_eols:
  1049. item = self._run_block_gamut(self._outdent(item))
  1050. else:
  1051. # Recursion for sub-lists:
  1052. item = self._do_lists(self._outdent(item))
  1053. if item.endswith('\n'):
  1054. item = item[:-1]
  1055. item = self._run_span_gamut(item)
  1056. self._last_li_endswith_two_eols = (len(match.group(5)) == 2)
  1057. return "<li>%s</li>\n" % item
  1058. def _process_list_items(self, list_str):
  1059. # Process the contents of a single ordered or unordered list,
  1060. # splitting it into individual list items.
  1061. # The $g_list_level global keeps track of when we're inside a list.
  1062. # Each time we enter a list, we increment it; when we leave a list,
  1063. # we decrement. If it's zero, we're not in a list anymore.
  1064. #
  1065. # We do this because when we're not inside a list, we want to treat
  1066. # something like this:
  1067. #
  1068. # I recommend upgrading to version
  1069. # 8. Oops, now this line is treated
  1070. # as a sub-list.
  1071. #
  1072. # As a single paragraph, despite the fact that the second line starts
  1073. # with a digit-period-space sequence.
  1074. #
  1075. # Whereas when we're inside a list (or sub-list), that line will be
  1076. # treated as the start of a sub-list. What a kludge, huh? This i