/Lib/json/decoder.py

http://unladen-swallow.googlecode.com/ · Python · 339 lines · 228 code · 38 blank · 73 comment · 58 complexity · b54c854c06e6cd8a23c4b90d8dffe01b MD5 · raw file

  1. """Implementation of JSONDecoder
  2. """
  3. import re
  4. import sys
  5. from json.scanner import Scanner, pattern
  6. try:
  7. from _json import scanstring as c_scanstring
  8. except ImportError:
  9. c_scanstring = None
  10. __all__ = ['JSONDecoder']
  11. FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
  12. NaN, PosInf, NegInf = float('nan'), float('inf'), float('-inf')
  13. def linecol(doc, pos):
  14. lineno = doc.count('\n', 0, pos) + 1
  15. if lineno == 1:
  16. colno = pos
  17. else:
  18. colno = pos - doc.rindex('\n', 0, pos)
  19. return lineno, colno
  20. def errmsg(msg, doc, pos, end=None):
  21. lineno, colno = linecol(doc, pos)
  22. if end is None:
  23. fmt = '{0}: line {1} column {2} (char {3})'
  24. return fmt.format(msg, lineno, colno, pos)
  25. endlineno, endcolno = linecol(doc, end)
  26. fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
  27. return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
  28. _CONSTANTS = {
  29. '-Infinity': NegInf,
  30. 'Infinity': PosInf,
  31. 'NaN': NaN,
  32. 'true': True,
  33. 'false': False,
  34. 'null': None,
  35. }
  36. def JSONConstant(match, context, c=_CONSTANTS):
  37. s = match.group(0)
  38. fn = getattr(context, 'parse_constant', None)
  39. if fn is None:
  40. rval = c[s]
  41. else:
  42. rval = fn(s)
  43. return rval, None
  44. pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
  45. def JSONNumber(match, context):
  46. match = JSONNumber.regex.match(match.string, *match.span())
  47. integer, frac, exp = match.groups()
  48. if frac or exp:
  49. fn = getattr(context, 'parse_float', None) or float
  50. res = fn(integer + (frac or '') + (exp or ''))
  51. else:
  52. fn = getattr(context, 'parse_int', None) or int
  53. res = fn(integer)
  54. return res, None
  55. pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
  56. STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
  57. BACKSLASH = {
  58. '"': u'"', '\\': u'\\', '/': u'/',
  59. 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
  60. }
  61. DEFAULT_ENCODING = "utf-8"
  62. def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
  63. if encoding is None:
  64. encoding = DEFAULT_ENCODING
  65. chunks = []
  66. _append = chunks.append
  67. begin = end - 1
  68. while 1:
  69. chunk = _m(s, end)
  70. if chunk is None:
  71. raise ValueError(
  72. errmsg("Unterminated string starting at", s, begin))
  73. end = chunk.end()
  74. content, terminator = chunk.groups()
  75. if content:
  76. if not isinstance(content, unicode):
  77. content = unicode(content, encoding)
  78. _append(content)
  79. if terminator == '"':
  80. break
  81. elif terminator != '\\':
  82. if strict:
  83. msg = "Invalid control character {0!r} at".format(terminator)
  84. raise ValueError(errmsg(msg, s, end))
  85. else:
  86. _append(terminator)
  87. continue
  88. try:
  89. esc = s[end]
  90. except IndexError:
  91. raise ValueError(
  92. errmsg("Unterminated string starting at", s, begin))
  93. if esc != 'u':
  94. try:
  95. m = _b[esc]
  96. except KeyError:
  97. msg = "Invalid \\escape: {0!r}".format(esc)
  98. raise ValueError(errmsg(msg, s, end))
  99. end += 1
  100. else:
  101. esc = s[end + 1:end + 5]
  102. next_end = end + 5
  103. msg = "Invalid \\uXXXX escape"
  104. try:
  105. if len(esc) != 4:
  106. raise ValueError
  107. uni = int(esc, 16)
  108. if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
  109. msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
  110. if not s[end + 5:end + 7] == '\\u':
  111. raise ValueError
  112. esc2 = s[end + 7:end + 11]
  113. if len(esc2) != 4:
  114. raise ValueError
  115. uni2 = int(esc2, 16)
  116. uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
  117. next_end += 6
  118. m = unichr(uni)
  119. except ValueError:
  120. raise ValueError(errmsg(msg, s, end))
  121. end = next_end
  122. _append(m)
  123. return u''.join(chunks), end
  124. # Use speedup
  125. if c_scanstring is not None:
  126. scanstring = c_scanstring
  127. else:
  128. scanstring = py_scanstring
  129. def JSONString(match, context):
  130. encoding = getattr(context, 'encoding', None)
  131. strict = getattr(context, 'strict', True)
  132. return scanstring(match.string, match.end(), encoding, strict)
  133. pattern(r'"')(JSONString)
  134. WHITESPACE = re.compile(r'\s*', FLAGS)
  135. def JSONObject(match, context, _w=WHITESPACE.match):
  136. pairs = {}
  137. s = match.string
  138. end = _w(s, match.end()).end()
  139. nextchar = s[end:end + 1]
  140. # Trivial empty object
  141. if nextchar == '}':
  142. return pairs, end + 1
  143. if nextchar != '"':
  144. raise ValueError(errmsg("Expecting property name", s, end))
  145. end += 1
  146. encoding = getattr(context, 'encoding', None)
  147. strict = getattr(context, 'strict', True)
  148. iterscan = JSONScanner.iterscan
  149. while True:
  150. key, end = scanstring(s, end, encoding, strict)
  151. end = _w(s, end).end()
  152. if s[end:end + 1] != ':':
  153. raise ValueError(errmsg("Expecting : delimiter", s, end))
  154. end = _w(s, end + 1).end()
  155. try:
  156. value, end = iterscan(s, idx=end, context=context).next()
  157. except StopIteration:
  158. raise ValueError(errmsg("Expecting object", s, end))
  159. pairs[key] = value
  160. end = _w(s, end).end()
  161. nextchar = s[end:end + 1]
  162. end += 1
  163. if nextchar == '}':
  164. break
  165. if nextchar != ',':
  166. raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
  167. end = _w(s, end).end()
  168. nextchar = s[end:end + 1]
  169. end += 1
  170. if nextchar != '"':
  171. raise ValueError(errmsg("Expecting property name", s, end - 1))
  172. object_hook = getattr(context, 'object_hook', None)
  173. if object_hook is not None:
  174. pairs = object_hook(pairs)
  175. return pairs, end
  176. pattern(r'{')(JSONObject)
  177. def JSONArray(match, context, _w=WHITESPACE.match):
  178. values = []
  179. s = match.string
  180. end = _w(s, match.end()).end()
  181. # Look-ahead for trivial empty array
  182. nextchar = s[end:end + 1]
  183. if nextchar == ']':
  184. return values, end + 1
  185. iterscan = JSONScanner.iterscan
  186. while True:
  187. try:
  188. value, end = iterscan(s, idx=end, context=context).next()
  189. except StopIteration:
  190. raise ValueError(errmsg("Expecting object", s, end))
  191. values.append(value)
  192. end = _w(s, end).end()
  193. nextchar = s[end:end + 1]
  194. end += 1
  195. if nextchar == ']':
  196. break
  197. if nextchar != ',':
  198. raise ValueError(errmsg("Expecting , delimiter", s, end))
  199. end = _w(s, end).end()
  200. return values, end
  201. pattern(r'\[')(JSONArray)
  202. ANYTHING = [
  203. JSONObject,
  204. JSONArray,
  205. JSONString,
  206. JSONConstant,
  207. JSONNumber,
  208. ]
  209. JSONScanner = Scanner(ANYTHING)
  210. class JSONDecoder(object):
  211. """Simple JSON <http://json.org> decoder
  212. Performs the following translations in decoding by default:
  213. +---------------+-------------------+
  214. | JSON | Python |
  215. +===============+===================+
  216. | object | dict |
  217. +---------------+-------------------+
  218. | array | list |
  219. +---------------+-------------------+
  220. | string | unicode |
  221. +---------------+-------------------+
  222. | number (int) | int, long |
  223. +---------------+-------------------+
  224. | number (real) | float |
  225. +---------------+-------------------+
  226. | true | True |
  227. +---------------+-------------------+
  228. | false | False |
  229. +---------------+-------------------+
  230. | null | None |
  231. +---------------+-------------------+
  232. It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
  233. their corresponding ``float`` values, which is outside the JSON spec.
  234. """
  235. _scanner = Scanner(ANYTHING)
  236. __all__ = ['__init__', 'decode', 'raw_decode']
  237. def __init__(self, encoding=None, object_hook=None, parse_float=None,
  238. parse_int=None, parse_constant=None, strict=True):
  239. """``encoding`` determines the encoding used to interpret any ``str``
  240. objects decoded by this instance (utf-8 by default). It has no
  241. effect when decoding ``unicode`` objects.
  242. Note that currently only encodings that are a superset of ASCII work,
  243. strings of other encodings should be passed in as ``unicode``.
  244. ``object_hook``, if specified, will be called with the result of
  245. every JSON object decoded and its return value will be used in
  246. place of the given ``dict``. This can be used to provide custom
  247. deserializations (e.g. to support JSON-RPC class hinting).
  248. ``parse_float``, if specified, will be called with the string
  249. of every JSON float to be decoded. By default this is equivalent to
  250. float(num_str). This can be used to use another datatype or parser
  251. for JSON floats (e.g. decimal.Decimal).
  252. ``parse_int``, if specified, will be called with the string
  253. of every JSON int to be decoded. By default this is equivalent to
  254. int(num_str). This can be used to use another datatype or parser
  255. for JSON integers (e.g. float).
  256. ``parse_constant``, if specified, will be called with one of the
  257. following strings: -Infinity, Infinity, NaN, null, true, false.
  258. This can be used to raise an exception if invalid JSON numbers
  259. are encountered.
  260. """
  261. self.encoding = encoding
  262. self.object_hook = object_hook
  263. self.parse_float = parse_float
  264. self.parse_int = parse_int
  265. self.parse_constant = parse_constant
  266. self.strict = strict
  267. def decode(self, s, _w=WHITESPACE.match):
  268. """
  269. Return the Python representation of ``s`` (a ``str`` or ``unicode``
  270. instance containing a JSON document)
  271. """
  272. obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  273. end = _w(s, end).end()
  274. if end != len(s):
  275. raise ValueError(errmsg("Extra data", s, end, len(s)))
  276. return obj
  277. def raw_decode(self, s, **kw):
  278. """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
  279. with a JSON document) and return a 2-tuple of the Python
  280. representation and the index in ``s`` where the document ended.
  281. This can be used to decode a JSON document from a string that may
  282. have extraneous data at the end.
  283. """
  284. kw.setdefault('context', self)
  285. try:
  286. obj, end = self._scanner.iterscan(s, **kw).next()
  287. except StopIteration:
  288. raise ValueError("No JSON object could be decoded")
  289. return obj, end