/Lib/json/decoder.py

  1"""Implementation of JSONDecoder
  2"""
  3
  4import re
  5import sys
  6
  7from json.scanner import Scanner, pattern
  8try:
  9    from _json import scanstring as c_scanstring
 10except ImportError:
 11    c_scanstring = None
 12
 13__all__ = ['JSONDecoder']
 14
 15FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
 16
 17NaN, PosInf, NegInf = float('nan'), float('inf'), float('-inf')
 18
 19
 20def linecol(doc, pos):
 21    lineno = doc.count('\n', 0, pos) + 1
 22    if lineno == 1:
 23        colno = pos
 24    else:
 25        colno = pos - doc.rindex('\n', 0, pos)
 26    return lineno, colno
 27
 28
 29def errmsg(msg, doc, pos, end=None):
 30    lineno, colno = linecol(doc, pos)
 31    if end is None:
 32        fmt = '{0}: line {1} column {2} (char {3})'
 33        return fmt.format(msg, lineno, colno, pos)
 34    endlineno, endcolno = linecol(doc, end)
 35    fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
 36    return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
 37
 38
 39_CONSTANTS = {
 40    '-Infinity': NegInf,
 41    'Infinity': PosInf,
 42    'NaN': NaN,
 43    'true': True,
 44    'false': False,
 45    'null': None,
 46}
 47
 48
 49def JSONConstant(match, context, c=_CONSTANTS):
 50    s = match.group(0)
 51    fn = getattr(context, 'parse_constant', None)
 52    if fn is None:
 53        rval = c[s]
 54    else:
 55        rval = fn(s)
 56    return rval, None
 57pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
 58
 59
 60def JSONNumber(match, context):
 61    match = JSONNumber.regex.match(match.string, *match.span())
 62    integer, frac, exp = match.groups()
 63    if frac or exp:
 64        fn = getattr(context, 'parse_float', None) or float
 65        res = fn(integer + (frac or '') + (exp or ''))
 66    else:
 67        fn = getattr(context, 'parse_int', None) or int
 68        res = fn(integer)
 69    return res, None
 70pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
 71
 72
 73STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
 74BACKSLASH = {
 75    '"': u'"', '\\': u'\\', '/': u'/',
 76    'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
 77}
 78
 79DEFAULT_ENCODING = "utf-8"
 80
 81
 82def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
 83    if encoding is None:
 84        encoding = DEFAULT_ENCODING
 85    chunks = []
 86    _append = chunks.append
 87    begin = end - 1
 88    while 1:
 89        chunk = _m(s, end)
 90        if chunk is None:
 91            raise ValueError(
 92                errmsg("Unterminated string starting at", s, begin))
 93        end = chunk.end()
 94        content, terminator = chunk.groups()
 95        if content:
 96            if not isinstance(content, unicode):
 97                content = unicode(content, encoding)
 98            _append(content)
 99        if terminator == '"':
100            break
101        elif terminator != '\\':
102            if strict:
103                msg = "Invalid control character {0!r} at".format(terminator)
104                raise ValueError(errmsg(msg, s, end))
105            else:
106                _append(terminator)
107                continue
108        try:
109            esc = s[end]
110        except IndexError:
111            raise ValueError(
112                errmsg("Unterminated string starting at", s, begin))
113        if esc != 'u':
114            try:
115                m = _b[esc]
116            except KeyError:
117                msg = "Invalid \\escape: {0!r}".format(esc)
118                raise ValueError(errmsg(msg, s, end))
119            end += 1
120        else:
121            esc = s[end + 1:end + 5]
122            next_end = end + 5
123            msg = "Invalid \\uXXXX escape"
124            try:
125                if len(esc) != 4:
126                    raise ValueError
127                uni = int(esc, 16)
128                if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
129                    msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
130                    if not s[end + 5:end + 7] == '\\u':
131                        raise ValueError
132                    esc2 = s[end + 7:end + 11]
133                    if len(esc2) != 4:
134                        raise ValueError
135                    uni2 = int(esc2, 16)
136                    uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
137                    next_end += 6
138                m = unichr(uni)
139            except ValueError:
140                raise ValueError(errmsg(msg, s, end))
141            end = next_end
142        _append(m)
143    return u''.join(chunks), end
144
145
146# Use speedup
147if c_scanstring is not None:
148    scanstring = c_scanstring
149else:
150    scanstring = py_scanstring
151
152def JSONString(match, context):
153    encoding = getattr(context, 'encoding', None)
154    strict = getattr(context, 'strict', True)
155    return scanstring(match.string, match.end(), encoding, strict)
156pattern(r'"')(JSONString)
157
158
159WHITESPACE = re.compile(r'\s*', FLAGS)
160
161
162def JSONObject(match, context, _w=WHITESPACE.match):
163    pairs = {}
164    s = match.string
165    end = _w(s, match.end()).end()
166    nextchar = s[end:end + 1]
167    # Trivial empty object
168    if nextchar == '}':
169        return pairs, end + 1
170    if nextchar != '"':
171        raise ValueError(errmsg("Expecting property name", s, end))
172    end += 1
173    encoding = getattr(context, 'encoding', None)
174    strict = getattr(context, 'strict', True)
175    iterscan = JSONScanner.iterscan
176    while True:
177        key, end = scanstring(s, end, encoding, strict)
178        end = _w(s, end).end()
179        if s[end:end + 1] != ':':
180            raise ValueError(errmsg("Expecting : delimiter", s, end))
181        end = _w(s, end + 1).end()
182        try:
183            value, end = iterscan(s, idx=end, context=context).next()
184        except StopIteration:
185            raise ValueError(errmsg("Expecting object", s, end))
186        pairs[key] = value
187        end = _w(s, end).end()
188        nextchar = s[end:end + 1]
189        end += 1
190        if nextchar == '}':
191            break
192        if nextchar != ',':
193            raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
194        end = _w(s, end).end()
195        nextchar = s[end:end + 1]
196        end += 1
197        if nextchar != '"':
198            raise ValueError(errmsg("Expecting property name", s, end - 1))
199    object_hook = getattr(context, 'object_hook', None)
200    if object_hook is not None:
201        pairs = object_hook(pairs)
202    return pairs, end
203pattern(r'{')(JSONObject)
204
205
206def JSONArray(match, context, _w=WHITESPACE.match):
207    values = []
208    s = match.string
209    end = _w(s, match.end()).end()
210    # Look-ahead for trivial empty array
211    nextchar = s[end:end + 1]
212    if nextchar == ']':
213        return values, end + 1
214    iterscan = JSONScanner.iterscan
215    while True:
216        try:
217            value, end = iterscan(s, idx=end, context=context).next()
218        except StopIteration:
219            raise ValueError(errmsg("Expecting object", s, end))
220        values.append(value)
221        end = _w(s, end).end()
222        nextchar = s[end:end + 1]
223        end += 1
224        if nextchar == ']':
225            break
226        if nextchar != ',':
227            raise ValueError(errmsg("Expecting , delimiter", s, end))
228        end = _w(s, end).end()
229    return values, end
230pattern(r'\[')(JSONArray)
231
232
233ANYTHING = [
234    JSONObject,
235    JSONArray,
236    JSONString,
237    JSONConstant,
238    JSONNumber,
239]
240
241JSONScanner = Scanner(ANYTHING)
242
243
244class JSONDecoder(object):
245    """Simple JSON <http://json.org> decoder
246
247    Performs the following translations in decoding by default:
248
249    +---------------+-------------------+
250    | JSON          | Python            |
251    +===============+===================+
252    | object        | dict              |
253    +---------------+-------------------+
254    | array         | list              |
255    +---------------+-------------------+
256    | string        | unicode           |
257    +---------------+-------------------+
258    | number (int)  | int, long         |
259    +---------------+-------------------+
260    | number (real) | float             |
261    +---------------+-------------------+
262    | true          | True              |
263    +---------------+-------------------+
264    | false         | False             |
265    +---------------+-------------------+
266    | null          | None              |
267    +---------------+-------------------+
268
269    It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
270    their corresponding ``float`` values, which is outside the JSON spec.
271    """
272
273    _scanner = Scanner(ANYTHING)
274    __all__ = ['__init__', 'decode', 'raw_decode']
275
276    def __init__(self, encoding=None, object_hook=None, parse_float=None,
277            parse_int=None, parse_constant=None, strict=True):
278        """``encoding`` determines the encoding used to interpret any ``str``
279        objects decoded by this instance (utf-8 by default).  It has no
280        effect when decoding ``unicode`` objects.
281
282        Note that currently only encodings that are a superset of ASCII work,
283        strings of other encodings should be passed in as ``unicode``.
284
285        ``object_hook``, if specified, will be called with the result of
286        every JSON object decoded and its return value will be used in
287        place of the given ``dict``.  This can be used to provide custom
288        deserializations (e.g. to support JSON-RPC class hinting).
289
290        ``parse_float``, if specified, will be called with the string
291        of every JSON float to be decoded. By default this is equivalent to
292        float(num_str). This can be used to use another datatype or parser
293        for JSON floats (e.g. decimal.Decimal).
294
295        ``parse_int``, if specified, will be called with the string
296        of every JSON int to be decoded. By default this is equivalent to
297        int(num_str). This can be used to use another datatype or parser
298        for JSON integers (e.g. float).
299
300        ``parse_constant``, if specified, will be called with one of the
301        following strings: -Infinity, Infinity, NaN, null, true, false.
302        This can be used to raise an exception if invalid JSON numbers
303        are encountered.
304
305        """
306        self.encoding = encoding
307        self.object_hook = object_hook
308        self.parse_float = parse_float
309        self.parse_int = parse_int
310        self.parse_constant = parse_constant
311        self.strict = strict
312
313    def decode(self, s, _w=WHITESPACE.match):
314        """
315        Return the Python representation of ``s`` (a ``str`` or ``unicode``
316        instance containing a JSON document)
317
318        """
319        obj, end = self.raw_decode(s, idx=_w(s, 0).end())
320        end = _w(s, end).end()
321        if end != len(s):
322            raise ValueError(errmsg("Extra data", s, end, len(s)))
323        return obj
324
325    def raw_decode(self, s, **kw):
326        """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
327        with a JSON document) and return a 2-tuple of the Python
328        representation and the index in ``s`` where the document ended.
329
330        This can be used to decode a JSON document from a string that may
331        have extraneous data at the end.
332
333        """
334        kw.setdefault('context', self)
335        try:
336            obj, end = self._scanner.iterscan(s, **kw).next()
337        except StopIteration:
338            raise ValueError("No JSON object could be decoded")
339        return obj, end