/src/com/google/maps/extras/arcgislink/json/JSONTokenizer.as

  1/*
  2  Copyright (c) 2008, Adobe Systems Incorporated
  3  All rights reserved.
  4
  5  Redistribution and use in source and binary forms, with or without 
  6  modification, are permitted provided that the following conditions are
  7  met:
  8
  9  * Redistributions of source code must retain the above copyright notice, 
 10    this list of conditions and the following disclaimer.
 11  
 12  * Redistributions in binary form must reproduce the above copyright
 13    notice, this list of conditions and the following disclaimer in the 
 14    documentation and/or other materials provided with the distribution.
 15  
 16  * Neither the name of Adobe Systems Incorporated nor the names of its 
 17    contributors may be used to endorse or promote products derived from 
 18    this software without specific prior written permission.
 19
 20  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 21  IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 22  THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 23  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 
 24  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 25  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 26  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 27  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 28  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 29  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 30  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 31*/
 32
 33//http://code.google.com/p/as3corelib/
 34//package com.adobe.serialization.json
 35package com.google.maps.extras.arcgislink.json {
 36
 37	public class JSONTokenizer {
 38		
 39		/** 
 40		 * Flag indicating if the tokenizer should only recognize
 41		 * standard JSON tokens.  Setting to <code>false</code> allows
 42		 * tokens such as NaN and allows numbers to be formatted as
 43		 * hex, etc.
 44		 */
 45		private var strict:Boolean;
 46	
 47		/** The object that will get parsed from the JSON string */
 48		private var obj:Object;
 49		
 50		/** The JSON string to be parsed */
 51		private var jsonString:String;
 52		
 53		/** The current parsing location in the JSON string */
 54		private var loc:int;
 55		
 56		/** The current character in the JSON string during parsing */
 57		private var ch:String;
 58		
 59		/**
 60		 * Constructs a new JSONDecoder to parse a JSON string 
 61		 * into a native object.
 62		 *
 63		 * @param s The JSON string to be converted
 64		 *		into a native object
 65		 */
 66		public function JSONTokenizer( s:String, strict:Boolean )
 67		{
 68			jsonString = s;
 69			this.strict = strict;
 70			loc = 0;
 71			
 72			// prime the pump by getting the first character
 73			nextChar();
 74		}
 75		
 76		/**
 77		 * Gets the next token in the input sting and advances
 78		* the character to the next character after the token
 79		 */
 80		public function getNextToken():JSONToken
 81		{
 82			var token:JSONToken = new JSONToken();
 83			
 84			// skip any whitespace / comments since the last 
 85			// token was read
 86			skipIgnored();
 87						
 88			// examine the new character and see what we have...
 89			switch ( ch )
 90			{	
 91				case '{':
 92					token.type = JSONTokenType.LEFT_BRACE;
 93					token.value = '{';
 94					nextChar();
 95					break
 96					
 97				case '}':
 98					token.type = JSONTokenType.RIGHT_BRACE;
 99					token.value = '}';
100					nextChar();
101					break
102					
103				case '[':
104					token.type = JSONTokenType.LEFT_BRACKET;
105					token.value = '[';
106					nextChar();
107					break
108					
109				case ']':
110					token.type = JSONTokenType.RIGHT_BRACKET;
111					token.value = ']';
112					nextChar();
113					break
114				
115				case ',':
116					token.type = JSONTokenType.COMMA;
117					token.value = ',';
118					nextChar();
119					break
120					
121				case ':':
122					token.type = JSONTokenType.COLON;
123					token.value = ':';
124					nextChar();
125					break;
126					
127				case 't': // attempt to read true
128					var possibleTrue:String = "t" + nextChar() + nextChar() + nextChar();
129					
130					if ( possibleTrue == "true" )
131					{
132						token.type = JSONTokenType.TRUE;
133						token.value = true;
134						nextChar();
135					}
136					else
137					{
138						parseError( "Expecting 'true' but found " + possibleTrue );
139					}
140					
141					break;
142					
143				case 'f': // attempt to read false
144					var possibleFalse:String = "f" + nextChar() + nextChar() + nextChar() + nextChar();
145					
146					if ( possibleFalse == "false" )
147					{
148						token.type = JSONTokenType.FALSE;
149						token.value = false;
150						nextChar();
151					}
152					else
153					{
154						parseError( "Expecting 'false' but found " + possibleFalse );
155					}
156					
157					break;
158					
159				case 'n': // attempt to read null
160					var possibleNull:String = "n" + nextChar() + nextChar() + nextChar();
161					
162					if ( possibleNull == "null" )
163					{
164						token.type = JSONTokenType.NULL;
165						token.value = null;
166						nextChar();
167					}
168					else
169					{
170						parseError( "Expecting 'null' but found " + possibleNull );
171					}
172					
173					break;
174					
175				case 'N': // attempt to read NaN
176					var possibleNaN:String = "N" + nextChar() + nextChar();
177					
178					if ( possibleNaN == "NaN" )
179					{
180						token.type = JSONTokenType.NAN;
181						token.value = NaN;
182						nextChar();
183					}
184					else
185					{
186						parseError( "Expecting 'NaN' but found " + possibleNaN );
187					}
188					
189					break;
190					
191				case '"': // the start of a string
192					token = readString();
193					break;
194					
195				default: 
196					// see if we can read a number
197					if ( isDigit( ch ) || ch == '-' )
198					{
199						token = readNumber();
200					}
201					else if ( ch == '' )
202					{
203						// check for reading past the end of the string
204						return null;
205					}
206					else
207					{						
208						// not sure what was in the input string - it's not
209						// anything we expected
210						parseError( "Unexpected " + ch + " encountered" );
211					}
212			}
213			
214			return token;
215		}
216		
217		/**
218		 * Attempts to read a string from the input string.  Places
219		 * the character location at the first character after the
220		 * string.  It is assumed that ch is " before this method is called.
221		 *
222		 * @return the JSONToken with the string value if a string could
223		 *		be read.  Throws an error otherwise.
224		 */
225		private function readString():JSONToken
226		{
227			// the string to store the string we'll try to read
228			var string:String = "";
229			
230			// advance past the first "
231			nextChar();
232			
233			while ( ch != '"' && ch != '' )
234			{					
235				// unescape the escape sequences in the string
236				if ( ch == '\\' )
237				{	
238					// get the next character so we know what
239					// to unescape
240					nextChar();
241					
242					switch ( ch )
243					{	
244						case '"': // quotation mark
245							string += '"';
246							break;
247						
248						case '/':	// solidus
249							string += "/";
250							break;
251							
252						case '\\':	// reverse solidus
253							string += '\\';
254							break;
255							
256						case 'b':	// bell
257							string += '\b';
258							break;
259							
260						case 'f':	// form feed
261							string += '\f';
262							break;
263							
264						case 'n':	// newline
265							string += '\n';
266							break;
267							
268						case 'r':	// carriage return
269							string += '\r';
270							break;
271							
272						case 't':	// horizontal tab
273							string += '\t'
274							break;
275						
276						case 'u':
277							// convert a unicode escape sequence
278							// to it's character value - expecting
279							// 4 hex digits
280							
281							// save the characters as a string we'll convert to an int
282							var hexValue:String = "";
283							
284							// try to find 4 hex characters
285							for ( var i:int = 0; i < 4; i++ )
286							{
287								// get the next character and determine
288								// if it's a valid hex digit or not
289								if ( !isHexDigit( nextChar() ) )
290								{
291									parseError( " Excepted a hex digit, but found: " + ch );
292								}
293								// valid, add it to the value
294								hexValue += ch;
295							}
296							
297							// convert hexValue to an integer, and use that
298							// integrer value to create a character to add
299							// to our string.
300							string += String.fromCharCode( parseInt( hexValue, 16 ) );
301							
302							break;
303					
304						default:
305							// couldn't unescape the sequence, so just
306							// pass it through
307							string += '\\' + ch;
308						
309					}
310					
311				}
312				else
313				{
314					// didn't have to unescape, so add the character to the string
315					string += ch;
316					
317				}
318				
319				// move to the next character
320				nextChar();
321			}
322			
323			// we read past the end of the string without closing it, which
324			// is a parse error
325			if ( ch == '' )
326			{
327				parseError( "Unterminated string literal" );
328			}
329			
330			// move past the closing " in the input string
331			nextChar();
332			
333			// the token for the string we've just read
334			var token:JSONToken = new JSONToken();
335			token.type = JSONTokenType.STRING;
336			// attach to the string to the token so we can return it
337			token.value = string;
338			
339			return token;
340		}
341		
342		/**
343		 * Attempts to read a number from the input string.  Places
344		 * the character location at the first character after the
345		 * number.
346		 * 
347		 * @return The JSONToken with the number value if a number could
348		 * 		be read.  Throws an error otherwise.
349		 */
350		private function readNumber():JSONToken
351		{
352			// the string to accumulate the number characters
353			// into that we'll convert to a number at the end
354			var input:String = "";
355			
356			// check for a negative number
357			if ( ch == '-' )
358			{
359				input += '-';
360				nextChar();
361			}
362			
363			// the number must start with a digit
364			if ( !isDigit( ch ) )
365			{
366				parseError( "Expecting a digit" );
367			}
368			
369			// 0 can only be the first digit if it
370			// is followed by a decimal point
371			if ( ch == '0' )
372			{
373				input += ch;
374				nextChar();
375				
376				// make sure no other digits come after 0
377				if ( isDigit( ch ) )
378				{
379					parseError( "A digit cannot immediately follow 0" );
380				}
381				// unless we have 0x which starts a hex number, but this
382				// doesn't match JSON spec so check for not strict mode.
383				else if ( !strict && ch == 'x' )
384				{
385					// include the x in the input
386					input += ch;
387					nextChar();
388					
389					// need at least one hex digit after 0x to
390					// be valid
391					if ( isHexDigit( ch ) )
392					{
393						input += ch;
394						nextChar();
395					}
396					else
397					{
398						parseError( "Number in hex format require at least one hex digit after \"0x\"" );	
399					}
400					
401					// consume all of the hex values
402					while ( isHexDigit( ch ) )
403					{
404						input += ch;
405						nextChar();
406					}
407				}
408			}
409			else
410			{
411				// read numbers while we can
412				while ( isDigit( ch ) )
413				{
414					input += ch;
415					nextChar();
416				}
417			}
418			
419			// check for a decimal value
420			if ( ch == '.' )
421			{
422				input += '.';
423				nextChar();
424				
425				// after the decimal there has to be a digit
426				if ( !isDigit( ch ) )
427				{
428					parseError( "Expecting a digit" );
429				}
430				
431				// read more numbers to get the decimal value
432				while ( isDigit( ch ) )
433				{
434					input += ch;
435					nextChar();
436				}
437			}
438			
439			// check for scientific notation
440			if ( ch == 'e' || ch == 'E' )
441			{
442				input += "e"
443				nextChar();
444				// check for sign
445				if ( ch == '+' || ch == '-' )
446				{
447					input += ch;
448					nextChar();
449				}
450				
451				// require at least one number for the exponent
452				// in this case
453				if ( !isDigit( ch ) )
454				{
455					parseError( "Scientific notation number needs exponent value" );
456				}
457							
458				// read in the exponent
459				while ( isDigit( ch ) )
460				{
461					input += ch;
462					nextChar();
463				}
464			}
465			
466			// convert the string to a number value
467			var num:Number = Number( input );
468			
469			if ( isFinite( num ) && !isNaN( num ) )
470			{
471				// the token for the number that we've read
472				var token:JSONToken = new JSONToken();
473				token.type = JSONTokenType.NUMBER;
474				token.value = num;
475				return token;
476			}
477			else
478			{
479				parseError( "Number " + num + " is not valid!" );
480			}
481			
482            return null;
483		}
484
485		/**
486		 * Reads the next character in the input
487		 * string and advances the character location.
488		 *
489		 * @return The next character in the input string, or
490		 *		null if we've read past the end.
491		 */
492		private function nextChar():String
493		{
494			return ch = jsonString.charAt( loc++ );
495		}
496		
497		/**
498		 * Advances the character location past any
499		 * sort of white space and comments
500		 */
501		private function skipIgnored():void
502		{
503			var originalLoc:int;
504			
505			// keep trying to skip whitespace and comments as long
506			// as we keep advancing past the original location 
507			do
508			{
509				originalLoc = loc;
510				skipWhite();
511				skipComments();
512			}
513			while ( originalLoc != loc );
514		}
515		
516		/**
517		 * Skips comments in the input string, either
518		 * single-line or multi-line.  Advances the character
519		 * to the first position after the end of the comment.
520		 */
521		private function skipComments():void
522		{
523			if ( ch == '/' )
524			{
525				// Advance past the first / to find out what type of comment
526				nextChar();
527				switch ( ch )
528				{
529					case '/': // single-line comment, read through end of line
530						
531						// Loop over the characters until we find
532						// a newline or until there's no more characters left
533						do
534						{
535							nextChar();
536						}
537						while ( ch != '\n' && ch != '' )
538						
539						// move past the \n
540						nextChar();
541						
542						break;
543					
544					case '*': // multi-line comment, read until closing */
545
546						// move past the opening *
547						nextChar();
548						
549						// try to find a trailing */
550						while ( true )
551						{
552							if ( ch == '*' )
553							{
554								// check to see if we have a closing /
555								nextChar();
556								if ( ch == '/')
557								{
558									// move past the end of the closing */
559									nextChar();
560									break;
561								}
562							}
563							else
564							{
565								// move along, looking if the next character is a *
566								nextChar();
567							}
568							
569							// when we're here we've read past the end of 
570							// the string without finding a closing */, so error
571							if ( ch == '' )
572							{
573								parseError( "Multi-line comment not closed" );
574							}
575						}
576
577						break;
578					
579					// Can't match a comment after a /, so it's a parsing error
580					default:
581						parseError( "Unexpected " + ch + " encountered (expecting '/' or '*' )" );
582				}
583			}
584			
585		}
586		
587		
588		/**
589		 * Skip any whitespace in the input string and advances
590		 * the character to the first character after any possible
591		 * whitespace.
592		 */
593		private function skipWhite():void
594		{	
595			// As long as there are spaces in the input 
596			// stream, advance the current location pointer
597			// past them
598			while ( isWhiteSpace( ch ) )
599			{
600				nextChar();
601			}
602			
603		}
604		
605		/**
606		 * Determines if a character is whitespace or not.
607		 *
608		 * @return True if the character passed in is a whitespace
609		 *	character
610		 */
611		private function isWhiteSpace( ch:String ):Boolean
612		{
613			return ( ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' );
614		}
615		
616		/**
617		 * Determines if a character is a digit [0-9].
618		 *
619		 * @return True if the character passed in is a digit
620		 */
621		private function isDigit( ch:String ):Boolean
622		{
623			return ( ch >= '0' && ch <= '9' );
624		}
625		
626		/**
627		 * Determines if a character is a digit [0-9].
628		 *
629		 * @return True if the character passed in is a digit
630		 */
631		private function isHexDigit( ch:String ):Boolean
632		{
633			// get the uppercase value of ch so we only have
634			// to compare the value between 'A' and 'F'
635			var uc:String = ch.toUpperCase();
636			
637			// a hex digit is a digit of A-F, inclusive ( using
638			// our uppercase constraint )
639			return ( isDigit( ch ) || ( uc >= 'A' && uc <= 'F' ) );
640		}
641	
642		/**
643		 * Raises a parsing error with a specified message, tacking
644		 * on the error location and the original string.
645		 *
646		 * @param message The message indicating why the error occurred
647		 */
648		public function parseError( message:String ):void
649		{
650			throw new JSONParseError( message, loc, jsonString );
651		}
652	}
653	
654}