/NRefactory/ICSharpCode.NRefactory.CSharp/Parser/mcs/cs-tokenizer.cs

Large files files are truncated, but you can click here to view the full file

   1//
   2// cs-tokenizer.cs: The Tokenizer for the C# compiler
   3//                  This also implements the preprocessor
   4//
   5// Author: Miguel de Icaza (miguel@gnu.org)
   6//         Marek Safar (marek.safar@gmail.com)
   7//
   8// Dual licensed under the terms of the MIT X11 or GNU GPL
   9//
  10// Copyright 2001, 2002 Ximian, Inc (http://www.ximian.com)
  11// Copyright 2004-2008 Novell, Inc
  12// Copyright 2011 Xamarin, Inc (http://www.xamarin.com)
  13//
  14using System;
  15using System.Text;
  16using System.Collections.Generic;
  17using System.Globalization;
  18using System.Diagnostics;
  19using System.Collections;
  20
  21namespace Mono.CSharp
  22{
  23	//
  24	// This class has to be used by parser only, it reuses token
  25	// details once a file is parsed
  26	//
  27	public class LocatedToken
  28	{
  29		public int row, column;
  30		public string value;
  31		public SourceFile file;
  32
  33		public LocatedToken ()
  34		{
  35		}
  36
  37		public LocatedToken (string value, Location loc)
  38		{
  39			this.value = value;
  40			file = loc.SourceFile;
  41			row = loc.Row;
  42			column = loc.Column;
  43		}
  44
  45		public override string ToString ()
  46		{
  47			return string.Format ("Token '{0}' at {1},{2}", Value, row, column);
  48		}
  49
  50		public Location Location
  51		{
  52			get { return new Location (file, row, column); }
  53		}
  54
  55		public string Value
  56		{
  57			get { return value; }
  58		}
  59	}
  60
  61	/// <summary>
  62	///    Tokenizer for C# source code. 
  63	/// </summary>
  64	public class Tokenizer : yyParser.yyInput
  65	{
  66		class KeywordEntry<T>
  67		{
  68			public readonly T Token;
  69			public KeywordEntry<T> Next;
  70			public readonly char[] Value;
  71
  72			public KeywordEntry (string value,T token)
  73			{
  74				this.Value = value.ToCharArray ();
  75				this.Token = token;
  76			}
  77		}
  78
  79		sealed class IdentifiersComparer : IEqualityComparer<char[]>
  80		{
  81			readonly int length;
  82
  83			public IdentifiersComparer (int length)
  84			{
  85				this.length = length;
  86			}
  87
  88			public bool Equals (char[] x, char[] y)
  89			{
  90				for (int i = 0; i < length; ++i)
  91					if (x [i] != y [i])
  92						return false;
  93
  94				return true;
  95			}
  96
  97			public int GetHashCode (char[] obj)
  98			{
  99				int h = 0;
 100				for (int i = 0; i < length; ++i)
 101					h = (h << 5) - h + obj [i];
 102
 103				return h;
 104			}
 105		}
 106
 107		public class LocatedTokenBuffer
 108		{
 109			readonly LocatedToken[] buffer;
 110			public int pos;
 111
 112			public LocatedTokenBuffer ()
 113			{
 114				buffer = new LocatedToken[0];
 115			}
 116
 117			public LocatedTokenBuffer (LocatedToken[] buffer)
 118			{
 119				this.buffer = buffer ?? new LocatedToken[0];
 120			}
 121
 122			public LocatedToken Create (SourceFile file, int row, int column)
 123			{
 124				return Create (null, file, row, column);
 125			}
 126
 127			public LocatedToken Create (string value, SourceFile file, int row, int column)
 128			{
 129				//
 130				// TODO: I am not very happy about the logic but it's the best
 131				// what I could come up with for now.
 132				// Ideally we should be using just tiny buffer (256 elements) which
 133				// is enough to hold all details for currect stack and recycle elements
 134				// poped from the stack but there is a trick needed to recycle
 135				// them properly.
 136				//
 137				LocatedToken entry;
 138				if (pos >= buffer.Length) {
 139					entry = new LocatedToken ();
 140				} else {
 141					entry = buffer[pos];
 142					if (entry == null) {
 143						entry = new LocatedToken ();
 144						buffer[pos] = entry;
 145					}
 146
 147					++pos;
 148				}
 149				entry.value = value;
 150				entry.file = file;
 151				entry.row = row;
 152				entry.column = column;
 153				return entry;
 154			}
 155
 156			//
 157			// Used for token not required by expression evaluator
 158			//
 159			[Conditional ("FULL_AST")]
 160			public void CreateOptional (SourceFile file, int row, int col, ref object token)
 161			{
 162				token = Create (file, row, col);
 163			}
 164		}
 165
 166		public enum PreprocessorDirective
 167		{
 168			Invalid = 0,
 169
 170			Region = 1,
 171			Endregion = 2,
 172			If = 3 | RequiresArgument,
 173			Endif = 4,
 174			Elif = 5 | RequiresArgument,
 175			Else = 6,
 176			Define = 7 | RequiresArgument,
 177			Undef = 8 | RequiresArgument,
 178			Error = 9,
 179			Warning = 10,
 180			Pragma = 11 | CustomArgumentsParsing,
 181			Line = 12 | CustomArgumentsParsing,
 182
 183			CustomArgumentsParsing = 1 << 10,
 184			RequiresArgument = 1 << 11
 185		}
 186
 187		readonly SeekableStreamReader reader;
 188		readonly CompilationSourceFile source_file;
 189		public CompilationSourceFile SourceFile { get { return source_file; } }
 190		readonly CompilerContext context;
 191		readonly Report Report;
 192
 193
 194		SourceFile current_source;
 195		Location hidden_block_start;
 196		int ref_line = 1;
 197		int line = 1;
 198		int col = 0;
 199		int previous_col;
 200		int current_token;
 201		readonly int tab_size;
 202		bool handle_get_set = false;
 203		bool handle_remove_add = false;
 204		bool handle_where;
 205		bool lambda_arguments_parsing;
 206		List<Location> escaped_identifiers;
 207		int parsing_generic_less_than;
 208		readonly bool doc_processing;
 209		readonly LocatedTokenBuffer ltb;
 210		
 211		//
 212		// Used mainly for parser optimizations. Some expressions for instance
 213		// can appear only in block (including initializer, base initializer)
 214		// scope only
 215		//
 216		public int parsing_block;
 217		internal bool query_parsing;
 218		
 219		// 
 220		// When parsing type only, useful for ambiguous nullable types
 221		//
 222		public int parsing_type;
 223		
 224		//
 225		// Set when parsing generic declaration (type or method header)
 226		//
 227		public bool parsing_generic_declaration;
 228		public bool parsing_generic_declaration_doc;
 229		
 230		//
 231		// The value indicates that we have not reach any declaration or
 232		// namespace yet
 233		//
 234		public int parsing_declaration;
 235		public bool parsing_attribute_section;
 236
 237		public bool parsing_modifiers;
 238
 239		//
 240		// The special characters to inject on streams to run the unit parser
 241		// in the special expression mode. Using private characters from
 242		// Plane Sixteen (U+100000 to U+10FFFD)
 243		//
 244		// This character is only tested just before the tokenizer is about to report
 245		// an error;   So on the regular operation mode, this addition will have no
 246		// impact on the tokenizer's performance.
 247		//
 248		
 249		public const int EvalStatementParserCharacter = 0x100000;
 250		public const int EvalCompilationUnitParserCharacter = 0x100001;
 251		public const int EvalUsingDeclarationsParserCharacter = 0x100002;
 252		public const int DocumentationXref = 0x100003;
 253
 254		const int UnicodeLS = 0x2028;
 255		const int UnicodePS = 0x2029;
 256		
 257		//
 258		// XML documentation buffer. The save point is used to divide
 259		// comments on types and comments on members.
 260		//
 261		StringBuilder xml_comment_buffer;
 262
 263		//
 264		// See comment on XmlCommentState enumeration.
 265		//
 266		XmlCommentState xml_doc_state = XmlCommentState.Allowed;
 267
 268		//
 269		// Whether tokens have been seen on this line
 270		//
 271		bool tokens_seen = false;
 272
 273		//
 274		// Set to true once the GENERATE_COMPLETION token has bee
 275		// returned.   This helps produce one GENERATE_COMPLETION,
 276		// as many COMPLETE_COMPLETION as necessary to complete the
 277		// AST tree and one final EOF.
 278		//
 279		bool generated;
 280		
 281		//
 282		// Whether a token has been seen on the file
 283		// This is needed because `define' is not allowed to be used
 284		// after a token has been seen.
 285		//
 286		bool any_token_seen;
 287
 288		//
 289		// Class variables
 290		// 
 291		static readonly KeywordEntry<int>[][] keywords;
 292		static readonly KeywordEntry<PreprocessorDirective>[][] keywords_preprocessor;
 293		static readonly HashSet<string> keyword_strings;
 294		static readonly NumberStyles styles;
 295		static readonly NumberFormatInfo csharp_format_info;
 296
 297		// Pragma arguments
 298		static readonly char[] pragma_warning = "warning".ToCharArray ();
 299		static readonly char[] pragma_warning_disable = "disable".ToCharArray ();
 300		static readonly char[] pragma_warning_restore = "restore".ToCharArray ();
 301		static readonly char[] pragma_checksum = "checksum".ToCharArray ();
 302		static readonly char[] line_hidden = "hidden".ToCharArray ();
 303		static readonly char[] line_default = "default".ToCharArray ();
 304
 305		static readonly char[] simple_whitespaces = new char[] { ' ', '\t' };
 306		bool startsLine = true;
 307		internal SpecialsBag sbag;
 308
 309		public bool PropertyParsing {
 310			get { return handle_get_set; }
 311			set { handle_get_set = value; }
 312		}
 313
 314		public bool EventParsing {
 315			get { return handle_remove_add; }
 316			set { handle_remove_add = value; }
 317		}
 318
 319		public bool ConstraintsParsing {
 320			get { return handle_where; }
 321			set { handle_where = value; }
 322		}
 323	
 324		public XmlCommentState doc_state {
 325			get { return xml_doc_state; }
 326			set {
 327				if (value == XmlCommentState.Allowed) {
 328					check_incorrect_doc_comment ();
 329					reset_doc_comment ();
 330				}
 331				xml_doc_state = value;
 332			}
 333		}
 334
 335		//
 336		// This is used to trigger completion generation on the parser
 337		public bool CompleteOnEOF;
 338
 339		void AddEscapedIdentifier (Location loc)
 340		{
 341			if (escaped_identifiers == null)
 342				escaped_identifiers = new List<Location> ();
 343
 344			escaped_identifiers.Add (loc);
 345		}
 346
 347		public bool IsEscapedIdentifier (ATypeNameExpression name)
 348		{
 349			return escaped_identifiers != null && escaped_identifiers.Contains (name.Location);
 350		}
 351
 352		//
 353		// Values for the associated token returned
 354		//
 355		internal int putback_char; 	// Used by repl only
 356		object val;
 357
 358		//
 359		// Pre-processor
 360		//
 361		const int TAKING        = 1;
 362		const int ELSE_SEEN = 4;
 363		const int PARENT_TAKING = 8;
 364		const int REGION = 16;		
 365
 366		//
 367		// pre-processor if stack state:
 368		//
 369		Stack<int> ifstack;
 370
 371		public const int MaxIdentifierLength = 512;
 372		public const int MaxNumberLength = 512;
 373
 374		readonly char[] id_builder;
 375		readonly Dictionary<char[], string>[] identifiers;
 376		readonly char[] number_builder;
 377		int number_pos;
 378
 379		char[] value_builder = new char[64];
 380
 381		public int Line {
 382			get {
 383				return ref_line;
 384			}
 385			set {
 386				ref_line = value;
 387			}
 388		}
 389
 390		public int Column {
 391			get {
 392				return col;
 393			}
 394			set {
 395				col = value;
 396			}
 397		}
 398
 399		//
 400		// This is used when the tokenizer needs to save
 401		// the current position as it needs to do some parsing
 402		// on its own to deamiguate a token in behalf of the
 403		// parser.
 404		//
 405		Stack<Position> position_stack = new Stack<Position> (2);
 406
 407		class Position
 408		{
 409			public int position;
 410			public int line;
 411			public int ref_line;
 412			public int col;
 413			public Location hidden;
 414			public int putback_char;
 415			public int previous_col;
 416			public Stack<int> ifstack;
 417			public int parsing_generic_less_than;
 418			public int current_token;
 419			public object val;
 420
 421			public Position (Tokenizer t)
 422			{
 423				position = t.reader.Position;
 424				line = t.line;
 425				ref_line = t.ref_line;
 426				col = t.col;
 427				hidden = t.hidden_block_start;
 428				putback_char = t.putback_char;
 429				previous_col = t.previous_col;
 430				if (t.ifstack != null && t.ifstack.Count != 0) {
 431					// There is no simple way to clone Stack<T> all
 432					// methods reverse the order
 433					var clone = t.ifstack.ToArray ();
 434					Array.Reverse (clone);
 435					ifstack = new Stack<int> (clone);
 436				}
 437				parsing_generic_less_than = t.parsing_generic_less_than;
 438				current_token = t.current_token;
 439				val = t.val;
 440			}
 441		}
 442
 443		public Tokenizer (SeekableStreamReader input, CompilationSourceFile file, ParserSession session, Report report)
 444		{
 445			this.source_file = file;
 446			this.context = file.Compiler;
 447			this.current_source = file.SourceFile;
 448			this.identifiers = session.Identifiers;
 449			this.id_builder = session.IDBuilder;
 450			this.number_builder = session.NumberBuilder;
 451			this.ltb = new LocatedTokenBuffer (session.LocatedTokens);
 452			this.Report = report;
 453
 454			reader = input;
 455
 456			putback_char = -1;
 457
 458			xml_comment_buffer = new StringBuilder ();
 459			doc_processing = context.Settings.DocumentationFile != null;
 460
 461			tab_size = context.Settings.TabSize;
 462		}
 463		
 464		public void PushPosition ()
 465		{
 466			position_stack.Push (new Position (this));
 467		}
 468
 469		public void PopPosition ()
 470		{
 471			Position p = position_stack.Pop ();
 472
 473			reader.Position = p.position;
 474			ref_line = p.ref_line;
 475			line = p.line;
 476			col = p.col;
 477			hidden_block_start = p.hidden;
 478			putback_char = p.putback_char;
 479			previous_col = p.previous_col;
 480			ifstack = p.ifstack;
 481			parsing_generic_less_than = p.parsing_generic_less_than;
 482			current_token = p.current_token;
 483			val = p.val;
 484		}
 485
 486		// Do not reset the position, ignore it.
 487		public void DiscardPosition ()
 488		{
 489			position_stack.Pop ();
 490	}
 491		
 492		static void AddKeyword (string kw, int token)
 493		{
 494			keyword_strings.Add (kw);
 495
 496			AddKeyword (keywords, kw, token);
 497}
 498
 499		static void AddPreprocessorKeyword (string kw, PreprocessorDirective directive)
 500		{
 501			AddKeyword (keywords_preprocessor, kw, directive);
 502		}
 503
 504		static void AddKeyword<T> (KeywordEntry<T>[][] keywords, string kw, T token)
 505		{
 506			int length = kw.Length;
 507			if (keywords[length] == null) {
 508				keywords[length] = new KeywordEntry<T>['z' - '_' + 1];
 509			}
 510
 511			int char_index = kw[0] - '_';
 512			var kwe = keywords[length][char_index];
 513			if (kwe == null) {
 514				keywords[length][char_index] = new KeywordEntry<T> (kw, token);
 515				return;
 516			}
 517
 518			while (kwe.Next != null) {
 519				kwe = kwe.Next;
 520			}
 521
 522			kwe.Next = new KeywordEntry<T> (kw, token);
 523		}
 524
 525		//
 526		// Class initializer
 527		// 
 528		static Tokenizer ()
 529		{
 530			keyword_strings = new HashSet<string> ();
 531
 532			// 11 is the length of the longest keyword for now
 533			keywords = new KeywordEntry<int>[11][];
 534
 535			AddKeyword ("__arglist", Token.ARGLIST);
 536			AddKeyword ("__makeref", Token.MAKEREF);
 537			AddKeyword ("__reftype", Token.REFTYPE);
 538			AddKeyword ("__refvalue", Token.REFVALUE);
 539			AddKeyword ("abstract", Token.ABSTRACT);
 540			AddKeyword ("as", Token.AS);
 541			AddKeyword ("add", Token.ADD);
 542			AddKeyword ("base", Token.BASE);
 543			AddKeyword ("bool", Token.BOOL);
 544			AddKeyword ("break", Token.BREAK);
 545			AddKeyword ("byte", Token.BYTE);
 546			AddKeyword ("case", Token.CASE);
 547			AddKeyword ("catch", Token.CATCH);
 548			AddKeyword ("char", Token.CHAR);
 549			AddKeyword ("checked", Token.CHECKED);
 550			AddKeyword ("class", Token.CLASS);
 551			AddKeyword ("const", Token.CONST);
 552			AddKeyword ("continue", Token.CONTINUE);
 553			AddKeyword ("decimal", Token.DECIMAL);
 554			AddKeyword ("default", Token.DEFAULT);
 555			AddKeyword ("delegate", Token.DELEGATE);
 556			AddKeyword ("do", Token.DO);
 557			AddKeyword ("double", Token.DOUBLE);
 558			AddKeyword ("else", Token.ELSE);
 559			AddKeyword ("enum", Token.ENUM);
 560			AddKeyword ("event", Token.EVENT);
 561			AddKeyword ("explicit", Token.EXPLICIT);
 562			AddKeyword ("extern", Token.EXTERN);
 563			AddKeyword ("false", Token.FALSE);
 564			AddKeyword ("finally", Token.FINALLY);
 565			AddKeyword ("fixed", Token.FIXED);
 566			AddKeyword ("float", Token.FLOAT);
 567			AddKeyword ("for", Token.FOR);
 568			AddKeyword ("foreach", Token.FOREACH);
 569			AddKeyword ("goto", Token.GOTO);
 570			AddKeyword ("get", Token.GET);
 571			AddKeyword ("if", Token.IF);
 572			AddKeyword ("implicit", Token.IMPLICIT);
 573			AddKeyword ("in", Token.IN);
 574			AddKeyword ("int", Token.INT);
 575			AddKeyword ("interface", Token.INTERFACE);
 576			AddKeyword ("internal", Token.INTERNAL);
 577			AddKeyword ("is", Token.IS);
 578			AddKeyword ("lock", Token.LOCK);
 579			AddKeyword ("long", Token.LONG);
 580			AddKeyword ("namespace", Token.NAMESPACE);
 581			AddKeyword ("new", Token.NEW);
 582			AddKeyword ("null", Token.NULL);
 583			AddKeyword ("object", Token.OBJECT);
 584			AddKeyword ("operator", Token.OPERATOR);
 585			AddKeyword ("out", Token.OUT);
 586			AddKeyword ("override", Token.OVERRIDE);
 587			AddKeyword ("params", Token.PARAMS);
 588			AddKeyword ("private", Token.PRIVATE);
 589			AddKeyword ("protected", Token.PROTECTED);
 590			AddKeyword ("public", Token.PUBLIC);
 591			AddKeyword ("readonly", Token.READONLY);
 592			AddKeyword ("ref", Token.REF);
 593			AddKeyword ("remove", Token.REMOVE);
 594			AddKeyword ("return", Token.RETURN);
 595			AddKeyword ("sbyte", Token.SBYTE);
 596			AddKeyword ("sealed", Token.SEALED);
 597			AddKeyword ("set", Token.SET);
 598			AddKeyword ("short", Token.SHORT);
 599			AddKeyword ("sizeof", Token.SIZEOF);
 600			AddKeyword ("stackalloc", Token.STACKALLOC);
 601			AddKeyword ("static", Token.STATIC);
 602			AddKeyword ("string", Token.STRING);
 603			AddKeyword ("struct", Token.STRUCT);
 604			AddKeyword ("switch", Token.SWITCH);
 605			AddKeyword ("this", Token.THIS);
 606			AddKeyword ("throw", Token.THROW);
 607			AddKeyword ("true", Token.TRUE);
 608			AddKeyword ("try", Token.TRY);
 609			AddKeyword ("typeof", Token.TYPEOF);
 610			AddKeyword ("uint", Token.UINT);
 611			AddKeyword ("ulong", Token.ULONG);
 612			AddKeyword ("unchecked", Token.UNCHECKED);
 613			AddKeyword ("unsafe", Token.UNSAFE);
 614			AddKeyword ("ushort", Token.USHORT);
 615			AddKeyword ("using", Token.USING);
 616			AddKeyword ("virtual", Token.VIRTUAL);
 617			AddKeyword ("void", Token.VOID);
 618			AddKeyword ("volatile", Token.VOLATILE);
 619			AddKeyword ("while", Token.WHILE);
 620			AddKeyword ("partial", Token.PARTIAL);
 621			AddKeyword ("where", Token.WHERE);
 622
 623			// LINQ keywords
 624			AddKeyword ("from", Token.FROM);
 625			AddKeyword ("join", Token.JOIN);
 626			AddKeyword ("on", Token.ON);
 627			AddKeyword ("equals", Token.EQUALS);
 628			AddKeyword ("select", Token.SELECT);
 629			AddKeyword ("group", Token.GROUP);
 630			AddKeyword ("by", Token.BY);
 631			AddKeyword ("let", Token.LET);
 632			AddKeyword ("orderby", Token.ORDERBY);
 633			AddKeyword ("ascending", Token.ASCENDING);
 634			AddKeyword ("descending", Token.DESCENDING);
 635			AddKeyword ("into", Token.INTO);
 636
 637			// Contextual async keywords
 638			AddKeyword ("async", Token.ASYNC);
 639			AddKeyword ("await", Token.AWAIT);
 640
 641			keywords_preprocessor = new KeywordEntry<PreprocessorDirective>[10][];
 642
 643			AddPreprocessorKeyword ("region", PreprocessorDirective.Region);
 644			AddPreprocessorKeyword ("endregion", PreprocessorDirective.Endregion);
 645			AddPreprocessorKeyword ("if", PreprocessorDirective.If);
 646			AddPreprocessorKeyword ("endif", PreprocessorDirective.Endif);
 647			AddPreprocessorKeyword ("elif", PreprocessorDirective.Elif);
 648			AddPreprocessorKeyword ("else", PreprocessorDirective.Else);
 649			AddPreprocessorKeyword ("define", PreprocessorDirective.Define);
 650			AddPreprocessorKeyword ("undef", PreprocessorDirective.Undef);
 651			AddPreprocessorKeyword ("error", PreprocessorDirective.Error);
 652			AddPreprocessorKeyword ("warning", PreprocessorDirective.Warning);
 653			AddPreprocessorKeyword ("pragma", PreprocessorDirective.Pragma);
 654			AddPreprocessorKeyword ("line", PreprocessorDirective.Line);
 655
 656			csharp_format_info = NumberFormatInfo.InvariantInfo;
 657			styles = NumberStyles.Float;
 658		}
 659
 660		int GetKeyword (char[] id, int id_len)
 661		{
 662			//
 663			// Keywords are stored in an array of arrays grouped by their
 664			// length and then by the first character
 665			//
 666			if (id_len >= keywords.Length || keywords [id_len] == null)
 667				return -1;
 668
 669			int first_index = id [0] - '_';
 670			if (first_index > 'z' - '_')
 671				return -1;
 672
 673			var kwe = keywords [id_len] [first_index];
 674			if (kwe == null)
 675				return -1;
 676
 677			int res;
 678			do {
 679				res = kwe.Token;
 680				for (int i = 1; i < id_len; ++i) {
 681					if (id [i] != kwe.Value [i]) {
 682						res = 0;
 683						kwe = kwe.Next;
 684						break;
 685					}
 686				}
 687			} while (res == 0 && kwe != null);
 688
 689			if (res == 0)
 690				return -1;
 691
 692			int next_token;
 693			switch (res) {
 694			case Token.GET:
 695			case Token.SET:
 696				if (!handle_get_set)
 697					res = -1;
 698				break;
 699			case Token.REMOVE:
 700			case Token.ADD:
 701				if (!handle_remove_add)
 702					res = -1;
 703				break;
 704			case Token.EXTERN:
 705				if (parsing_declaration == 0)
 706					res = Token.EXTERN_ALIAS;
 707				break;
 708			case Token.DEFAULT:
 709				if (peek_token () == Token.COLON) {
 710					token ();
 711					res = Token.DEFAULT_COLON;
 712				}
 713				break;
 714			case Token.WHERE:
 715				if (!(handle_where && current_token != Token.COLON) && !query_parsing)
 716					res = -1;
 717				break;
 718			case Token.FROM:
 719				//
 720				// A query expression is any expression that starts with `from identifier'
 721				// followed by any token except ; , =
 722				// 
 723				if (!query_parsing) {
 724					if (lambda_arguments_parsing || parsing_block == 0) {
 725						res = -1;
 726						break;
 727					}
 728
 729					PushPosition ();
 730					// HACK: to disable generics micro-parser, because PushPosition does not
 731					// store identifiers array
 732					parsing_generic_less_than = 1;
 733					switch (xtoken ()) {
 734					case Token.IDENTIFIER:
 735					case Token.INT:
 736					case Token.BOOL:
 737					case Token.BYTE:
 738					case Token.CHAR:
 739					case Token.DECIMAL:
 740					case Token.DOUBLE:
 741					case Token.FLOAT:
 742					case Token.LONG:
 743					case Token.OBJECT:
 744					case Token.STRING:
 745					case Token.UINT:
 746					case Token.ULONG:
 747						next_token = xtoken ();
 748						if (next_token == Token.SEMICOLON || next_token == Token.COMMA || next_token == Token.EQUALS || next_token == Token.ASSIGN)
 749							goto default;
 750						
 751						res = Token.FROM_FIRST;
 752						query_parsing = true;
 753						if (context.Settings.Version <= LanguageVersion.ISO_2)
 754							Report.FeatureIsNotAvailable (context, Location, "query expressions");
 755						break;
 756					case Token.VOID:
 757						Expression.Error_VoidInvalidInTheContext (Location, Report);
 758						break;
 759					default:
 760						PopPosition ();
 761						// HACK: A token is not a keyword so we need to restore identifiers buffer
 762						// which has been overwritten before we grabbed the identifier
 763						id_builder [0] = 'f'; id_builder [1] = 'r'; id_builder [2] = 'o'; id_builder [3] = 'm';
 764						return -1;
 765					}
 766					PopPosition ();
 767				}
 768				break;
 769			case Token.JOIN:
 770			case Token.ON:
 771			case Token.EQUALS:
 772			case Token.SELECT:
 773			case Token.GROUP:
 774			case Token.BY:
 775			case Token.LET:
 776			case Token.ORDERBY:
 777			case Token.ASCENDING:
 778			case Token.DESCENDING:
 779			case Token.INTO:
 780				if (!query_parsing)
 781					res = -1;
 782				break;
 783				
 784			case Token.USING:
 785			case Token.NAMESPACE:
 786				// TODO: some explanation needed
 787				check_incorrect_doc_comment ();
 788				parsing_modifiers = false;
 789				break;
 790				
 791			case Token.PARTIAL:
 792				if (parsing_block > 0) {
 793					res = -1;
 794					break;
 795				}
 796
 797				// Save current position and parse next token.
 798				PushPosition ();
 799
 800				next_token = token ();
 801				bool ok = (next_token == Token.CLASS) ||
 802					(next_token == Token.STRUCT) ||
 803					(next_token == Token.INTERFACE) ||
 804					(next_token == Token.VOID);
 805
 806				PopPosition ();
 807
 808				if (ok) {
 809					if (next_token == Token.VOID) {
 810						if (context.Settings.Version <= LanguageVersion.ISO_2)
 811							Report.FeatureIsNotAvailable (context, Location, "partial methods");
 812					} else if (context.Settings.Version == LanguageVersion.ISO_1)
 813						Report.FeatureIsNotAvailable (context, Location, "partial types");
 814
 815					return res;
 816				}
 817
 818				if (next_token < Token.LAST_KEYWORD) {
 819					Report.Error (267, Location,
 820						"The `partial' modifier can be used only immediately before `class', `struct', `interface', or `void' keyword");
 821					return token ();
 822				}
 823
 824				// HACK: A token is not a keyword so we need to restore identifiers buffer
 825				// which has been overwritten before we grabbed the identifier
 826				id_builder[0] = 'p';
 827				id_builder[1] = 'a';
 828				id_builder[2] = 'r';
 829				id_builder[3] = 't';
 830				id_builder[4] = 'i';
 831				id_builder[5] = 'a';
 832				id_builder[6] = 'l';
 833				res = -1;
 834				break;
 835
 836			case Token.ASYNC:
 837				if (parsing_modifiers) {
 838					//
 839					// Skip attributes section or constructor called async
 840					//
 841					if (parsing_attribute_section || peek_token () == Token.OPEN_PARENS) {
 842						res = -1;
 843					} else {
 844						// async is keyword
 845					}
 846				} else if (parsing_block > 0) {
 847					switch (peek_token ()) {
 848					case Token.DELEGATE:
 849					case Token.OPEN_PARENS_LAMBDA:
 850						// async is keyword
 851						break;
 852					case Token.IDENTIFIER:
 853						PushPosition ();
 854						xtoken ();
 855						if (xtoken () != Token.ARROW) {
 856							PopPosition ();
 857							goto default;
 858						}
 859
 860						PopPosition ();
 861						break;
 862					default:
 863						// peek_token could overwrite id_buffer
 864						id_builder [0] = 'a'; id_builder [1] = 's'; id_builder [2] = 'y'; id_builder [3] = 'n'; id_builder [4] = 'c';
 865						res = -1;
 866						break;
 867					}
 868				} else {
 869					res = -1;
 870				}
 871
 872				if (res == Token.ASYNC && context.Settings.Version <= LanguageVersion.V_4) {
 873					Report.FeatureIsNotAvailable (context, Location, "asynchronous functions");
 874				}
 875				
 876				break;
 877
 878			case Token.AWAIT:
 879				if (parsing_block == 0)
 880					res = -1;
 881
 882				break;
 883			}
 884
 885			return res;
 886		}
 887
 888		static PreprocessorDirective GetPreprocessorDirective (char[] id, int id_len)
 889		{
 890			//
 891			// Keywords are stored in an array of arrays grouped by their
 892			// length and then by the first character
 893			//
 894			if (id_len >= keywords_preprocessor.Length || keywords_preprocessor[id_len] == null)
 895				return PreprocessorDirective.Invalid;
 896
 897			int first_index = id[0] - '_';
 898			if (first_index > 'z' - '_')
 899				return PreprocessorDirective.Invalid;
 900
 901			var kwe = keywords_preprocessor[id_len][first_index];
 902			if (kwe == null)
 903				return PreprocessorDirective.Invalid;
 904
 905			PreprocessorDirective res = PreprocessorDirective.Invalid;
 906			do {
 907				res = kwe.Token;
 908				for (int i = 1; i < id_len; ++i) {
 909					if (id[i] != kwe.Value[i]) {
 910						res = 0;
 911						kwe = kwe.Next;
 912						break;
 913					}
 914				}
 915			} while (res == PreprocessorDirective.Invalid && kwe != null);
 916
 917			return res;
 918		}
 919
 920		public Location Location {
 921			get {
 922				return new Location (current_source, ref_line, col);
 923			}
 924		}
 925
 926		static bool is_identifier_start_character (int c)
 927		{
 928			return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || Char.IsLetter ((char)c);
 929		}
 930
 931		static bool is_identifier_part_character (char c)
 932		{
 933			if (c >= 'a' && c <= 'z')
 934				return true;
 935
 936			if (c >= 'A' && c <= 'Z')
 937				return true;
 938
 939			if (c == '_' || (c >= '0' && c <= '9'))
 940				return true;
 941
 942			if (c < 0x80)
 943				return false;
 944
 945			return is_identifier_part_character_slow_part (c);
 946		}
 947
 948		static bool is_identifier_part_character_slow_part (char c)
 949		{
 950			if (Char.IsLetter (c))
 951				return true;
 952
 953			switch (Char.GetUnicodeCategory (c)) {
 954				case UnicodeCategory.ConnectorPunctuation:
 955
 956				// combining-character: A Unicode character of classes Mn or Mc
 957				case UnicodeCategory.NonSpacingMark:
 958				case UnicodeCategory.SpacingCombiningMark:
 959
 960				// decimal-digit-character: A Unicode character of the class Nd 
 961				case UnicodeCategory.DecimalDigitNumber:
 962				return true;
 963			}
 964
 965			return false;
 966		}
 967
 968		public static bool IsKeyword (string s)
 969		{
 970			return keyword_strings.Contains (s);
 971		}
 972
 973		//
 974		// Open parens micro parser. Detects both lambda and cast ambiguity.
 975		//	
 976		int TokenizeOpenParens ()
 977		{
 978			int ptoken;
 979			current_token = -1;
 980
 981			int bracket_level = 0;
 982			bool is_type = false;
 983			bool can_be_type = false;
 984			
 985			while (true) {
 986				ptoken = current_token;
 987				token ();
 988
 989				switch (current_token) {
 990				case Token.CLOSE_PARENS:
 991					token ();
 992					
 993					//
 994					// Expression inside parens is lambda, (int i) => 
 995					//
 996					if (current_token == Token.ARROW)
 997						return Token.OPEN_PARENS_LAMBDA;
 998
 999					//
1000					// Expression inside parens is single type, (int[])
1001					//
1002					if (is_type) {
1003						if (current_token == Token.SEMICOLON)
1004							return Token.OPEN_PARENS;
1005
1006						return Token.OPEN_PARENS_CAST;
1007					}
1008
1009					//
1010					// Expression is possible cast, look at next token, (T)null
1011					//
1012					if (can_be_type) {
1013						switch (current_token) {
1014						case Token.OPEN_PARENS:
1015						case Token.BANG:
1016						case Token.TILDE:
1017						case Token.IDENTIFIER:
1018						case Token.LITERAL:
1019						case Token.BASE:
1020						case Token.CHECKED:
1021						case Token.DELEGATE:
1022						case Token.FALSE:
1023						case Token.FIXED:
1024						case Token.NEW:
1025						case Token.NULL:
1026						case Token.SIZEOF:
1027						case Token.THIS:
1028						case Token.THROW:
1029						case Token.TRUE:
1030						case Token.TYPEOF:
1031						case Token.UNCHECKED:
1032						case Token.UNSAFE:
1033						case Token.DEFAULT:
1034						case Token.AWAIT:
1035
1036						//
1037						// These can be part of a member access
1038						//
1039						case Token.INT:
1040						case Token.UINT:
1041						case Token.SHORT:
1042						case Token.USHORT:
1043						case Token.LONG:
1044						case Token.ULONG:
1045						case Token.DOUBLE:
1046						case Token.FLOAT:
1047						case Token.CHAR:
1048						case Token.BYTE:
1049						case Token.DECIMAL:
1050						case Token.BOOL:
1051							return Token.OPEN_PARENS_CAST;
1052						}
1053					}
1054					return Token.OPEN_PARENS;
1055					
1056				case Token.DOT:
1057				case Token.DOUBLE_COLON:
1058					if (ptoken != Token.IDENTIFIER && ptoken != Token.OP_GENERICS_GT)
1059						goto default;
1060
1061					continue;
1062
1063				case Token.IDENTIFIER:
1064				case Token.AWAIT:
1065					switch (ptoken) {
1066					case Token.DOT:
1067						if (bracket_level == 0) {
1068							is_type = false;
1069							can_be_type = true;
1070						}
1071
1072						continue;
1073					case Token.OP_GENERICS_LT:
1074					case Token.COMMA:
1075					case Token.DOUBLE_COLON:
1076					case -1:
1077						if (bracket_level == 0)
1078							can_be_type = true;
1079						continue;
1080					default:
1081						can_be_type = is_type = false;
1082						continue;
1083					}
1084
1085				case Token.OBJECT:
1086				case Token.STRING:
1087				case Token.BOOL:
1088				case Token.DECIMAL:
1089				case Token.FLOAT:
1090				case Token.DOUBLE:
1091				case Token.SBYTE:
1092				case Token.BYTE:
1093				case Token.SHORT:
1094				case Token.USHORT:
1095				case Token.INT:
1096				case Token.UINT:
1097				case Token.LONG:
1098				case Token.ULONG:
1099				case Token.CHAR:
1100				case Token.VOID:
1101					if (bracket_level == 0)
1102						is_type = true;
1103					continue;
1104
1105				case Token.COMMA:
1106					if (bracket_level == 0) {
1107						bracket_level = 100;
1108						can_be_type = is_type = false;
1109					}
1110					continue;
1111
1112				case Token.OP_GENERICS_LT:
1113				case Token.OPEN_BRACKET:
1114					if (bracket_level++ == 0)
1115						is_type = true;
1116					continue;
1117
1118				case Token.OP_GENERICS_GT:
1119				case Token.CLOSE_BRACKET:
1120					--bracket_level;
1121					continue;
1122
1123				case Token.INTERR_NULLABLE:
1124				case Token.STAR:
1125					if (bracket_level == 0)
1126						is_type = true;
1127					continue;
1128
1129				case Token.REF:
1130				case Token.OUT:
1131					can_be_type = is_type = false;
1132					continue;
1133
1134				default:
1135					return Token.OPEN_PARENS;
1136				}
1137			}
1138		}
1139
1140		public static bool IsValidIdentifier (string s)
1141		{
1142			if (s == null || s.Length == 0)
1143				return false;
1144
1145			if (!is_identifier_start_character (s [0]))
1146				return false;
1147			
1148			for (int i = 1; i < s.Length; i ++)
1149				if (! is_identifier_part_character (s [i]))
1150					return false;
1151			
1152			return true;
1153		}
1154
1155		Stack<List<Location>> genericDimensionLocations = new Stack<List<Location>> ();
1156
1157		public List<Location> GenericDimensionLocations {
1158			get {
1159				if (genericDimensionLocations.Count == 0)
1160					return null;
1161				return genericDimensionLocations.Pop ();
1162			}
1163		}
1164
1165		bool parse_less_than (ref int genericDimension)
1166		{
1167			genericDimensionLocations.Push (new List<Location> ()); 
1168			genericDimensionLocations.Peek ().Add (Location);
1169			start:
1170			int the_token = token ();
1171			if (the_token == Token.OPEN_BRACKET) {
1172				while (true) {
1173					the_token = token ();
1174					if (the_token == Token.EOF)
1175						return true;
1176
1177					if (the_token == Token.CLOSE_BRACKET)
1178						break;
1179				}
1180				the_token = token ();
1181			} else if (the_token == Token.IN || the_token == Token.OUT) {
1182				the_token = token ();
1183			}
1184			switch (the_token) {
1185			case Token.IDENTIFIER:
1186			case Token.OBJECT:
1187			case Token.STRING:
1188			case Token.BOOL:
1189			case Token.DECIMAL:
1190			case Token.FLOAT:
1191			case Token.DOUBLE:
1192			case Token.SBYTE:
1193			case Token.BYTE:
1194			case Token.SHORT:
1195			case Token.USHORT:
1196			case Token.INT:
1197			case Token.UINT:
1198			case Token.LONG:
1199			case Token.ULONG:
1200			case Token.CHAR:
1201			case Token.VOID:
1202				break;
1203			case Token.OP_GENERICS_GT:
1204				genericDimension = 1;
1205				genericDimensionLocations.Peek ().Add (Location);
1206				return true;
1207			case Token.IN:
1208			case Token.OUT:
1209				genericDimensionLocations.Pop ();
1210				return true;
1211			case Token.COMMA:
1212				do {
1213					++genericDimension;
1214					if (genericDimensionLocations.Count > 0)
1215						genericDimensionLocations.Peek ().Add (Location);
1216					the_token = token ();
1217				} while (the_token == Token.COMMA);
1218
1219				if (the_token == Token.OP_GENERICS_GT) {
1220					++genericDimension;
1221					if (genericDimensionLocations.Count > 0)
1222						genericDimensionLocations.Peek ().Add (Location);
1223					return true;
1224				}
1225
1226				genericDimensionLocations.Pop ();
1227				return false;
1228			default:
1229				genericDimensionLocations.Pop ();
1230				return false;
1231			}
1232			again:
1233			the_token = token ();
1234			if (the_token == Token.OP_GENERICS_GT) {
1235				genericDimensionLocations.Peek ().Add (Location);
1236				return true;
1237			}
1238			else if (the_token == Token.COMMA || the_token == Token.DOT || the_token == Token.DOUBLE_COLON)
1239				goto start;
1240			else if (the_token == Token.INTERR_NULLABLE || the_token == Token.STAR)
1241				goto again;
1242			else if (the_token == Token.OP_GENERICS_LT) {
1243				if (!parse_less_than (ref genericDimension)) {
1244					genericDimensionLocations.Pop ();
1245					return false;
1246				}
1247				goto again;
1248			} else if (the_token == Token.OPEN_BRACKET) {
1249				rank_specifiers:
1250				the_token = token ();
1251				if (the_token == Token.CLOSE_BRACKET)
1252					goto again;
1253				else if (the_token == Token.COMMA)
1254					goto rank_specifiers;
1255				genericDimensionLocations.Pop ();
1256				return false;
1257			}
1258
1259			genericDimensionLocations.Pop ();
1260			return false;
1261		}
1262
1263
1264		public int peek_token ()
1265		{
1266			int the_token;
1267
1268			PushPosition ();
1269			sbag.Suppress = true;
1270			the_token = token ();
1271			sbag.Suppress = false;
1272			PopPosition ();
1273			
1274			return the_token;
1275		}
1276					
1277		//
1278		// Tonizes `?' using custom disambiguous rules to return one
1279		// of following tokens: INTERR_NULLABLE, OP_COALESCING, INTERR
1280		//
1281		// Tricky expression looks like:
1282		//
1283		// Foo ? a = x ? b : c;
1284		//
1285		int TokenizePossibleNullableType ()
1286		{
1287			if (parsing_block == 0 || parsing_type > 0)
1288				return Token.INTERR_NULLABLE;
1289
1290			int d = peek_char ();
1291			if (d == '?') {
1292				get_char ();
1293				return Token.OP_COALESCING;
1294			}
1295
1296			if (d == '.') {
1297				return Token.INTERR_OPERATOR;
1298			}
1299
1300			if (d != ' ') {
1301				if (d == ',' || d == ';' || d == '>')
1302					return Token.INTERR_NULLABLE;
1303				if (d == '*' || (d >= '0' && d <= '9'))
1304					return Token.INTERR;
1305			}
1306
1307			PushPosition ();
1308			current_token = Token.NONE;
1309			int next_token;
1310			int parens = 0;
1311			int generics = 0;
1312
1313			var nt = xtoken ();
1314			switch (nt) {
1315			case Token.DOT:
1316			case Token.OPEN_BRACKET_EXPR:
1317				next_token = Token.INTERR_OPERATOR;
1318				break;
1319			case Token.LITERAL:
1320			case Token.TRUE:
1321			case Token.FALSE:
1322			case Token.NULL:
1323			case Token.THIS:
1324			case Token.NEW:
1325				next_token = Token.INTERR;
1326				break;
1327				
1328			case Token.SEMICOLON:
1329			case Token.COMMA:
1330			case Token.CLOSE_PARENS:
1331			case Token.OPEN_BRACKET:
1332			case Token.OP_GENERICS_GT:
1333			case Token.INTERR:
1334			case Token.OP_COALESCING:
1335			case Token.COLON:
1336				next_token = Token.INTERR_NULLABLE;
1337				break;
1338
1339			case Token.OPEN_PARENS:
1340			case Token.OPEN_PARENS_CAST:
1341			case Token.OPEN_PARENS_LAMBDA:
1342				next_token = -1;
1343				++parens;
1344				break;
1345
1346			case Token.OP_GENERICS_LT:
1347			case Token.OP_GENERICS_LT_DECL:
1348			case Token.GENERIC_DIMENSION:
1349				next_token = -1;
1350				++generics;
1351				break;
1352
1353			default:
1354				next_token = -1;
1355				break;
1356			}
1357
1358			if (next_token == -1) {
1359				switch (xtoken ()) {
1360				case Token.COMMA:
1361				case Token.SEMICOLON:
1362				case Token.OPEN_BRACE:
1363				case Token.IN:
1364					next_token = Token.INTERR_NULLABLE;
1365					break;
1366					
1367				case Token.COLON:
1368					next_token = Token.INTERR;
1369					break;
1370
1371				case Token.OPEN_PARENS:
1372				case Token.OPEN_PARENS_CAST:
1373				case Token.OPEN_PARENS_LAMBDA:
1374					++parens;
1375					goto default;
1376
1377				case Token.CLOSE_PARENS:
1378					--parens;
1379					goto default;
1380
1381				case Token.OP_GENERICS_LT:
1382				case Token.OP_GENERICS_LT_DECL:
1383				case Token.GENERIC_DIMENSION:
1384					++generics;
1385					goto default;
1386
1387				default:
1388					int ntoken;
1389					int interrs = 1;
1390					int colons = 0;
1391					int braces = 0;
1392					int brackets = 0;
1393					//
1394					// All shorcuts failed, do it hard way
1395					//
1396					while ((ntoken = xtoken ()) != Token.EOF) {
1397						switch (ntoken) {
1398						case Token.OPEN_BRACE:
1399							++braces;
1400							continue;
1401						case Token.OPEN_PARENS:
1402						case Token.OPEN_PARENS_CAST:
1403						case Token.OPEN_PARENS_LAMBDA:
1404							++parens;
1405							continue;
1406						case Token.CLOSE_BRACE:
1407							--braces;
1408							continue;
1409						case Token.OP_GENERICS_LT:
1410						case Token.OP_GENERICS_LT_DECL:
1411						case Token.GENERIC_DIMENSION:
1412							++generics;
1413							continue;
1414						case Token.OPEN_BRACKET:
1415						case Token.OPEN_BRACKET_EXPR:
1416							++brackets;
1417							continue;
1418						case Token.CLOSE_BRACKET:
1419							--brackets;
1420							continue;
1421						case Token.CLOSE_PARENS:
1422							if (parens > 0) {
1423								--parens;
1424								continue;
1425							}
1426
1427							PopPosition ();
1428							return Token.INTERR_NULLABLE;
1429
1430						case Token.OP_GENERICS_GT:
1431							if (generics > 0) {
1432								--generics;
1433								continue;
1434							}
1435
1436							PopPosition ();
1437							return Token.INTERR_NULLABLE;
1438						}
1439
1440						if (braces != 0)
1441							continue;
1442
1443						if (ntoken == Token.SEMICOLON)
1444							break;
1445
1446						if (parens != 0)
1447							continue;
1448
1449						if (ntoken == Token.COMMA) {
1450							if (generics != 0 || brackets != 0)
1451								continue;
1452
1453							PopPosition ();
1454							return Token.INTERR_NULLABLE;
1455						}
1456						
1457						if (ntoken == Token.COLON) {
1458							if (++colons == interrs)
1459								break;
1460							continue;
1461						}
1462						
1463						if (ntoken == Token.INTERR) {
1464							++interrs;
1465							continue;
1466						}
1467					}
1468					
1469					next_token = colons != interrs && braces == 0 ? Token.INTERR_NULLABLE : Token.INTERR;
1470					break;
1471				}
1472			}
1473			
1474			PopPosition ();
1475			return next_token;
1476		}
1477
1478		bool decimal_digits (int c)
1479		{
1480			int d;
1481			bool seen_digits = false;
1482			
1483			if (c != -1){
1484				if (number_pos == MaxNumberLength)
1485					Error_NumericConstantTooLong ();
1486				number_builder [number_pos++] = (char) c;
1487			}
1488			
1489			//
1490			// We use peek_char2, because decimal_digits needs to do a 
1491			// 2-character look-ahead (5.ToString for example).
1492			//
1493			while ((d = peek_char2 ()) != -1){
1494				if (d >= '0' && d <= '9'){
1495					if (number_pos == MaxNumberLength)
1496						Error_NumericConstantTooLong ();
1497					number_builder [number_pos++] = (char) d;
1498					get_char ();
1499					seen_digits = true;
1500				} else
1501					break;
1502			}
1503			
1504			return seen_digits;
1505		}
1506
1507		static bool is_hex (int e)
1508		{
1509			return (e >= '0' && e <= '9') || (e >= 'A' && e <= 'F') || (e >= 'a' && e <= 'f');
1510		}
1511
1512		static TypeCode real_type_suffix (int c)
1513		{
1514			switch (c){
1515			case 'F': case 'f':
1516				return TypeCode.Single;
1517			case 'D': case 'd':
1518				return TypeCode.Double;
1519			case 'M': case 'm':
1520				return TypeCode.Decimal;
1521			default:
1522				return TypeCode.Empty;
1523			}
1524		}
1525
1526		ILiteralConstant integer_type_suffix (ulong ul, int c, Location loc)
1527		{
1528			bool is_unsigned = false;
1529			bool is_long = false;
1530
1531			if (c != -1){
1532				bool scanning = true;
1533				do {
1534					switch (c){
1535					case 'U': case 'u':
1536						if (is_unsigned)
1537							scanning = false;
1538						is_unsigned = true;
1539						get_char ();
1540						break;
1541
1542					case 'l':
1543						if (!is_unsigned){
1544							//
1545							// if we have not seen anything in between
1546							// report this error
1547							//
1548							Report.Warning (78, 4, Location, "The `l' suffix is easily confused with the digit `1' (use `L' for clarity)");
1549						}
1550
1551						goto case 'L';
1552
1553					case 'L': 
1554						if (is_long)
1555							scanning = false;
1556						is_long = true;
1557						get_char ();
1558						break;
1559						
1560					default:
1561						scanning = false;
1562						break;
1563					}
1564					c = peek_char ();
1565				} while (scanning);
1566			}
1567
1568			if (is_long && is_unsigned){
1569				return new ULongLiteral (context.BuiltinTypes, ul, loc);
1570			}
1571			
1572			if (is_unsigned){
1573				// uint if possible, or ulong else.
1574
1575				if ((ul & 0xffffffff00000000) == 0)
1576					return new UIntLiteral (context.BuiltinTypes, (uint) ul, loc);
1577				else
1578					return new ULongLiteral (context.BuiltinTypes, ul, loc);
1579			} else if (is_long){
1580				// long if possible, ulong otherwise
1581				if ((ul & 0x8000000000000000) != 0)
1582					return new ULongLiteral (context.BuiltinTypes, ul, loc);
1583				else
1584					return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1585			} else {
1586				// int, uint, long or ulong in that order
1587				if ((ul & 0xffffffff00000000) == 0){
1588					uint ui = (uint) ul;
1589					
1590					if ((ui & 0x80000000) != 0)
1591						return new UIntLiteral (context.BuiltinTypes, ui, loc);
1592					else
1593						return new IntLiteral (context.BuiltinTypes, (int) ui, loc);
1594				} else {
1595					if ((ul & 0x8000000000000000) != 0)
1596						return new ULongLiteral (context.BuiltinTypes, ul, loc);
1597					else
1598						return new LongLiteral (context.BuiltinTypes, (long) ul, loc);
1599				}
1600			}
1601		}
1602				
1603		//
1604		// given `c' as the next char in the input decide whether
1605		// we need to convert to a special type, and then choose
1606		// the best representation for the integer
1607		//
1608		ILiteralConstant adjust_int (int c, Location loc)
1609		{
1610			try {
1611				if (number_pos > 9){
1612					ulong ul = (uint) (number_builder [0] - '0');
1613
1614					for (int i = 1; i < number_pos; i++){
1615						ul = checked ((ul * 10) + ((uint)(number_builder [i] - '0')));
1616					}
1617
1618					return integer_type_suffix (ul, c, loc);
1619				} else {
1620					uint ui = (uint) (number_builder [0] - '0');
1621
1622					for (int i = 1; i < number_pos; i++){
1623						ui = checked ((ui * 10) + ((uint)(number_builder [i] - '0')));
1624					}
1625
1626					return integer_type_suffix (ui, c, loc);
1627				}
1628			} catch (OverflowException) {
1629				Error_NumericConstantTooLong ();
1630				return new IntLiteral (context.BuiltinTypes, 0, loc);
1631			}
1632			catch (FormatException) {
1633				Report.Error (1013, Location, "Invalid number");
1634				return new IntLiteral (context.BuiltinTypes, 0, loc);
1635			}
1636		}
1637		
1638		ILiteralConstant adjust_real (TypeCode t, Location loc)
1639		{
1640			string s = new string (number_builder, 0, number_pos);
1641			const string error_details = "Floating-point constant is outside the range of type `{0}'";
1642
1643			switch (t){
1644			case TypeCode.Decimal:
1645				try {
1646					return new DecimalLiteral (context.BuiltinTypes, decimal.Parse (s, styles, csharp_format_info), loc);
1647				} catch (OverflowException) {
1648					Report.Error (594, Location, error_details, "decimal");
1649					return new DecimalLiteral (context.BuiltinTypes, 0, loc);
1650				}
1651			case TypeCode.Single:
1652				try {
1653					return new FloatLiteral (context.BuiltinTypes, float.Parse (s, styles, csharp_format_info), loc);
1654				} catch (OverflowException) {
1655					Report.Error (594, Location, error_details, "float");
1656					return new FloatLiteral (context.BuiltinTypes, 0, loc);
1657				}
1658			default:
1659				try {
1660					return new DoubleLiteral (context.BuiltinTypes, double.Parse (s, styles, csharp_format_info), loc);
1661				} catch (OverflowException) {
1662					Report.Error (594, loc, error_details, "double");
1663					return new DoubleLiteral (context.BuiltinTypes, 0, loc);
1664				}
1665			}
1666		}
1667
1668		ILiteralConstant handle_hex (Location loc)
1669		{
1670			int d;
1671			ulong ul;
1672			
1673			get_char ();
1674			while ((d = peek_char ()) != -1){
1675				if (is_hex (d)){
1676					number_builder [number_pos++] = (char) d;
1677					get_char ();
1678				} else
1679					break;
1680			}
1681			
1682			string s = new String (number_builder, 0, number_pos);
1683
1684			try {
1685				if (number_pos <= 8)
1686					ul = System.UInt32.Parse (s, NumberStyles.HexNumber);
1687				else
1688					ul = System.UInt64.Parse (s, NumberStyles.HexNumber);
1689
1690				return integer_type_suffix (ul, peek_char (), loc);
1691			} catch (OverflowException){
1692				Error_NumericConstantTooLong ();
1693				return new IntLiteral (context.BuiltinTypes, 0, loc);
1694			}
1695			catch (FormatException) {
1696				Report.Error (1013, Location, "Invalid number");
1697				return new IntLiteral (context.BuiltinTypes, 0, loc);
1698			}
1699		}
1700
1701		//
1702		// Invoked if we know we have .digits or digits
1703		//
1704		int is_number (int c, bool dotLead)
1705		{
1706			ILiteralConstant res;
1707
1708#if FULL_AST
1709			int read_start = reader.Position - 1;
1710			if (dotLead) {
1711				//
1712				// Caller did peek_char
1713				//
1714				--read_start;
1715			}
1716#endif
1717			number_pos = 0;
1718			var loc = Location;
1719			bool hasLeadingDot = c == '.';
1720
1721			if (!dotLead){
1722				if (c == '0'){
1723					int peek = peek_char ();
1724
1725					if (peek == 'x' || peek == 'X') {
1726						val = res = handle_hex (loc);
1727#if FULL_AST
1728						res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1729#endif
1730
1731						return Token.LITERAL;
1732					}
1733				}
1734				decimal_digits (c);
1735				c = peek_char ();
1736			}
1737
1738			//
1739			// We need to handle the case of
1740			// "1.1" vs "1.string" (LITERAL_FLOAT vs NUMBER DOT IDENTIFIER)
1741			//
1742			bool is_real = false;
1743			if (c == '.'){
1744				if (!dotLead)
1745					get_char ();
1746
1747				if (decimal_digits ('.')){
1748					is_real = true;
1749					c = peek_char ();
1750				} else {
1751					putback ('.');
1752					number_pos--;
1753					val = res = adjust_int (-1, loc);
1754#if FULL_AST
1755					res.ParsedValue = reader.ReadChars (read_start, reader.Position - 1);
1756#endif
1757					return Token.LITERAL;
1758				}
1759			}
1760			
1761			if (c == 'e' || c == 'E'){
1762				is_real = true;
1763				get_char ();
1764				if (number_pos == MaxNumberLength)
1765					Error_NumericConstantTooLong ();
1766				number_builder [number_pos++] = (char) c;
1767				c = get_char ();
1768				
1769				if (c == '+'){
1770					if (number_pos == MaxNumberLength)
1771						Error_NumericConstantTooLong ();
1772					number_builder [number_pos++] = '+';
1773					c = -1;
1774				} else if (c == '-') {
1775					if (number_pos == MaxNumberLength)
1776						Error_NumericConstantTooLong ();
1777					number_builder [number_pos++] = '-';
1778					c = -1;
1779				} else {
1780					if (number_pos == MaxNumberLength)
1781						Error_NumericConstantTooLong ();
1782					number_builder [number_pos++] = '+';
1783				}
1784					
1785				decimal_digits (c);
1786				c = peek_char ();
1787			}
1788
1789			var type = real_type_suffix (c);
1790			if (type == TypeCode.Empty && !is_real) {
1791				res = adjust_int (c, loc);
1792			} else {
1793				is_real = true;
1794
1795				if (type != TypeCode.Empty) {
1796					get_char ();
1797				}
1798
1799				res = adjust_real (type, loc);
1800			}
1801
1802			val = res;
1803#if FULL_AST
1804			var chars = reader.ReadChars (read_start, reader.Position - (type == TypeCode.Empty && c > 0 ? 1 : 0));
1805			if (chars[chars.Length - 1] == '\r')
1806				Array.Resize (ref chars, chars.Length - 1);
1807			res.ParsedValue = chars;
1808#endif
1809
1810			return Token.LITERAL;
1811		}
1812
1813		//
1814		// Accepts exactly count (4 or 8) hex, no more no less
1815		//
1816		int getHex (int count, out int surrogate, out bool error)
1817		{
1818			int i;
1819			int total = 0;
1820			int c;
1821			int top = count != -1 ? count : 4;
1822			
1823			get_char ();
1824			error = false;
1825			surrogate = 0;
1826			for (i = 0; i < top; i++){
1827				c = get_char ();
1828
1829				if (c >= '0' && c <= '9')
1830					c = (int) c - (int) '0';
1831				else if (c >= 'A' && c <= 'F')
1832					c = (int) c - (int) 'A' + 10;
1833				else if (c >= 'a' && c <= 'f')
1834					c = (int) c - (int) 'a' + 10;
1835				else {
1836					error = true;
1837					return 0;
1838				}
1839				
1840				total = (total * 16) + c;
1841				if (count == -1){
1842					int p = peek_char ();
1843					if (p == -1)
1844						break;
1845					if (!is_hex ((char)p))
1846						break;
1847				}
1848			}
1849
1850			if (top == 8) {
1851				if (total > 0x0010FFFF) {
1852					error = true;
1853					return 0;
1854				}
1855
1856				if (total >= 0x00010000) {
1857					surrogate = ((total - 0x00010000) % 0x0400 + 0xDC00);					
1858					total = ((total - 0x00010000) / 0x0400 + 0xD800);
1859				}
1860			}
1861
1862			return total;
1863		}
1864
1865		int escape (int c, out int surrogate)
1866		{
1867			bool error;
1868			int d;
1869			int v;
1870
1871			d = peek_char ();
1872			if (c != '\\') {
1873				surrogate = 0;
1874				return c;
1875			}
1876			
1877			switch (d){
1878			case 'a':
1879				v = '\a'; break;
1880			case 'b':
1881				v = '\b'; break;
1882			case 'n':
1883				v = '\n'; break;
1884			case 't':
1885				v = '\t'; break;
1886			case 'v':
1887				v = '\v'; break;
1888			case 'r':
1889				v = '\r'; break;
1890			case '\\':
1891				v = '\\'; break;
1892			case 'f':
1893				v = '\f'; break;
1894			case '0':
1895				v = 0; break;
1896			case '"':
1897				v = '"'; break;
1898			case '\'':
1899				v = '\''; break;
1900			case 'x':
1901				v = getHex (-1, out surrogate, out error);
1902				if (error)
1903					goto default;
1904				return v;
1905			case 'u':
1906			case 'U':
1907				return EscapeUnicode (d, out surrogate);
1908			default:
1909				surrogate = 0;
1910				Report.Error (1009, Location, "Unrecognized escape sequence `\\{0}'", ((char)d).ToString ());
1911				return d;
1912			}
1913
1914			get_char ();
1915			surrogate = 0;
1916			return v;
1917		}
1918
1919		int EscapeUnicode (int ch, out int surrogate)
1920		{
1921			bool error;
1922			if (ch == 'U') {
1923				ch = getHex (8, out surrogate, out error);
1924			} else {
1925				ch = getHex (4, out surrogate, out error);
1926			}
1927
1928			if (error)
1929				Report.Error (1009, Location, "Unrecognized escape sequence");
1930
1931			return ch;
1932		}
1933
1934		int get_char ()
1935		{
1936			int x;
1937			if (putback_char != -1) {
1938				x = putback_char;
1939				putback_char = -1;
1940			} else {
1941				x = reader.Read ();
1942			}
1943			
1944			if (x <= 13) {
1945				if (x == '\r') {
1946					if (peek_char () == '\n') {
1947						putback_char = -1;
1948						advance_line (SpecialsBag.NewLine.Windows);
1949					} else {
1950						advance_line (SpecialsBag.NewLine.Unix);
1951					}
1952
1953					x = '\n';
1954				} else if (x == '\n') {
1955					advance_line (SpecialsBag.NewLine.Unix);
1956				} else {
1957					col++;
1958				}
1959			} else if (x >= UnicodeLS && x <= UnicodePS) {
1960				advance_line (SpecialsBag.NewLine.Unix);
1961			} else {
1962				col++;
1963			}
1964
1965			return x;
1966		}
1967
1968		bool recordNewLine = true;
1969		void advance_line (SpecialsBag.NewLine newLine)
1970		{
1971			if (recordNewLine)
1972				sbag.AddNewLine (line, col, newLine);
1973			line++;
1974			ref_line++;
1975			previous_col = col;
1976			col = 0;
1977			startsLine = true;
1978		}
1979
1980		int peek_char ()
1981		{
1982			if (putback_char == -1)
1983				putback_char = reader.Read ();
1984			return putback_char;
1985		}
1986
1987		int peek_char2 ()
1988		{
1989			if (putback_char != -1)
1990				return putback_char;
1991			return reader.Peek ();
1992		}
1993		
1994		public void putback (int c)
1995		{
1996			if (putback_char != -1) {
1997				throw new InternalErrorException (string.Format ("Secondary putback [{0}] putting back [{1}] is not allowed", (char)putback_char, (char) c), Location);
1998			}
1999
2000			if (c == '\n' || col == 0 || (c >= UnicodeLS && c <= UnicodePS)) {
2001				// It won't happen though.
2002				line--;
2003				ref_line--;
2004				col = previous_col;
2005			}
2006			else
2007				col--;
2008			putback_char = c;
2009		}
2010
2011		public bool advance ()
2012		{
2013			return peek_char () != -1 || CompleteOnEOF;
2014		}
2015
2016		public Object Value {
2017			get {
2018				return val;
2019			}
2020		}
2021
2022		public Object value ()
2023		{
2024			return val;
2025		}
2026
2027		public int token ()
2028		{
2029			current_token = xtoken ();
2030			return current_token;
2031		}
2032
2033		int TokenizePreprocessorIdentifier (out int c)
2034		{
2035			int startCol, endLine, endCol;
2036			return TokenizePreprocessorIdentifier (out c, out startCol, out endLine, out endCol);
2037		}
2038
2039		int TokenizePreprocessorIdentifier (out int c, out int startCol, out int endLine, out int endCol)
2040		{
2041			// skip over white space
2042			do {
2043				endLine = line;
2044				endCol = col;
2045				c = get_char ();
2046			} while (c == ' ' || c == '\t');
2047			startCol = col;
2048			int pos = 0;
2049			while (c != -1 && c >= 'a' && c <= 'z') {
2050				id_builder[pos++] = (char) c;
2051				endCol = col + 1;
2052				c = get_char ();
2053				if (c == '\\') {
2054					int peek = peek_char ();
2055					if (peek == 'U' || peek == 'u') {
2056						int surrogate;
2057						c = EscapeUnicode (c, out surrogate);
2058						if (surrogate != 0) {
2059							if (is_identifier_part_character ((char) c)) {
2060								id_builder[pos++] = (char) c;
2061							}
2062							c = surrogate;
2063						}
2064					}
2065				}
2066			}
2067
2068			return pos;
2069		}
2070
2071		Prepro…