PageRenderTime 93ms CodeModel.GetById 16ms app.highlight 65ms RepoModel.GetById 1ms app.codeStats 1ms

/NRefactory/ICSharpCode.NRefactory.VB/Lexer/VBLexer.cs

http://github.com/icsharpcode/ILSpy
C# | 1487 lines | 1473 code | 12 blank | 2 comment | 58 complexity | aea4d647444e07fb626ece55c7c34ace MD5 | raw file
   1// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt)
   2// This code is distributed under MIT X11 license (for details please see \doc\license.txt)
   3
   4using System;
   5using System.Collections;
   6using System.Collections.Generic;
   7using System.Diagnostics;
   8using System.Globalization;
   9using System.IO;
  10using System.Linq;
  11using System.Text;
  12using System.Xml;
  13
  14namespace ICSharpCode.NRefactory.VB.Parser
  15{
  16	public class VBLexer : IDisposable
  17	{
  18		bool lineEnd = true;
  19		bool isAtLineBegin = false; // TODO: handle line begin, if neccessarry
  20		bool misreadExclamationMarkAsTypeCharacter;
  21		bool encounteredLineContinuation;
  22		
  23		ExpressionFinder ef;
  24		
  25		bool inXmlMode;
  26		
  27		Stack<XmlModeInfo> xmlModeStack = new Stack<XmlModeInfo>();
  28		
  29		public VBLexer(TextReader reader)
  30		{
  31			this.reader = new LATextReader(reader);
  32			ef = new ExpressionFinder();
  33		}
  34		
  35		public VBLexer(TextReader reader, VBLexerMemento state) : this(reader)
  36		{
  37			SetInitialLocation(new TextLocation(state.Line, state.Column));
  38			lastToken = new Token(state.PrevTokenKind, 0, 0);
  39			ef = new ExpressionFinder(state.ExpressionFinder);
  40			lineEnd = state.LineEnd;
  41			isAtLineBegin = state.IsAtLineBegin;
  42			encounteredLineContinuation = state.EncounteredLineContinuation;
  43			misreadExclamationMarkAsTypeCharacter = state.MisreadExclamationMarkAsTypeCharacter;
  44			xmlModeStack = new Stack<XmlModeInfo>(state.XmlModeInfoStack.Select(i => (XmlModeInfo)i.Clone()).Reverse());
  45			inXmlMode = state.InXmlMode;
  46		}
  47		
  48		Token NextInternal()
  49		{
  50			if (misreadExclamationMarkAsTypeCharacter) {
  51				misreadExclamationMarkAsTypeCharacter = false;
  52				return new Token(Tokens.ExclamationMark, Col - 1, Line);
  53			}
  54			
  55			unchecked {
  56				while (true) {
  57					TextLocation startLocation = new TextLocation(Line, Col);
  58					int nextChar = ReaderRead();
  59					if (nextChar == -1)
  60						return new Token(Tokens.EOF, Col, Line, string.Empty);
  61					char ch = (char)nextChar;
  62					#region XML mode
  63					CheckXMLState(startLocation);
  64					if (inXmlMode && xmlModeStack.Peek().level <= 0 && !xmlModeStack.Peek().isDocumentStart && !xmlModeStack.Peek().inXmlTag) {
  65						XmlModeInfo info = xmlModeStack.Peek();
  66						int peek = nextChar;
  67						while (true) {
  68							int step = -1;
  69							while (peek != -1 && XmlConvert.IsWhitespaceChar((char)peek)) {
  70								step++;
  71								peek = ReaderPeek(step);
  72							}
  73							
  74							if (peek == '<' && (ReaderPeek(step + 1) == '!' || ReaderPeek(step + 1) == '?')) {
  75								char lastCh = '\0';
  76								for (int i = 0; i < step + 2; i++)
  77									lastCh = (char)ReaderRead();
  78								
  79								if (lastCh == '!')
  80									return ReadXmlCommentOrCData(Col - 2, Line);
  81								else
  82									return ReadXmlProcessingInstruction(Col - 2, Line);
  83							}
  84							
  85							break;
  86						}
  87						inXmlMode = false;
  88						xmlModeStack.Pop();
  89					}
  90					if (inXmlMode) {
  91						XmlModeInfo info = xmlModeStack.Peek();
  92						int x = Col - 1;
  93						int y = Line;
  94						switch (ch) {
  95							case '<':
  96								if (ReaderPeek() == '/') {
  97									ReaderRead();
  98									info.inXmlCloseTag = true;
  99									return new Token(Tokens.XmlOpenEndTag, new TextLocation(y, x), new TextLocation(Line, Col));
 100								}
 101								if (ReaderPeek() == '%' && ReaderPeek(1) == '=') {
 102									inXmlMode = false;
 103									ReaderRead(); ReaderRead();
 104									return new Token(Tokens.XmlStartInlineVB, new TextLocation(y, x), new TextLocation(Line, Col));
 105								}
 106								if (ReaderPeek() == '?') {
 107									ReaderRead();
 108									Token t = ReadXmlProcessingInstruction(x, y);
 109									return t;
 110								}
 111								if (ReaderPeek() == '!') {
 112									ReaderRead();
 113									Token token = ReadXmlCommentOrCData(x, y);
 114									return token;
 115								}
 116								info.level++;
 117								info.isDocumentStart = false;
 118								info.inXmlTag = true;
 119								return new Token(Tokens.XmlOpenTag, x, y);
 120							case '/':
 121								if (ReaderPeek() == '>') {
 122									ReaderRead();
 123									info.inXmlTag = false;
 124									info.level--;
 125									return new Token(Tokens.XmlCloseTagEmptyElement, new TextLocation(y, x), new TextLocation(Line, Col));
 126								}
 127								break;
 128							case '>':
 129								if (info.inXmlCloseTag)
 130									info.level--;
 131								info.inXmlTag = info.inXmlCloseTag = false;
 132								return new Token(Tokens.XmlCloseTag, x, y);
 133							case '=':
 134								return new Token(Tokens.Assign, x, y);
 135							case '\'':
 136							case '"':
 137								string s = ReadXmlString(ch);
 138								return new Token(Tokens.LiteralString, x, y, ch + s + ch, s);
 139							default:
 140								if (info.inXmlCloseTag || info.inXmlTag) {
 141									if (XmlConvert.IsWhitespaceChar(ch))
 142										continue;
 143									return new Token(Tokens.Identifier, x, y, ReadXmlIdent(ch));
 144								} else {
 145									string content = ReadXmlContent(ch);
 146									return new Token(Tokens.XmlContent, startLocation, new TextLocation(Line, Col), content, null);
 147								}
 148						}
 149						#endregion
 150					} else {
 151						#region Standard Mode
 152						if (Char.IsWhiteSpace(ch)) {
 153							if (HandleLineEnd(ch)) {
 154								if (lineEnd) {
 155									// second line end before getting to a token
 156									// -> here was a blank line
 157//									specialTracker.AddEndOfLine(startLocation);
 158								} else {
 159									lineEnd = true;
 160									return new Token(Tokens.EOL, startLocation, new TextLocation(Line, Col), null, null);
 161								}
 162							}
 163							continue;
 164						}
 165						if (ch == '_') {
 166							if (ReaderPeek() == -1) {
 167								errors.Error(Line, Col, String.Format("No EOF expected after _"));
 168								return new Token(Tokens.EOF, Col, Line, string.Empty);
 169							}
 170							if (!Char.IsWhiteSpace((char)ReaderPeek())) {
 171								int x = Col - 1;
 172								int y = Line;
 173								string s = ReadIdent('_');
 174								lineEnd = false;
 175								return new Token(Tokens.Identifier, x, y, s);
 176							}
 177							encounteredLineContinuation = true;
 178							ch = (char)ReaderRead();
 179							
 180							bool oldLineEnd = lineEnd;
 181							lineEnd = false;
 182							while (Char.IsWhiteSpace(ch)) {
 183								if (HandleLineEnd(ch)) {
 184									lineEnd = true;
 185									break;
 186								}
 187								if (ReaderPeek() != -1) {
 188									ch = (char)ReaderRead();
 189								} else {
 190									errors.Error(Line, Col, String.Format("No EOF expected after _"));
 191									return new Token(Tokens.EOF, Col, Line, string.Empty);
 192								}
 193							}
 194							if (!lineEnd) {
 195								errors.Error(Line, Col, String.Format("NewLine expected"));
 196							}
 197							lineEnd = oldLineEnd;
 198							continue;
 199						}
 200						
 201						if (ch == '#') {
 202							while (Char.IsWhiteSpace((char)ReaderPeek())) {
 203								ReaderRead();
 204							}
 205							if (Char.IsDigit((char)ReaderPeek())) {
 206								int x = Col - 1;
 207								int y = Line;
 208								string s = ReadDate();
 209								DateTime time = new DateTime(1, 1, 1, 0, 0, 0);
 210								try {
 211									time = DateTime.Parse(s, System.Globalization.CultureInfo.InvariantCulture, DateTimeStyles.NoCurrentDateDefault);
 212								} catch (Exception e) {
 213									errors.Error(Line, Col, String.Format("Invalid date time {0}", e));
 214								}
 215								return new Token(Tokens.LiteralDate, x, y, s, time);
 216							} else {
 217								ReadPreprocessorDirective();
 218								continue;
 219							}
 220						}
 221						
 222						if (ch == '[') { // Identifier
 223							lineEnd = false;
 224							if (ReaderPeek() == -1) {
 225								errors.Error(Line, Col, String.Format("Identifier expected"));
 226							}
 227							ch = (char)ReaderRead();
 228							if (ch == ']' || Char.IsWhiteSpace(ch)) {
 229								errors.Error(Line, Col, String.Format("Identifier expected"));
 230							}
 231							int x = Col - 1;
 232							int y = Line;
 233							string s = ReadIdent(ch);
 234							if (ReaderPeek() == -1) {
 235								errors.Error(Line, Col, String.Format("']' expected"));
 236							}
 237							ch = (char)ReaderRead();
 238							if (!(ch == ']')) {
 239								errors.Error(Line, Col, String.Format("']' expected"));
 240							}
 241							return new Token(Tokens.Identifier, x, y, s);
 242						}
 243						if (Char.IsLetter(ch)) {
 244							int x = Col - 1;
 245							int y = Line;
 246							char typeCharacter;
 247							string s = ReadIdent(ch, out typeCharacter);
 248							if (typeCharacter == '\0') {
 249								int keyWordToken = Keywords.GetToken(s);
 250								if (keyWordToken >= 0) {
 251									// handle 'REM' comments
 252									if (keyWordToken == Tokens.Rem) {
 253										ReadComment();
 254										if (!lineEnd) {
 255											lineEnd = true;
 256											return new Token(Tokens.EOL, Col, Line, "\n");
 257										}
 258										continue;
 259									}
 260									
 261									lineEnd = false;
 262									return new Token(keyWordToken, x, y, s);
 263								}
 264							}
 265							
 266							lineEnd = false;
 267							return new Token(Tokens.Identifier, x, y, s);
 268							
 269						}
 270						if (Char.IsDigit(ch)) {
 271							lineEnd = false;
 272							return ReadDigit(ch, Col - 1);
 273						}
 274						if (ch == '&') {
 275							lineEnd = false;
 276							if (ReaderPeek() == -1) {
 277								return ReadOperator('&');
 278							}
 279							ch = (char)ReaderPeek();
 280							if (Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'H' || Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'O') {
 281								return ReadDigit('&', Col - 1);
 282							}
 283							return ReadOperator('&');
 284						}
 285						if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {
 286							int x = Col - 1;
 287							int y = Line;
 288							ReadComment();
 289							if (!lineEnd) {
 290								lineEnd = true;
 291								return new Token(Tokens.EOL, x, y, "\n");
 292							}
 293							continue;
 294						}
 295						if (ch == '"') {
 296							lineEnd = false;
 297							int x = Col - 1;
 298							int y = Line;
 299							string s = ReadString();
 300							if (ReaderPeek() != -1 && (ReaderPeek() == 'C' || ReaderPeek() == 'c')) {
 301								ReaderRead();
 302								if (s.Length != 1) {
 303									errors.Error(Line, Col, String.Format("Chars can only have Length 1 "));
 304								}
 305								if (s.Length == 0) {
 306									s = "\0";
 307								}
 308								return new Token(Tokens.LiteralCharacter, x, y, '"' + s  + "\"C", s[0]);
 309							}
 310							return new Token(Tokens.LiteralString, x, y, '"' + s + '"', s);
 311						}
 312						if (ch == '%' && ReaderPeek() == '>') {
 313							int x = Col - 1;
 314							int y = Line;
 315							inXmlMode = true;
 316							ReaderRead();
 317							return new Token(Tokens.XmlEndInlineVB, new TextLocation(y, x), new TextLocation(Line, Col));
 318						}
 319						#endregion
 320						if (ch == '<' && (ef.NextTokenIsPotentialStartOfExpression || ef.NextTokenIsStartOfImportsOrAccessExpression)) {
 321							xmlModeStack.Push(new XmlModeInfo(ef.NextTokenIsStartOfImportsOrAccessExpression));
 322							XmlModeInfo info = xmlModeStack.Peek();
 323							int x = Col - 1;
 324							int y = Line;
 325							inXmlMode = true;
 326							if (ReaderPeek() == '/') {
 327								ReaderRead();
 328								info.inXmlCloseTag = true;
 329								return new Token(Tokens.XmlOpenEndTag, new TextLocation(y, x), new TextLocation(Line, Col));
 330							}
 331							// should we allow <%= at start of an expression? not valid with vbc ...
 332							if (ReaderPeek() == '%' && ReaderPeek(1) == '=') {
 333								inXmlMode = false;
 334								ReaderRead(); ReaderRead();
 335								return new Token(Tokens.XmlStartInlineVB, new TextLocation(y, x), new TextLocation(Line, Col));
 336							}
 337							if (ReaderPeek() == '!') {
 338								ReaderRead();
 339								Token t = ReadXmlCommentOrCData(x, y);
 340								return t;
 341							}
 342							if (ReaderPeek() == '?') {
 343								ReaderRead();
 344								Token t = ReadXmlProcessingInstruction(x, y);
 345								info.isDocumentStart = t.val.Trim().StartsWith("xml", StringComparison.OrdinalIgnoreCase);
 346								return t;
 347							}
 348							info.inXmlTag = true;
 349							info.level++;
 350							return new Token(Tokens.XmlOpenTag, x, y);
 351						}
 352						Token token = ReadOperator(ch);
 353						if (token != null) {
 354							lineEnd = false;
 355							return token;
 356						}
 357					}
 358					
 359					errors.Error(Line, Col, String.Format("Unknown char({0}) which can't be read", ch));
 360				}
 361			}
 362		}
 363
 364		void CheckXMLState(TextLocation startLocation)
 365		{
 366			if (inXmlMode && !xmlModeStack.Any())
 367				throw new InvalidOperationException("invalid XML stack state at " + startLocation);
 368		}
 369		
 370		Token prevToken;
 371		
 372		Token Next()
 373		{
 374			Token t = NextInternal();
 375			if (t.kind == Tokens.EOL) {
 376				Debug.Assert(t.next == null); // NextInternal() must return only 1 token
 377				t.next = NextInternal();
 378				Debug.Assert(t.next.next == null);
 379				if (SkipEOL(prevToken.kind, t.next.kind)) {
 380					t = t.next;
 381				}
 382			} else
 383				encounteredLineContinuation = false;
 384			// inform EF only once we're sure it's really a token
 385			// this means we inform it about EOL tokens "1 token too late", but that's not a problem because
 386			// XML literals cannot start immediately after an EOL token
 387			ef.InformToken(t);
 388			if (t.next != null) {
 389				// Next() isn't called again when it returns 2 tokens, so we need to process both tokens
 390				ef.InformToken(t.next);
 391				prevToken = t.next;
 392			} else {
 393				prevToken = t;
 394			}
 395			ef.Advance();
 396			Debug.Assert(t != null);
 397			return t;
 398		}
 399		
 400		/// <remarks>see VB language specification 10; pg. 6</remarks>
 401		bool SkipEOL(int prevTokenKind, int nextTokenKind)
 402		{
 403			// exception directly after _
 404			if (encounteredLineContinuation) {
 405				return encounteredLineContinuation = false;
 406			}
 407			
 408			// 1st rule
 409			// after a comma (,), open parenthesis ((), open curly brace ({), or open embedded expression (<%=)
 410			if (new[] { Tokens.Comma, Tokens.OpenParenthesis, Tokens.OpenCurlyBrace, Tokens.XmlStartInlineVB }
 411			    .Contains(prevTokenKind))
 412				return true;
 413			
 414			// 2nd rule
 415			// after a member qualifier (. or .@ or ...), provided that something is being qualified (i.e. is not
 416			// using an implicit With context)
 417			if (new[] { Tokens.Dot, Tokens.DotAt, Tokens.TripleDot }.Contains(prevTokenKind)
 418			    && !ef.WasQualifierTokenAtStart)
 419				return true;
 420			
 421			// 3rd rule
 422			// before a close parenthesis ()), close curly brace (}), or close embedded expression (%>)
 423			if (new[] { Tokens.CloseParenthesis, Tokens.CloseCurlyBrace, Tokens.XmlEndInlineVB }
 424			    .Contains(nextTokenKind))
 425				return true;
 426			
 427			// 4th rule
 428			// after a less-than (<) in an attribute context
 429			if (prevTokenKind == Tokens.LessThan && ef.InContext(Context.Attribute))
 430				return true;
 431			
 432			// 5th rule
 433			// before a greater-than (>) in an attribute context
 434			if (nextTokenKind == Tokens.GreaterThan && ef.InContext(Context.Attribute))
 435				return true;
 436			
 437			// 6th rule
 438			// after a greater-than (>) in a non-file-level attribute context
 439			if (ef.WasNormalAttribute && prevTokenKind == Tokens.GreaterThan)
 440				return true;
 441			
 442			// 7th rule
 443			// before and after query operators (Where, Order, Select, etc.)
 444			var queryOperators = new int[] { Tokens.From, Tokens.Aggregate, Tokens.Select, Tokens.Distinct,
 445				Tokens.Where, Tokens.Order, Tokens.By, Tokens.Ascending, Tokens.Descending, Tokens.Take,
 446				Tokens.Skip, Tokens.Let, Tokens.Group, Tokens.Into, Tokens.On, Tokens.While, Tokens.Join };
 447			if (ef.InContext(Context.Query)) {
 448				// Ascending, Descending, Distinct are special
 449				// fixes http://community.sharpdevelop.net/forums/p/12068/32893.aspx#32893
 450				var specialQueryOperators = new int[] { Tokens.Ascending, Tokens.Descending, Tokens.Distinct };
 451				if (specialQueryOperators.Contains(prevTokenKind) && !queryOperators.Contains(nextTokenKind))
 452					return false;
 453				
 454				if ((queryOperators.Contains(prevTokenKind) || queryOperators.Contains(nextTokenKind)))
 455					return true;
 456			}
 457			
 458			// 8th rule
 459			// after binary operators (+, -, /, *, etc.) in an expression context
 460			if (new[] { Tokens.Plus, Tokens.Minus, Tokens.Div, Tokens.DivInteger, Tokens.Times, Tokens.Mod, Tokens.Power,
 461			    	Tokens.Assign, Tokens.NotEqual, Tokens.LessThan, Tokens.LessEqual, Tokens.GreaterThan, Tokens.GreaterEqual,
 462			    	Tokens.Like, Tokens.ConcatString, Tokens.AndAlso, Tokens.OrElse, Tokens.And, Tokens.Or, Tokens.Xor,
 463			    	Tokens.ShiftLeft, Tokens.ShiftRight }.Contains(prevTokenKind) && ef.CurrentBlock.context == Context.Expression)
 464				return true;
 465			
 466			// 9th rule
 467			// after assignment operators (=, :=, +=, -=, etc.) in any context.
 468			if (new[] { Tokens.Assign, Tokens.ColonAssign, Tokens.ConcatStringAssign, Tokens.DivAssign,
 469			    	Tokens.DivIntegerAssign, Tokens.MinusAssign, Tokens.PlusAssign, Tokens.PowerAssign,
 470			    	Tokens.ShiftLeftAssign, Tokens.ShiftRightAssign, Tokens.TimesAssign }.Contains(prevTokenKind))
 471				return true;
 472			
 473			return false;
 474		}
 475		
 476		/// <summary>
 477		/// Reads the next token.
 478		/// </summary>
 479		/// <returns>A <see cref="Token"/> object.</returns>
 480		public Token NextToken()
 481		{
 482			if (curToken == null) { // first call of NextToken()
 483				curToken = Next();
 484				//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
 485				return curToken;
 486			}
 487			
 488			lastToken = curToken;
 489			
 490			if (curToken.next == null) {
 491				curToken.next = Next();
 492			}
 493			
 494			curToken = curToken.next;
 495			
 496			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
 497				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, string.Empty);
 498				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, string.Empty);
 499			}
 500			//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
 501			return curToken;
 502		}
 503		
 504		#region VB Readers
 505		string ReadIdent(char ch)
 506		{
 507			char typeCharacter;
 508			return ReadIdent(ch, out typeCharacter);
 509		}
 510		
 511		string ReadIdent(char ch, out char typeCharacter)
 512		{
 513			typeCharacter = '\0';
 514			
 515			if (ef.ReadXmlIdentifier) {
 516				ef.ReadXmlIdentifier = false;
 517				return ReadXmlIdent(ch);
 518			}
 519			
 520			sb.Length = 0;
 521			sb.Append(ch);
 522			int peek;
 523			while ((peek = ReaderPeek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) {
 524				ReaderRead();
 525				sb.Append(ch.ToString());
 526			}
 527			if (peek == -1) {
 528				return sb.ToString();
 529			}
 530			
 531			if ("%&@!#$".IndexOf((char)peek) != -1) {
 532				typeCharacter = (char)peek;
 533				ReaderRead();
 534				if (typeCharacter == '!') {
 535					peek = ReaderPeek();
 536					if (peek != -1 && (peek == '_' || peek == '[' || char.IsLetter((char)peek))) {
 537						misreadExclamationMarkAsTypeCharacter = true;
 538					}
 539				}
 540			}
 541			return sb.ToString();
 542		}
 543		
 544		[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1818:DoNotConcatenateStringsInsideLoops")]
 545		Token ReadDigit(char ch, int x)
 546		{
 547			sb.Length = 0;
 548			sb.Append(ch);
 549			
 550			int y = Line;
 551			string digit = "";
 552			if (ch != '&') {
 553				digit += ch;
 554			}
 555			
 556			bool isHex      = false;
 557			bool isOct      = false;
 558			bool isSingle   = false;
 559			bool isDouble   = false;
 560			bool isDecimal  = false;
 561			
 562			if (ReaderPeek() == -1) {
 563				if (ch == '&') {
 564					errors.Error(Line, Col, String.Format("digit expected"));
 565				}
 566				return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0');
 567			}
 568			if (ch == '.') {
 569				if (Char.IsDigit((char)ReaderPeek())) {
 570					isDouble = true; // double is default
 571					if (isHex || isOct) {
 572						errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
 573					}
 574					while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot
 575						digit += (char)ReaderRead();
 576					}
 577				}
 578			} else if (ch == '&' && PeekUpperChar() == 'H') {
 579				const string hex = "0123456789ABCDEF";
 580				sb.Append((char)ReaderRead()); // skip 'H'
 581				while (ReaderPeek() != -1 && hex.IndexOf(PeekUpperChar()) != -1) {
 582					ch = (char)ReaderRead();
 583					sb.Append(ch);
 584					digit += Char.ToUpper(ch, CultureInfo.InvariantCulture);
 585				}
 586				isHex = true;
 587			} else if (ReaderPeek() != -1 && ch == '&' && PeekUpperChar() == 'O') {
 588				const string okt = "01234567";
 589				sb.Append((char)ReaderRead()); // skip 'O'
 590				while (ReaderPeek() != -1 && okt.IndexOf(PeekUpperChar()) != -1) {
 591					ch = (char)ReaderRead();
 592					sb.Append(ch);
 593					digit += Char.ToUpper(ch, CultureInfo.InvariantCulture);
 594				}
 595				isOct = true;
 596			} else {
 597				while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) {
 598					ch = (char)ReaderRead();;
 599					digit += ch;
 600					sb.Append(ch);
 601				}
 602			}
 603			
 604			if (digit.Length == 0) {
 605				errors.Error(Line, Col, String.Format("digit expected"));
 606				return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
 607			}
 608			
 609			if (ReaderPeek() != -1 && "%&SILU".IndexOf(PeekUpperChar()) != -1 || isHex || isOct) {
 610				bool unsigned = false;
 611				if (ReaderPeek() != -1) {
 612					ch = (char)ReaderPeek();
 613					sb.Append(ch);
 614					ch = Char.ToUpper(ch, CultureInfo.InvariantCulture);
 615					unsigned = ch == 'U';
 616					if (unsigned) {
 617						ReaderRead(); // read the U
 618						ch = (char)ReaderPeek();
 619						sb.Append(ch);
 620						ch = Char.ToUpper(ch, CultureInfo.InvariantCulture);
 621						if (ch != 'I' && ch != 'L' && ch != 'S') {
 622							errors.Error(Line, Col, "Invalid type character: U" + ch);
 623						}
 624					}
 625				}
 626				try {
 627					if (isOct) {
 628						ReaderRead();
 629						ulong number = 0L;
 630						for (int i = 0; i < digit.Length; ++i) {
 631							number = number * 8 + digit[i] - '0';
 632						}
 633						if (ch == 'S') {
 634							if (unsigned)
 635								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ushort)number);
 636							else
 637								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (short)number);
 638						} else if (ch == '%' || ch == 'I') {
 639							if (unsigned)
 640								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (uint)number);
 641							else
 642								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
 643						} else if (ch == '&' || ch == 'L') {
 644							if (unsigned)
 645								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ulong)number);
 646							else
 647								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number);
 648						} else {
 649							if (number > uint.MaxValue) {
 650								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number));
 651							} else {
 652								return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number));
 653							}
 654						}
 655					}
 656					if (ch == 'S') {
 657						ReaderRead();
 658						if (unsigned)
 659							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt16.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 660						else
 661							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int16.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 662					} else if (ch == '%' || ch == 'I') {
 663						ReaderRead();
 664						if (unsigned)
 665							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 666						else
 667							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 668					} else if (ch == '&' || ch == 'L') {
 669						ReaderRead();
 670						if (unsigned)
 671							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 672						else
 673							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 674					} else if (isHex) {
 675						ulong number = UInt64.Parse(digit, NumberStyles.HexNumber);
 676						if (number > uint.MaxValue) {
 677							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number));
 678						} else {
 679							return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number));
 680						}
 681					}
 682				} catch (OverflowException ex) {
 683					errors.Error(Line, Col, ex.Message);
 684					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
 685				} catch (FormatException) {
 686					errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
 687					return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
 688				}
 689			}
 690			Token nextToken = null; // if we accidently read a 'dot'
 691			if (!isDouble && ReaderPeek() == '.') { // read floating point number
 692				ReaderRead();
 693				if (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) {
 694					isDouble = true; // double is default
 695					if (isHex || isOct) {
 696						errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
 697					}
 698					digit += '.';
 699					while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot
 700						digit += (char)ReaderRead();
 701					}
 702				} else {
 703					nextToken = new Token(Tokens.Dot, Col - 1, Line);
 704				}
 705			}
 706			
 707			if (ReaderPeek() != -1 && PeekUpperChar() == 'E') { // read exponent
 708				isDouble = true;
 709				digit +=  (char)ReaderRead();
 710				if (ReaderPeek() != -1 && (ReaderPeek() == '-' || ReaderPeek() == '+')) {
 711					digit += (char)ReaderRead();
 712				}
 713				while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { // read exponent value
 714					digit += (char)ReaderRead();
 715				}
 716			}
 717			
 718			if (ReaderPeek() != -1) {
 719				switch (PeekUpperChar()) {
 720					case 'R':
 721					case '#':
 722						ReaderRead();
 723						isDouble = true;
 724						break;
 725					case 'D':
 726					case '@':
 727						ReaderRead();
 728						isDecimal = true;
 729						break;
 730					case 'F':
 731					case '!':
 732						ReaderRead();
 733						isSingle = true;
 734						break;
 735				}
 736			}
 737			
 738			try {
 739				if (isSingle) {
 740					return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), Single.Parse(digit, CultureInfo.InvariantCulture));
 741				}
 742				if (isDecimal) {
 743					return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), Decimal.Parse(digit, NumberStyles.Currency | NumberStyles.AllowExponent, CultureInfo.InvariantCulture));
 744				}
 745				if (isDouble) {
 746					return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), Double.Parse(digit, CultureInfo.InvariantCulture));
 747				}
 748			} catch (FormatException) {
 749				errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
 750				if (isSingle)
 751					return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), 0f);
 752				if (isDecimal)
 753					return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), 0m);
 754				if (isDouble)
 755					return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), 0.0);
 756			}
 757			Token token;
 758			try {
 759				token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 760			} catch (Exception) {
 761				try {
 762					token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
 763				} catch (FormatException) {
 764					errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
 765					// fallback, when nothing helps :)
 766					token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
 767				} catch (OverflowException) {
 768					errors.Error(Line, Col, String.Format("{0} is too long for a integer literal", digit));
 769					// fallback, when nothing helps :)
 770					token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
 771				}
 772			}
 773			token.next = nextToken;
 774			return token;
 775		}
 776		
 777		void ReadPreprocessorDirective()
 778		{
 779			TextLocation start = new TextLocation(Line, Col - 1);
 780			string directive = ReadIdent('#');
 781			// TODO : expression parser for PP directives
 782			// needed for proper conversion to e. g. C#
 783			string argument  = ReadToEndOfLine();
 784//			this.specialTracker.AddPreprocessingDirective(new PreprocessingDirective(directive, argument.Trim(), start, new AstLocation(start.Line, start.Column + directive.Length + argument.Length)));
 785		}
 786		
 787		string ReadDate()
 788		{
 789			char ch = '\0';
 790			sb.Length = 0;
 791			int nextChar;
 792			while ((nextChar = ReaderRead()) != -1) {
 793				ch = (char)nextChar;
 794				if (ch == '#') {
 795					break;
 796				} else if (ch == '\n') {
 797					errors.Error(Line, Col, String.Format("No return allowed inside Date literal"));
 798				} else {
 799					sb.Append(ch);
 800				}
 801			}
 802			if (ch != '#') {
 803				errors.Error(Line, Col, String.Format("End of File reached before Date literal terminated"));
 804			}
 805			return sb.ToString();
 806		}
 807		
 808		string ReadString()
 809		{
 810			char ch = '\0';
 811			sb.Length = 0;
 812			int nextChar;
 813			while ((nextChar = ReaderRead()) != -1) {
 814				ch = (char)nextChar;
 815				if (ch == '"') {
 816					if (ReaderPeek() != -1 && ReaderPeek() == '"') {
 817						sb.Append('"');
 818						ReaderRead();
 819					} else {
 820						break;
 821					}
 822				} else if (ch == '\n') {
 823					errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
 824				} else {
 825					sb.Append(ch);
 826				}
 827			}
 828			if (ch != '"') {
 829				errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
 830			}
 831			return sb.ToString();
 832		}
 833		
 834		void ReadComment()
 835		{
 836			TextLocation startPos = new TextLocation(Line, Col);
 837			sb.Length = 0;
 838			StringBuilder curWord = specialCommentHash != null ? new StringBuilder() : null;
 839			int missingApostrophes = 2; // no. of ' missing until it is a documentation comment
 840			int nextChar;
 841			while ((nextChar = ReaderRead()) != -1) {
 842				char ch = (char)nextChar;
 843				
 844				if (HandleLineEnd(ch)) {
 845					break;
 846				}
 847				
 848				sb.Append(ch);
 849				
 850				if (missingApostrophes > 0) {
 851					if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {
 852						if (--missingApostrophes == 0) {
 853//							specialTracker.StartComment(CommentType.Documentation, isAtLineBegin, startPos);
 854							sb.Length = 0;
 855						}
 856					} else {
 857//						specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos);
 858						missingApostrophes = 0;
 859					}
 860				}
 861				
 862				if (specialCommentHash != null) {
 863					if (Char.IsLetter(ch)) {
 864						curWord.Append(ch);
 865					} else {
 866						string tag = curWord.ToString();
 867						curWord.Length = 0;
 868						if (specialCommentHash.ContainsKey(tag)) {
 869							TextLocation p = new TextLocation(Line, Col);
 870							string comment = ch + ReadToEndOfLine();
 871//							this.TagComments.Add(new TagComment(tag, comment, isAtLineBegin, p, new Location(Col, Line)));
 872							sb.Append(comment);
 873							break;
 874						}
 875					}
 876				}
 877			}
 878//			if (missingApostrophes > 0) {
 879//				specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos);
 880//			}
 881//			specialTracker.AddString(sb.ToString());
 882//			specialTracker.FinishComment(new Location(Col, Line));
 883		}
 884		
 885		Token ReadOperator(char ch)
 886		{
 887			int x = Col - 1;
 888			int y = Line;
 889			switch(ch) {
 890				case '+':
 891					switch (ReaderPeek()) {
 892						case '=':
 893							ReaderRead();
 894							return new Token(Tokens.PlusAssign, x, y);
 895						default:
 896							break;
 897					}
 898					return new Token(Tokens.Plus, x, y);
 899				case '-':
 900					switch (ReaderPeek()) {
 901						case '=':
 902							ReaderRead();
 903							return new Token(Tokens.MinusAssign, x, y);
 904						default:
 905							break;
 906					}
 907					return new Token(Tokens.Minus, x, y);
 908				case '*':
 909					switch (ReaderPeek()) {
 910						case '=':
 911							ReaderRead();
 912							return new Token(Tokens.TimesAssign, x, y);
 913						default:
 914							break;
 915					}
 916					return new Token(Tokens.Times, x, y, "*");
 917				case '/':
 918					switch (ReaderPeek()) {
 919						case '=':
 920							ReaderRead();
 921							return new Token(Tokens.DivAssign, x, y);
 922						default:
 923							break;
 924					}
 925					return new Token(Tokens.Div, x, y);
 926				case '\\':
 927					switch (ReaderPeek()) {
 928						case '=':
 929							ReaderRead();
 930							return new Token(Tokens.DivIntegerAssign, x, y);
 931						default:
 932							break;
 933					}
 934					return new Token(Tokens.DivInteger, x, y);
 935				case '&':
 936					switch (ReaderPeek()) {
 937						case '=':
 938							ReaderRead();
 939							return new Token(Tokens.ConcatStringAssign, x, y);
 940						default:
 941							break;
 942					}
 943					return new Token(Tokens.ConcatString, x, y);
 944				case '^':
 945					switch (ReaderPeek()) {
 946						case '=':
 947							ReaderRead();
 948							return new Token(Tokens.PowerAssign, x, y);
 949						default:
 950							break;
 951					}
 952					return new Token(Tokens.Power, x, y);
 953				case ':':
 954					if (ReaderPeek() == '=') {
 955						ReaderRead();
 956						return new Token(Tokens.ColonAssign, x, y);
 957					}
 958					return new Token(Tokens.Colon, x, y);
 959				case '=':
 960					return new Token(Tokens.Assign, x, y);
 961				case '<':
 962					switch (ReaderPeek()) {
 963						case '=':
 964							ReaderRead();
 965							return new Token(Tokens.LessEqual, x, y);
 966						case '>':
 967							ReaderRead();
 968							return new Token(Tokens.NotEqual, x, y);
 969						case '<':
 970							ReaderRead();
 971							switch (ReaderPeek()) {
 972								case '=':
 973									ReaderRead();
 974									return new Token(Tokens.ShiftLeftAssign, x, y);
 975								default:
 976									break;
 977							}
 978							return new Token(Tokens.ShiftLeft, x, y);
 979					}
 980					return new Token(Tokens.LessThan, x, y);
 981				case '>':
 982					switch (ReaderPeek()) {
 983						case '=':
 984							ReaderRead();
 985							return new Token(Tokens.GreaterEqual, x, y);
 986						case '>':
 987							ReaderRead();
 988							if (ReaderPeek() != -1) {
 989								switch (ReaderPeek()) {
 990									case '=':
 991										ReaderRead();
 992										return new Token(Tokens.ShiftRightAssign, x, y);
 993									default:
 994										break;
 995								}
 996							}
 997							return new Token(Tokens.ShiftRight, x, y);
 998					}
 999					return new Token(Tokens.GreaterThan, x, y);
1000				case ',':
1001					return new Token(Tokens.Comma, x, y);
1002				case '.':
1003					// Prevent OverflowException when Peek returns -1
1004					int tmp = ReaderPeek(); int tmp2 = ReaderPeek(1);
1005					if (tmp > 0) {
1006						if (char.IsDigit((char)tmp))
1007							return ReadDigit('.', Col);
1008						else if ((char)tmp == '@') {
1009							ReaderRead();
1010							return new Token(Tokens.DotAt, x, y);
1011						} else if ((char)tmp == '.' && tmp2 > 0 && (char)tmp2 == '.') {
1012							ReaderRead(); ReaderRead();
1013							return new Token(Tokens.TripleDot, x, y);
1014						}
1015					}
1016					return new Token(Tokens.Dot, x, y);
1017				case '(':
1018					return new Token(Tokens.OpenParenthesis, x, y);
1019				case ')':
1020					return new Token(Tokens.CloseParenthesis, x, y);
1021				case '{':
1022					return new Token(Tokens.OpenCurlyBrace, x, y);
1023				case '}':
1024					return new Token(Tokens.CloseCurlyBrace, x, y);
1025				case '?':
1026					return new Token(Tokens.QuestionMark, x, y);
1027				case '!':
1028					return new Token(Tokens.ExclamationMark, x, y);
1029			}
1030			return null;
1031		}
1032		#endregion
1033		
1034		#region XML Readers
1035		Token ReadXmlProcessingInstruction(int x, int y)
1036		{
1037			sb.Length = 0;
1038			int nextChar = -1;
1039			
1040			while (ReaderPeek() != '?' || ReaderPeek(1) != '>') {
1041				nextChar = ReaderRead();
1042				if (nextChar == -1)
1043					break;
1044				sb.Append((char)nextChar);
1045			}
1046			
1047			ReaderSkip("?>".Length);
1048			
1049			return new Token(Tokens.XmlProcessingInstruction, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
1050		}
1051		
1052		Token ReadXmlCommentOrCData(int x, int y)
1053		{
1054			sb.Length = 0;
1055			int nextChar = -1;
1056			
1057			if (string.CompareOrdinal(ReaderPeekString("--".Length), "--") == 0) {
1058				ReaderSkip("--".Length);
1059				while ((nextChar = ReaderRead()) != -1) {
1060					sb.Append((char)nextChar);
1061					if (string.CompareOrdinal(ReaderPeekString("-->".Length), "-->") == 0) {
1062						ReaderSkip("-->".Length);
1063						return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
1064					}
1065				}
1066			}
1067			
1068			if (string.CompareOrdinal(ReaderPeekString("[CDATA[".Length), "[CDATA[") == 0) {
1069				ReaderSkip("[CDATA[".Length);
1070				while ((nextChar = ReaderRead()) != -1) {
1071					sb.Append((char)nextChar);
1072					if (string.CompareOrdinal(ReaderPeekString("]]>".Length), "]]>") == 0) {
1073						ReaderSkip("]]>".Length);
1074						return new Token(Tokens.XmlCData, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
1075					}
1076				}
1077			}
1078			
1079			return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
1080		}
1081		
1082		string ReadXmlContent(char ch)
1083		{
1084			sb.Length = 0;
1085			while (true) {
1086				sb.Append(ch);
1087				int next = ReaderPeek();
1088				
1089				if (next == -1 || next == '<')
1090					break;
1091				ch = (char)ReaderRead();
1092			}
1093			
1094			return sb.ToString();
1095		}
1096		
1097		string ReadXmlString(char terminator)
1098		{
1099			char ch = '\0';
1100			sb.Length = 0;
1101			int nextChar;
1102			while ((nextChar = ReaderRead()) != -1) {
1103				ch = (char)nextChar;
1104				if (ch == terminator) {
1105					break;
1106				} else if (ch == '\n') {
1107					errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
1108				} else {
1109					sb.Append(ch);
1110				}
1111			}
1112			if (ch != terminator) {
1113				errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
1114			}
1115			return sb.ToString();
1116		}
1117		
1118		string ReadXmlIdent(char ch)
1119		{
1120			sb.Length = 0;
1121			sb.Append(ch);
1122			
1123			int peek;
1124			
1125			while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) {
1126				sb.Append((char)ReaderRead());
1127			}
1128			
1129			return sb.ToString();
1130		}
1131		#endregion
1132		
1133		char PeekUpperChar()
1134		{
1135			return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture);
1136		}
1137		
1138		/// <summary>
1139		/// Skips to the end of the current code block.
1140		/// For this, the lexer must have read the next token AFTER the token opening the
1141		/// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).
1142		/// After the call, Lexer.LookAhead will be the block-closing token.
1143		/// </summary>
1144		public void SkipCurrentBlock(int targetToken)
1145		{
1146			int lastKind = -1;
1147			int kind = lastToken.kind;
1148			while (kind != Tokens.EOF &&
1149			       !(lastKind == Tokens.End && kind == targetToken))
1150			{
1151				lastKind = kind;
1152				NextToken();
1153				kind = lastToken.kind;
1154			}
1155		}
1156		
1157		public void SetInitialContext(SnippetType type)
1158		{
1159			ef.SetContext(type);
1160		}
1161		
1162		public VBLexerMemento Export()
1163		{
1164			return new VBLexerMemento() {
1165				Column = Col,
1166				Line = Line,
1167				EncounteredLineContinuation = encounteredLineContinuation,
1168				ExpressionFinder = ef.Export(),
1169				InXmlMode = inXmlMode,
1170				IsAtLineBegin = isAtLineBegin,
1171				LineEnd = lineEnd,
1172				PrevTokenKind = prevToken.kind,
1173				MisreadExclamationMarkAsTypeCharacter = misreadExclamationMarkAsTypeCharacter,
1174				XmlModeInfoStack = new Stack<XmlModeInfo>(xmlModeStack.Select(i => (XmlModeInfo)i.Clone()).Reverse())
1175			};
1176		}
1177		
1178		LATextReader reader;
1179		int col  = 1;
1180		int line = 1;
1181		
1182		protected Errors errors = new Errors();
1183		
1184		protected Token lastToken = null;
1185		protected Token curToken  = null;
1186		protected Token peekToken = null;
1187		
1188		string[]  specialCommentTags = null;
1189		protected Hashtable specialCommentHash  = null;
1190//		List<TagComment> tagComments  = new List<TagComment>();
1191		protected StringBuilder sb              = new StringBuilder();
1192//		protected SpecialTracker specialTracker = new SpecialTracker();
1193		
1194		// used for the original value of strings (with escape sequences).
1195		protected StringBuilder originalValue = new StringBuilder();
1196		
1197		public bool SkipAllComments { get; set; }
1198		public bool EvaluateConditionalCompilation { get; set; }
1199		public virtual IDictionary<string, object> ConditionalCompilationSymbols {
1200			get { throw new NotSupportedException(); }
1201		}
1202		
1203		protected static IEnumerable<string> GetSymbols (string symbols)
1204		{
1205			if (!string.IsNullOrEmpty(symbols)) {
1206				foreach (string symbol in symbols.Split (';', ' ', '\t')) {
1207					string s = symbol.Trim ();
1208					if (s.Length == 0)
1209						continue;
1210					yield return s;
1211				}
1212			}
1213		}
1214		
1215		public void SetConditionalCompilationSymbols (string symbols)
1216		{
1217			throw new NotSupportedException ();
1218		}
1219		
1220		protected int Line {
1221			get {
1222				return line;
1223			}
1224		}
1225		protected int Col {
1226			get {
1227				return col;
1228			}
1229		}
1230		
1231		protected bool recordRead = false;
1232		protected StringBuilder recordedText = new StringBuilder ();
1233		
1234		protected int ReaderRead()
1235		{
1236			int val = reader.Read();
1237			if (recordRead && val >= 0)
1238				recordedText.Append ((char)val);
1239			if ((val == '\r' && reader.Peek() != '\n') || val == '\n') {
1240				++line;
1241				col = 1;
1242				LineBreak();
1243			} else if (val >= 0) {
1244				col++;
1245			}
1246			return val;
1247		}
1248		
1249		protected int ReaderPeek()
1250		{
1251			return reader.Peek();
1252		}
1253		
1254		protected int ReaderPeek(int step)
1255		{
1256			return reader.Peek(step);
1257		}
1258		
1259		protected void ReaderSkip(int steps)
1260		{
1261			for (int i = 0; i < steps; i++) {
1262				ReaderRead();
1263			}
1264		}
1265		
1266		protected string ReaderPeekString(int length)
1267		{
1268			StringBuilder builder = new StringBuilder();
1269			
1270			for (int i = 0; i < length; i++) {
1271				int peek = ReaderPeek(i);
1272				if (peek != -1)
1273					builder.Append((char)peek);
1274			}
1275			
1276			return builder.ToString();
1277		}
1278		
1279		public void SetInitialLocation(TextLocation location)
1280		{
1281			if (lastToken != null || curToken != null || peekToken != null)
1282				throw new InvalidOperationException();
1283			this.line = location.Line;
1284			this.col = location.Column;
1285		}
1286		
1287		public Errors Errors {
1288			get {
1289				return errors;
1290			}
1291		}
1292		
1293		/// <summary>
1294		/// Returns the comments that had been read and containing tag key words.
1295		/// </summary>
1296//		public List<TagComment> TagComments {
1297//			get {
1298//				return tagComments;
1299//			}
1300//		}
1301		
1302//		public SpecialTracker SpecialTracker {
1303//			get {
1304//				return specialTracker;
1305//			}
1306//		}
1307		
1308		/// <summary>
1309		/// Special comment tags are tags like TODO, HACK or UNDONE which are read by the lexer and stored in <see cref="TagComments"/>.
1310		/// </summary>
1311		public string[] SpecialCommentTags {
1312			get {
1313				return specialCommentTags;
1314			}
1315			set {
1316				specialCommentTags = value;
1317				specialCommentHash = null;
1318				if (specialCommentTags != null && specialCommentTags.Length > 0) {
1319					specialCommentHash = new Hashtable();
1320					foreach (string str in specialCommentTags) {
1321						specialCommentHash.Add(str, null);
1322					}
1323				}
1324			}
1325		}
1326		
1327		/// <summary>
1328		/// The current Token. <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/>
1329		/// </summary>
1330		public Token Token {
1331			get {
1332//				Console.WriteLine("Call to Token");
1333				return lastToken;
1334			}
1335		}
1336		
1337		/// <summary>
1338		/// The next Token (The <see cref="Token"/> after <see cref="NextToken"/> call) . <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/>
1339		/// </summary>
1340		public Token LookAhead {
1341			get {
1342//				Console.WriteLine("Call to LookAhead");
1343				return curToken;
1344			}
1345		}
1346		
1347		#region System.IDisposable interface implementation
1348		public virtual void Dispose()
1349		{
1350			reader.Close();
1351			reader = null;
1352			errors = null;
1353			lastToken = curToken = peekToken = null;
1354			specialCommentHash = null;
1355			sb = originalValue = null;
1356		}
1357		#endregion
1358		
1359		/// <summary>
1360		/// Must be called before a peek operation.
1361		/// </summary>
1362		public void StartPeek()
1363		{
1364			peekToken = curToken;
1365		}
1366		
1367		/// <summary>
1368		/// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on.
1369		/// </summary>
1370		/// <returns>An <see cref="Token"/> object.</returns>
1371		public Token Peek()
1372		{
1373//			Console.WriteLine("Call to Peek");
1374			if (peekToken.next == null) {
1375				peekToken.next = Next();
1376			}
1377			peekToken = peekToken.next;
1378			return peekToken;
1379		}
1380		
1381		protected static bool IsIdentifierPart(int ch)
1382		{
1383			if (ch == 95) return true;  // 95 = '_'
1384			if (ch == -1) return false;
1385			return char.IsLetterOrDigit((char)ch); // accept unicode letters
1386		}
1387		
1388		protected static bool IsHex(char digit)
1389		{
1390			return Char.IsDigit(digit) || ('A' <= digit && digit <= 'F') || ('a' <= digit && digit <= 'f');
1391		}
1392		
1393		protected int GetHexNumber(char digit)
1394		{
1395			if (Char.IsDigit(digit)) {
1396				return digit - '0';
1397			}
1398			if ('A' <= digit && digit <= 'F') {
1399				return digit - 'A' + 0xA;
1400			}
1401			if ('a' <= digit && digit <= 'f') {
1402				return digit - 'a' + 0xA;
1403			}
1404			errors.Error(line, col, String.Format("Invalid hex number '" + digit + "'"));
1405			return 0;
1406		}
1407		protected TextLocation lastLineEnd = new TextLocation(1, 1);
1408		protected TextLocation curLineEnd = new TextLocation(1, 1);
1409		protected void LineBreak ()
1410		{
1411			lastLineEnd = curLineEnd;
1412			curLineEnd = new TextLocation (line, col - 1);
1413		}
1414		protected bool HandleLineEnd(char ch)
1415		{
1416			// Handle MS-DOS or MacOS line ends.
1417			if (ch == '\r') {
1418				if (reader.Peek() == '\n') { // MS-DOS line end '\r\n'
1419					ReaderRead(); // LineBreak (); called by ReaderRead ();
1420					return true;
1421				} else { // assume MacOS line end which is '\r'
1422					LineBreak ();
1423					return true;
1424				}
1425			}
1426			if (ch == '\n') {
1427				LineBreak ();
1428				return true;
1429			}
1430			return false;
1431		}
1432		
1433		protected void SkipToEndOfLine()
1434		{
1435			int nextChar;
1436			while ((nextChar = reader.Read()) != -1) {
1437				if (nextChar == '\r') {
1438					if (reader.Peek() == '\n')
1439						reader.Read();
1440					nextChar = '\n';
1441				}
1442				if (nextChar == '\n') {
1443					++line;
1444					col = 1;
1445					break;
1446				}
1447			}
1448		}
1449		
1450		protected string ReadToEndOfLine()
1451		{
1452			sb.Length = 0;
1453			int nextChar;
1454			while ((nextChar = reader.Read()) != -1) {
1455				char ch = (char)nextChar;
1456				
1457				if (nextChar == '\r') {
1458					if (reader.Peek() == '\n')
1459						reader.Read();
1460					nextChar = '\n';
1461				}
1462				// Return read string, if EOL is reached
1463				if (nextChar == '\n') {
1464					++line;
1465					col = 1;
1466					return sb.ToString();
1467				}
1468				
1469				sb.Append(ch);
1470			}
1471			
1472			// Got EOF before EOL
1473			string retStr = sb.ToString();
1474			col += retStr.Length;
1475			return retStr;
1476		}
1477		
1478		public event EventHandler<SavepointEventArgs> SavepointReached;
1479		
1480		protected virtual void OnSavepointReached(SavepointEventArgs e)
1481		{
1482			if (SavepointReached != null) {
1483				SavepointReached(this, e);
1484			}
1485		}
1486	}
1487}