/NRefactory/ICSharpCode.NRefactory.VB/Lexer/VBLexer.cs
C# | 1487 lines | 1473 code | 12 blank | 2 comment | 58 complexity | aea4d647444e07fb626ece55c7c34ace MD5 | raw file
1// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt) 2// This code is distributed under MIT X11 license (for details please see \doc\license.txt) 3 4using System; 5using System.Collections; 6using System.Collections.Generic; 7using System.Diagnostics; 8using System.Globalization; 9using System.IO; 10using System.Linq; 11using System.Text; 12using System.Xml; 13 14namespace ICSharpCode.NRefactory.VB.Parser 15{ 16 public class VBLexer : IDisposable 17 { 18 bool lineEnd = true; 19 bool isAtLineBegin = false; // TODO: handle line begin, if neccessarry 20 bool misreadExclamationMarkAsTypeCharacter; 21 bool encounteredLineContinuation; 22 23 ExpressionFinder ef; 24 25 bool inXmlMode; 26 27 Stack<XmlModeInfo> xmlModeStack = new Stack<XmlModeInfo>(); 28 29 public VBLexer(TextReader reader) 30 { 31 this.reader = new LATextReader(reader); 32 ef = new ExpressionFinder(); 33 } 34 35 public VBLexer(TextReader reader, VBLexerMemento state) : this(reader) 36 { 37 SetInitialLocation(new TextLocation(state.Line, state.Column)); 38 lastToken = new Token(state.PrevTokenKind, 0, 0); 39 ef = new ExpressionFinder(state.ExpressionFinder); 40 lineEnd = state.LineEnd; 41 isAtLineBegin = state.IsAtLineBegin; 42 encounteredLineContinuation = state.EncounteredLineContinuation; 43 misreadExclamationMarkAsTypeCharacter = state.MisreadExclamationMarkAsTypeCharacter; 44 xmlModeStack = new Stack<XmlModeInfo>(state.XmlModeInfoStack.Select(i => (XmlModeInfo)i.Clone()).Reverse()); 45 inXmlMode = state.InXmlMode; 46 } 47 48 Token NextInternal() 49 { 50 if (misreadExclamationMarkAsTypeCharacter) { 51 misreadExclamationMarkAsTypeCharacter = false; 52 return new Token(Tokens.ExclamationMark, Col - 1, Line); 53 } 54 55 unchecked { 56 while (true) { 57 TextLocation startLocation = new TextLocation(Line, Col); 58 int nextChar = ReaderRead(); 59 if (nextChar == -1) 60 return new Token(Tokens.EOF, Col, Line, string.Empty); 61 char ch = (char)nextChar; 62 #region XML mode 63 CheckXMLState(startLocation); 64 if (inXmlMode && xmlModeStack.Peek().level <= 0 && !xmlModeStack.Peek().isDocumentStart && !xmlModeStack.Peek().inXmlTag) { 65 XmlModeInfo info = xmlModeStack.Peek(); 66 int peek = nextChar; 67 while (true) { 68 int step = -1; 69 while (peek != -1 && XmlConvert.IsWhitespaceChar((char)peek)) { 70 step++; 71 peek = ReaderPeek(step); 72 } 73 74 if (peek == '<' && (ReaderPeek(step + 1) == '!' || ReaderPeek(step + 1) == '?')) { 75 char lastCh = '\0'; 76 for (int i = 0; i < step + 2; i++) 77 lastCh = (char)ReaderRead(); 78 79 if (lastCh == '!') 80 return ReadXmlCommentOrCData(Col - 2, Line); 81 else 82 return ReadXmlProcessingInstruction(Col - 2, Line); 83 } 84 85 break; 86 } 87 inXmlMode = false; 88 xmlModeStack.Pop(); 89 } 90 if (inXmlMode) { 91 XmlModeInfo info = xmlModeStack.Peek(); 92 int x = Col - 1; 93 int y = Line; 94 switch (ch) { 95 case '<': 96 if (ReaderPeek() == '/') { 97 ReaderRead(); 98 info.inXmlCloseTag = true; 99 return new Token(Tokens.XmlOpenEndTag, new TextLocation(y, x), new TextLocation(Line, Col)); 100 } 101 if (ReaderPeek() == '%' && ReaderPeek(1) == '=') { 102 inXmlMode = false; 103 ReaderRead(); ReaderRead(); 104 return new Token(Tokens.XmlStartInlineVB, new TextLocation(y, x), new TextLocation(Line, Col)); 105 } 106 if (ReaderPeek() == '?') { 107 ReaderRead(); 108 Token t = ReadXmlProcessingInstruction(x, y); 109 return t; 110 } 111 if (ReaderPeek() == '!') { 112 ReaderRead(); 113 Token token = ReadXmlCommentOrCData(x, y); 114 return token; 115 } 116 info.level++; 117 info.isDocumentStart = false; 118 info.inXmlTag = true; 119 return new Token(Tokens.XmlOpenTag, x, y); 120 case '/': 121 if (ReaderPeek() == '>') { 122 ReaderRead(); 123 info.inXmlTag = false; 124 info.level--; 125 return new Token(Tokens.XmlCloseTagEmptyElement, new TextLocation(y, x), new TextLocation(Line, Col)); 126 } 127 break; 128 case '>': 129 if (info.inXmlCloseTag) 130 info.level--; 131 info.inXmlTag = info.inXmlCloseTag = false; 132 return new Token(Tokens.XmlCloseTag, x, y); 133 case '=': 134 return new Token(Tokens.Assign, x, y); 135 case '\'': 136 case '"': 137 string s = ReadXmlString(ch); 138 return new Token(Tokens.LiteralString, x, y, ch + s + ch, s); 139 default: 140 if (info.inXmlCloseTag || info.inXmlTag) { 141 if (XmlConvert.IsWhitespaceChar(ch)) 142 continue; 143 return new Token(Tokens.Identifier, x, y, ReadXmlIdent(ch)); 144 } else { 145 string content = ReadXmlContent(ch); 146 return new Token(Tokens.XmlContent, startLocation, new TextLocation(Line, Col), content, null); 147 } 148 } 149 #endregion 150 } else { 151 #region Standard Mode 152 if (Char.IsWhiteSpace(ch)) { 153 if (HandleLineEnd(ch)) { 154 if (lineEnd) { 155 // second line end before getting to a token 156 // -> here was a blank line 157// specialTracker.AddEndOfLine(startLocation); 158 } else { 159 lineEnd = true; 160 return new Token(Tokens.EOL, startLocation, new TextLocation(Line, Col), null, null); 161 } 162 } 163 continue; 164 } 165 if (ch == '_') { 166 if (ReaderPeek() == -1) { 167 errors.Error(Line, Col, String.Format("No EOF expected after _")); 168 return new Token(Tokens.EOF, Col, Line, string.Empty); 169 } 170 if (!Char.IsWhiteSpace((char)ReaderPeek())) { 171 int x = Col - 1; 172 int y = Line; 173 string s = ReadIdent('_'); 174 lineEnd = false; 175 return new Token(Tokens.Identifier, x, y, s); 176 } 177 encounteredLineContinuation = true; 178 ch = (char)ReaderRead(); 179 180 bool oldLineEnd = lineEnd; 181 lineEnd = false; 182 while (Char.IsWhiteSpace(ch)) { 183 if (HandleLineEnd(ch)) { 184 lineEnd = true; 185 break; 186 } 187 if (ReaderPeek() != -1) { 188 ch = (char)ReaderRead(); 189 } else { 190 errors.Error(Line, Col, String.Format("No EOF expected after _")); 191 return new Token(Tokens.EOF, Col, Line, string.Empty); 192 } 193 } 194 if (!lineEnd) { 195 errors.Error(Line, Col, String.Format("NewLine expected")); 196 } 197 lineEnd = oldLineEnd; 198 continue; 199 } 200 201 if (ch == '#') { 202 while (Char.IsWhiteSpace((char)ReaderPeek())) { 203 ReaderRead(); 204 } 205 if (Char.IsDigit((char)ReaderPeek())) { 206 int x = Col - 1; 207 int y = Line; 208 string s = ReadDate(); 209 DateTime time = new DateTime(1, 1, 1, 0, 0, 0); 210 try { 211 time = DateTime.Parse(s, System.Globalization.CultureInfo.InvariantCulture, DateTimeStyles.NoCurrentDateDefault); 212 } catch (Exception e) { 213 errors.Error(Line, Col, String.Format("Invalid date time {0}", e)); 214 } 215 return new Token(Tokens.LiteralDate, x, y, s, time); 216 } else { 217 ReadPreprocessorDirective(); 218 continue; 219 } 220 } 221 222 if (ch == '[') { // Identifier 223 lineEnd = false; 224 if (ReaderPeek() == -1) { 225 errors.Error(Line, Col, String.Format("Identifier expected")); 226 } 227 ch = (char)ReaderRead(); 228 if (ch == ']' || Char.IsWhiteSpace(ch)) { 229 errors.Error(Line, Col, String.Format("Identifier expected")); 230 } 231 int x = Col - 1; 232 int y = Line; 233 string s = ReadIdent(ch); 234 if (ReaderPeek() == -1) { 235 errors.Error(Line, Col, String.Format("']' expected")); 236 } 237 ch = (char)ReaderRead(); 238 if (!(ch == ']')) { 239 errors.Error(Line, Col, String.Format("']' expected")); 240 } 241 return new Token(Tokens.Identifier, x, y, s); 242 } 243 if (Char.IsLetter(ch)) { 244 int x = Col - 1; 245 int y = Line; 246 char typeCharacter; 247 string s = ReadIdent(ch, out typeCharacter); 248 if (typeCharacter == '\0') { 249 int keyWordToken = Keywords.GetToken(s); 250 if (keyWordToken >= 0) { 251 // handle 'REM' comments 252 if (keyWordToken == Tokens.Rem) { 253 ReadComment(); 254 if (!lineEnd) { 255 lineEnd = true; 256 return new Token(Tokens.EOL, Col, Line, "\n"); 257 } 258 continue; 259 } 260 261 lineEnd = false; 262 return new Token(keyWordToken, x, y, s); 263 } 264 } 265 266 lineEnd = false; 267 return new Token(Tokens.Identifier, x, y, s); 268 269 } 270 if (Char.IsDigit(ch)) { 271 lineEnd = false; 272 return ReadDigit(ch, Col - 1); 273 } 274 if (ch == '&') { 275 lineEnd = false; 276 if (ReaderPeek() == -1) { 277 return ReadOperator('&'); 278 } 279 ch = (char)ReaderPeek(); 280 if (Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'H' || Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'O') { 281 return ReadDigit('&', Col - 1); 282 } 283 return ReadOperator('&'); 284 } 285 if (ch == '\'' || ch == '\u2018' || ch == '\u2019') { 286 int x = Col - 1; 287 int y = Line; 288 ReadComment(); 289 if (!lineEnd) { 290 lineEnd = true; 291 return new Token(Tokens.EOL, x, y, "\n"); 292 } 293 continue; 294 } 295 if (ch == '"') { 296 lineEnd = false; 297 int x = Col - 1; 298 int y = Line; 299 string s = ReadString(); 300 if (ReaderPeek() != -1 && (ReaderPeek() == 'C' || ReaderPeek() == 'c')) { 301 ReaderRead(); 302 if (s.Length != 1) { 303 errors.Error(Line, Col, String.Format("Chars can only have Length 1 ")); 304 } 305 if (s.Length == 0) { 306 s = "\0"; 307 } 308 return new Token(Tokens.LiteralCharacter, x, y, '"' + s + "\"C", s[0]); 309 } 310 return new Token(Tokens.LiteralString, x, y, '"' + s + '"', s); 311 } 312 if (ch == '%' && ReaderPeek() == '>') { 313 int x = Col - 1; 314 int y = Line; 315 inXmlMode = true; 316 ReaderRead(); 317 return new Token(Tokens.XmlEndInlineVB, new TextLocation(y, x), new TextLocation(Line, Col)); 318 } 319 #endregion 320 if (ch == '<' && (ef.NextTokenIsPotentialStartOfExpression || ef.NextTokenIsStartOfImportsOrAccessExpression)) { 321 xmlModeStack.Push(new XmlModeInfo(ef.NextTokenIsStartOfImportsOrAccessExpression)); 322 XmlModeInfo info = xmlModeStack.Peek(); 323 int x = Col - 1; 324 int y = Line; 325 inXmlMode = true; 326 if (ReaderPeek() == '/') { 327 ReaderRead(); 328 info.inXmlCloseTag = true; 329 return new Token(Tokens.XmlOpenEndTag, new TextLocation(y, x), new TextLocation(Line, Col)); 330 } 331 // should we allow <%= at start of an expression? not valid with vbc ... 332 if (ReaderPeek() == '%' && ReaderPeek(1) == '=') { 333 inXmlMode = false; 334 ReaderRead(); ReaderRead(); 335 return new Token(Tokens.XmlStartInlineVB, new TextLocation(y, x), new TextLocation(Line, Col)); 336 } 337 if (ReaderPeek() == '!') { 338 ReaderRead(); 339 Token t = ReadXmlCommentOrCData(x, y); 340 return t; 341 } 342 if (ReaderPeek() == '?') { 343 ReaderRead(); 344 Token t = ReadXmlProcessingInstruction(x, y); 345 info.isDocumentStart = t.val.Trim().StartsWith("xml", StringComparison.OrdinalIgnoreCase); 346 return t; 347 } 348 info.inXmlTag = true; 349 info.level++; 350 return new Token(Tokens.XmlOpenTag, x, y); 351 } 352 Token token = ReadOperator(ch); 353 if (token != null) { 354 lineEnd = false; 355 return token; 356 } 357 } 358 359 errors.Error(Line, Col, String.Format("Unknown char({0}) which can't be read", ch)); 360 } 361 } 362 } 363 364 void CheckXMLState(TextLocation startLocation) 365 { 366 if (inXmlMode && !xmlModeStack.Any()) 367 throw new InvalidOperationException("invalid XML stack state at " + startLocation); 368 } 369 370 Token prevToken; 371 372 Token Next() 373 { 374 Token t = NextInternal(); 375 if (t.kind == Tokens.EOL) { 376 Debug.Assert(t.next == null); // NextInternal() must return only 1 token 377 t.next = NextInternal(); 378 Debug.Assert(t.next.next == null); 379 if (SkipEOL(prevToken.kind, t.next.kind)) { 380 t = t.next; 381 } 382 } else 383 encounteredLineContinuation = false; 384 // inform EF only once we're sure it's really a token 385 // this means we inform it about EOL tokens "1 token too late", but that's not a problem because 386 // XML literals cannot start immediately after an EOL token 387 ef.InformToken(t); 388 if (t.next != null) { 389 // Next() isn't called again when it returns 2 tokens, so we need to process both tokens 390 ef.InformToken(t.next); 391 prevToken = t.next; 392 } else { 393 prevToken = t; 394 } 395 ef.Advance(); 396 Debug.Assert(t != null); 397 return t; 398 } 399 400 /// <remarks>see VB language specification 10; pg. 6</remarks> 401 bool SkipEOL(int prevTokenKind, int nextTokenKind) 402 { 403 // exception directly after _ 404 if (encounteredLineContinuation) { 405 return encounteredLineContinuation = false; 406 } 407 408 // 1st rule 409 // after a comma (,), open parenthesis ((), open curly brace ({), or open embedded expression (<%=) 410 if (new[] { Tokens.Comma, Tokens.OpenParenthesis, Tokens.OpenCurlyBrace, Tokens.XmlStartInlineVB } 411 .Contains(prevTokenKind)) 412 return true; 413 414 // 2nd rule 415 // after a member qualifier (. or .@ or ...), provided that something is being qualified (i.e. is not 416 // using an implicit With context) 417 if (new[] { Tokens.Dot, Tokens.DotAt, Tokens.TripleDot }.Contains(prevTokenKind) 418 && !ef.WasQualifierTokenAtStart) 419 return true; 420 421 // 3rd rule 422 // before a close parenthesis ()), close curly brace (}), or close embedded expression (%>) 423 if (new[] { Tokens.CloseParenthesis, Tokens.CloseCurlyBrace, Tokens.XmlEndInlineVB } 424 .Contains(nextTokenKind)) 425 return true; 426 427 // 4th rule 428 // after a less-than (<) in an attribute context 429 if (prevTokenKind == Tokens.LessThan && ef.InContext(Context.Attribute)) 430 return true; 431 432 // 5th rule 433 // before a greater-than (>) in an attribute context 434 if (nextTokenKind == Tokens.GreaterThan && ef.InContext(Context.Attribute)) 435 return true; 436 437 // 6th rule 438 // after a greater-than (>) in a non-file-level attribute context 439 if (ef.WasNormalAttribute && prevTokenKind == Tokens.GreaterThan) 440 return true; 441 442 // 7th rule 443 // before and after query operators (Where, Order, Select, etc.) 444 var queryOperators = new int[] { Tokens.From, Tokens.Aggregate, Tokens.Select, Tokens.Distinct, 445 Tokens.Where, Tokens.Order, Tokens.By, Tokens.Ascending, Tokens.Descending, Tokens.Take, 446 Tokens.Skip, Tokens.Let, Tokens.Group, Tokens.Into, Tokens.On, Tokens.While, Tokens.Join }; 447 if (ef.InContext(Context.Query)) { 448 // Ascending, Descending, Distinct are special 449 // fixes http://community.sharpdevelop.net/forums/p/12068/32893.aspx#32893 450 var specialQueryOperators = new int[] { Tokens.Ascending, Tokens.Descending, Tokens.Distinct }; 451 if (specialQueryOperators.Contains(prevTokenKind) && !queryOperators.Contains(nextTokenKind)) 452 return false; 453 454 if ((queryOperators.Contains(prevTokenKind) || queryOperators.Contains(nextTokenKind))) 455 return true; 456 } 457 458 // 8th rule 459 // after binary operators (+, -, /, *, etc.) in an expression context 460 if (new[] { Tokens.Plus, Tokens.Minus, Tokens.Div, Tokens.DivInteger, Tokens.Times, Tokens.Mod, Tokens.Power, 461 Tokens.Assign, Tokens.NotEqual, Tokens.LessThan, Tokens.LessEqual, Tokens.GreaterThan, Tokens.GreaterEqual, 462 Tokens.Like, Tokens.ConcatString, Tokens.AndAlso, Tokens.OrElse, Tokens.And, Tokens.Or, Tokens.Xor, 463 Tokens.ShiftLeft, Tokens.ShiftRight }.Contains(prevTokenKind) && ef.CurrentBlock.context == Context.Expression) 464 return true; 465 466 // 9th rule 467 // after assignment operators (=, :=, +=, -=, etc.) in any context. 468 if (new[] { Tokens.Assign, Tokens.ColonAssign, Tokens.ConcatStringAssign, Tokens.DivAssign, 469 Tokens.DivIntegerAssign, Tokens.MinusAssign, Tokens.PlusAssign, Tokens.PowerAssign, 470 Tokens.ShiftLeftAssign, Tokens.ShiftRightAssign, Tokens.TimesAssign }.Contains(prevTokenKind)) 471 return true; 472 473 return false; 474 } 475 476 /// <summary> 477 /// Reads the next token. 478 /// </summary> 479 /// <returns>A <see cref="Token"/> object.</returns> 480 public Token NextToken() 481 { 482 if (curToken == null) { // first call of NextToken() 483 curToken = Next(); 484 //Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val); 485 return curToken; 486 } 487 488 lastToken = curToken; 489 490 if (curToken.next == null) { 491 curToken.next = Next(); 492 } 493 494 curToken = curToken.next; 495 496 if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token 497 curToken = new Token(Tokens.EOL, curToken.col, curToken.line, string.Empty); 498 curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, string.Empty); 499 } 500 //Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val); 501 return curToken; 502 } 503 504 #region VB Readers 505 string ReadIdent(char ch) 506 { 507 char typeCharacter; 508 return ReadIdent(ch, out typeCharacter); 509 } 510 511 string ReadIdent(char ch, out char typeCharacter) 512 { 513 typeCharacter = '\0'; 514 515 if (ef.ReadXmlIdentifier) { 516 ef.ReadXmlIdentifier = false; 517 return ReadXmlIdent(ch); 518 } 519 520 sb.Length = 0; 521 sb.Append(ch); 522 int peek; 523 while ((peek = ReaderPeek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) { 524 ReaderRead(); 525 sb.Append(ch.ToString()); 526 } 527 if (peek == -1) { 528 return sb.ToString(); 529 } 530 531 if ("%&@!#$".IndexOf((char)peek) != -1) { 532 typeCharacter = (char)peek; 533 ReaderRead(); 534 if (typeCharacter == '!') { 535 peek = ReaderPeek(); 536 if (peek != -1 && (peek == '_' || peek == '[' || char.IsLetter((char)peek))) { 537 misreadExclamationMarkAsTypeCharacter = true; 538 } 539 } 540 } 541 return sb.ToString(); 542 } 543 544 [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1818:DoNotConcatenateStringsInsideLoops")] 545 Token ReadDigit(char ch, int x) 546 { 547 sb.Length = 0; 548 sb.Append(ch); 549 550 int y = Line; 551 string digit = ""; 552 if (ch != '&') { 553 digit += ch; 554 } 555 556 bool isHex = false; 557 bool isOct = false; 558 bool isSingle = false; 559 bool isDouble = false; 560 bool isDecimal = false; 561 562 if (ReaderPeek() == -1) { 563 if (ch == '&') { 564 errors.Error(Line, Col, String.Format("digit expected")); 565 } 566 return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0'); 567 } 568 if (ch == '.') { 569 if (Char.IsDigit((char)ReaderPeek())) { 570 isDouble = true; // double is default 571 if (isHex || isOct) { 572 errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed")); 573 } 574 while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot 575 digit += (char)ReaderRead(); 576 } 577 } 578 } else if (ch == '&' && PeekUpperChar() == 'H') { 579 const string hex = "0123456789ABCDEF"; 580 sb.Append((char)ReaderRead()); // skip 'H' 581 while (ReaderPeek() != -1 && hex.IndexOf(PeekUpperChar()) != -1) { 582 ch = (char)ReaderRead(); 583 sb.Append(ch); 584 digit += Char.ToUpper(ch, CultureInfo.InvariantCulture); 585 } 586 isHex = true; 587 } else if (ReaderPeek() != -1 && ch == '&' && PeekUpperChar() == 'O') { 588 const string okt = "01234567"; 589 sb.Append((char)ReaderRead()); // skip 'O' 590 while (ReaderPeek() != -1 && okt.IndexOf(PeekUpperChar()) != -1) { 591 ch = (char)ReaderRead(); 592 sb.Append(ch); 593 digit += Char.ToUpper(ch, CultureInfo.InvariantCulture); 594 } 595 isOct = true; 596 } else { 597 while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { 598 ch = (char)ReaderRead();; 599 digit += ch; 600 sb.Append(ch); 601 } 602 } 603 604 if (digit.Length == 0) { 605 errors.Error(Line, Col, String.Format("digit expected")); 606 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0); 607 } 608 609 if (ReaderPeek() != -1 && "%&SILU".IndexOf(PeekUpperChar()) != -1 || isHex || isOct) { 610 bool unsigned = false; 611 if (ReaderPeek() != -1) { 612 ch = (char)ReaderPeek(); 613 sb.Append(ch); 614 ch = Char.ToUpper(ch, CultureInfo.InvariantCulture); 615 unsigned = ch == 'U'; 616 if (unsigned) { 617 ReaderRead(); // read the U 618 ch = (char)ReaderPeek(); 619 sb.Append(ch); 620 ch = Char.ToUpper(ch, CultureInfo.InvariantCulture); 621 if (ch != 'I' && ch != 'L' && ch != 'S') { 622 errors.Error(Line, Col, "Invalid type character: U" + ch); 623 } 624 } 625 } 626 try { 627 if (isOct) { 628 ReaderRead(); 629 ulong number = 0L; 630 for (int i = 0; i < digit.Length; ++i) { 631 number = number * 8 + digit[i] - '0'; 632 } 633 if (ch == 'S') { 634 if (unsigned) 635 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ushort)number); 636 else 637 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (short)number); 638 } else if (ch == '%' || ch == 'I') { 639 if (unsigned) 640 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (uint)number); 641 else 642 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number); 643 } else if (ch == '&' || ch == 'L') { 644 if (unsigned) 645 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ulong)number); 646 else 647 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number); 648 } else { 649 if (number > uint.MaxValue) { 650 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number)); 651 } else { 652 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number)); 653 } 654 } 655 } 656 if (ch == 'S') { 657 ReaderRead(); 658 if (unsigned) 659 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt16.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 660 else 661 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int16.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 662 } else if (ch == '%' || ch == 'I') { 663 ReaderRead(); 664 if (unsigned) 665 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 666 else 667 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 668 } else if (ch == '&' || ch == 'L') { 669 ReaderRead(); 670 if (unsigned) 671 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 672 else 673 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 674 } else if (isHex) { 675 ulong number = UInt64.Parse(digit, NumberStyles.HexNumber); 676 if (number > uint.MaxValue) { 677 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number)); 678 } else { 679 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number)); 680 } 681 } 682 } catch (OverflowException ex) { 683 errors.Error(Line, Col, ex.Message); 684 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0); 685 } catch (FormatException) { 686 errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit)); 687 return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0); 688 } 689 } 690 Token nextToken = null; // if we accidently read a 'dot' 691 if (!isDouble && ReaderPeek() == '.') { // read floating point number 692 ReaderRead(); 693 if (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { 694 isDouble = true; // double is default 695 if (isHex || isOct) { 696 errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed")); 697 } 698 digit += '.'; 699 while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot 700 digit += (char)ReaderRead(); 701 } 702 } else { 703 nextToken = new Token(Tokens.Dot, Col - 1, Line); 704 } 705 } 706 707 if (ReaderPeek() != -1 && PeekUpperChar() == 'E') { // read exponent 708 isDouble = true; 709 digit += (char)ReaderRead(); 710 if (ReaderPeek() != -1 && (ReaderPeek() == '-' || ReaderPeek() == '+')) { 711 digit += (char)ReaderRead(); 712 } 713 while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { // read exponent value 714 digit += (char)ReaderRead(); 715 } 716 } 717 718 if (ReaderPeek() != -1) { 719 switch (PeekUpperChar()) { 720 case 'R': 721 case '#': 722 ReaderRead(); 723 isDouble = true; 724 break; 725 case 'D': 726 case '@': 727 ReaderRead(); 728 isDecimal = true; 729 break; 730 case 'F': 731 case '!': 732 ReaderRead(); 733 isSingle = true; 734 break; 735 } 736 } 737 738 try { 739 if (isSingle) { 740 return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), Single.Parse(digit, CultureInfo.InvariantCulture)); 741 } 742 if (isDecimal) { 743 return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), Decimal.Parse(digit, NumberStyles.Currency | NumberStyles.AllowExponent, CultureInfo.InvariantCulture)); 744 } 745 if (isDouble) { 746 return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), Double.Parse(digit, CultureInfo.InvariantCulture)); 747 } 748 } catch (FormatException) { 749 errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit)); 750 if (isSingle) 751 return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), 0f); 752 if (isDecimal) 753 return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), 0m); 754 if (isDouble) 755 return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), 0.0); 756 } 757 Token token; 758 try { 759 token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 760 } catch (Exception) { 761 try { 762 token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number)); 763 } catch (FormatException) { 764 errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit)); 765 // fallback, when nothing helps :) 766 token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0); 767 } catch (OverflowException) { 768 errors.Error(Line, Col, String.Format("{0} is too long for a integer literal", digit)); 769 // fallback, when nothing helps :) 770 token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0); 771 } 772 } 773 token.next = nextToken; 774 return token; 775 } 776 777 void ReadPreprocessorDirective() 778 { 779 TextLocation start = new TextLocation(Line, Col - 1); 780 string directive = ReadIdent('#'); 781 // TODO : expression parser for PP directives 782 // needed for proper conversion to e. g. C# 783 string argument = ReadToEndOfLine(); 784// this.specialTracker.AddPreprocessingDirective(new PreprocessingDirective(directive, argument.Trim(), start, new AstLocation(start.Line, start.Column + directive.Length + argument.Length))); 785 } 786 787 string ReadDate() 788 { 789 char ch = '\0'; 790 sb.Length = 0; 791 int nextChar; 792 while ((nextChar = ReaderRead()) != -1) { 793 ch = (char)nextChar; 794 if (ch == '#') { 795 break; 796 } else if (ch == '\n') { 797 errors.Error(Line, Col, String.Format("No return allowed inside Date literal")); 798 } else { 799 sb.Append(ch); 800 } 801 } 802 if (ch != '#') { 803 errors.Error(Line, Col, String.Format("End of File reached before Date literal terminated")); 804 } 805 return sb.ToString(); 806 } 807 808 string ReadString() 809 { 810 char ch = '\0'; 811 sb.Length = 0; 812 int nextChar; 813 while ((nextChar = ReaderRead()) != -1) { 814 ch = (char)nextChar; 815 if (ch == '"') { 816 if (ReaderPeek() != -1 && ReaderPeek() == '"') { 817 sb.Append('"'); 818 ReaderRead(); 819 } else { 820 break; 821 } 822 } else if (ch == '\n') { 823 errors.Error(Line, Col, String.Format("No return allowed inside String literal")); 824 } else { 825 sb.Append(ch); 826 } 827 } 828 if (ch != '"') { 829 errors.Error(Line, Col, String.Format("End of File reached before String terminated ")); 830 } 831 return sb.ToString(); 832 } 833 834 void ReadComment() 835 { 836 TextLocation startPos = new TextLocation(Line, Col); 837 sb.Length = 0; 838 StringBuilder curWord = specialCommentHash != null ? new StringBuilder() : null; 839 int missingApostrophes = 2; // no. of ' missing until it is a documentation comment 840 int nextChar; 841 while ((nextChar = ReaderRead()) != -1) { 842 char ch = (char)nextChar; 843 844 if (HandleLineEnd(ch)) { 845 break; 846 } 847 848 sb.Append(ch); 849 850 if (missingApostrophes > 0) { 851 if (ch == '\'' || ch == '\u2018' || ch == '\u2019') { 852 if (--missingApostrophes == 0) { 853// specialTracker.StartComment(CommentType.Documentation, isAtLineBegin, startPos); 854 sb.Length = 0; 855 } 856 } else { 857// specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos); 858 missingApostrophes = 0; 859 } 860 } 861 862 if (specialCommentHash != null) { 863 if (Char.IsLetter(ch)) { 864 curWord.Append(ch); 865 } else { 866 string tag = curWord.ToString(); 867 curWord.Length = 0; 868 if (specialCommentHash.ContainsKey(tag)) { 869 TextLocation p = new TextLocation(Line, Col); 870 string comment = ch + ReadToEndOfLine(); 871// this.TagComments.Add(new TagComment(tag, comment, isAtLineBegin, p, new Location(Col, Line))); 872 sb.Append(comment); 873 break; 874 } 875 } 876 } 877 } 878// if (missingApostrophes > 0) { 879// specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos); 880// } 881// specialTracker.AddString(sb.ToString()); 882// specialTracker.FinishComment(new Location(Col, Line)); 883 } 884 885 Token ReadOperator(char ch) 886 { 887 int x = Col - 1; 888 int y = Line; 889 switch(ch) { 890 case '+': 891 switch (ReaderPeek()) { 892 case '=': 893 ReaderRead(); 894 return new Token(Tokens.PlusAssign, x, y); 895 default: 896 break; 897 } 898 return new Token(Tokens.Plus, x, y); 899 case '-': 900 switch (ReaderPeek()) { 901 case '=': 902 ReaderRead(); 903 return new Token(Tokens.MinusAssign, x, y); 904 default: 905 break; 906 } 907 return new Token(Tokens.Minus, x, y); 908 case '*': 909 switch (ReaderPeek()) { 910 case '=': 911 ReaderRead(); 912 return new Token(Tokens.TimesAssign, x, y); 913 default: 914 break; 915 } 916 return new Token(Tokens.Times, x, y, "*"); 917 case '/': 918 switch (ReaderPeek()) { 919 case '=': 920 ReaderRead(); 921 return new Token(Tokens.DivAssign, x, y); 922 default: 923 break; 924 } 925 return new Token(Tokens.Div, x, y); 926 case '\\': 927 switch (ReaderPeek()) { 928 case '=': 929 ReaderRead(); 930 return new Token(Tokens.DivIntegerAssign, x, y); 931 default: 932 break; 933 } 934 return new Token(Tokens.DivInteger, x, y); 935 case '&': 936 switch (ReaderPeek()) { 937 case '=': 938 ReaderRead(); 939 return new Token(Tokens.ConcatStringAssign, x, y); 940 default: 941 break; 942 } 943 return new Token(Tokens.ConcatString, x, y); 944 case '^': 945 switch (ReaderPeek()) { 946 case '=': 947 ReaderRead(); 948 return new Token(Tokens.PowerAssign, x, y); 949 default: 950 break; 951 } 952 return new Token(Tokens.Power, x, y); 953 case ':': 954 if (ReaderPeek() == '=') { 955 ReaderRead(); 956 return new Token(Tokens.ColonAssign, x, y); 957 } 958 return new Token(Tokens.Colon, x, y); 959 case '=': 960 return new Token(Tokens.Assign, x, y); 961 case '<': 962 switch (ReaderPeek()) { 963 case '=': 964 ReaderRead(); 965 return new Token(Tokens.LessEqual, x, y); 966 case '>': 967 ReaderRead(); 968 return new Token(Tokens.NotEqual, x, y); 969 case '<': 970 ReaderRead(); 971 switch (ReaderPeek()) { 972 case '=': 973 ReaderRead(); 974 return new Token(Tokens.ShiftLeftAssign, x, y); 975 default: 976 break; 977 } 978 return new Token(Tokens.ShiftLeft, x, y); 979 } 980 return new Token(Tokens.LessThan, x, y); 981 case '>': 982 switch (ReaderPeek()) { 983 case '=': 984 ReaderRead(); 985 return new Token(Tokens.GreaterEqual, x, y); 986 case '>': 987 ReaderRead(); 988 if (ReaderPeek() != -1) { 989 switch (ReaderPeek()) { 990 case '=': 991 ReaderRead(); 992 return new Token(Tokens.ShiftRightAssign, x, y); 993 default: 994 break; 995 } 996 } 997 return new Token(Tokens.ShiftRight, x, y); 998 } 999 return new Token(Tokens.GreaterThan, x, y); 1000 case ',': 1001 return new Token(Tokens.Comma, x, y); 1002 case '.': 1003 // Prevent OverflowException when Peek returns -1 1004 int tmp = ReaderPeek(); int tmp2 = ReaderPeek(1); 1005 if (tmp > 0) { 1006 if (char.IsDigit((char)tmp)) 1007 return ReadDigit('.', Col); 1008 else if ((char)tmp == '@') { 1009 ReaderRead(); 1010 return new Token(Tokens.DotAt, x, y); 1011 } else if ((char)tmp == '.' && tmp2 > 0 && (char)tmp2 == '.') { 1012 ReaderRead(); ReaderRead(); 1013 return new Token(Tokens.TripleDot, x, y); 1014 } 1015 } 1016 return new Token(Tokens.Dot, x, y); 1017 case '(': 1018 return new Token(Tokens.OpenParenthesis, x, y); 1019 case ')': 1020 return new Token(Tokens.CloseParenthesis, x, y); 1021 case '{': 1022 return new Token(Tokens.OpenCurlyBrace, x, y); 1023 case '}': 1024 return new Token(Tokens.CloseCurlyBrace, x, y); 1025 case '?': 1026 return new Token(Tokens.QuestionMark, x, y); 1027 case '!': 1028 return new Token(Tokens.ExclamationMark, x, y); 1029 } 1030 return null; 1031 } 1032 #endregion 1033 1034 #region XML Readers 1035 Token ReadXmlProcessingInstruction(int x, int y) 1036 { 1037 sb.Length = 0; 1038 int nextChar = -1; 1039 1040 while (ReaderPeek() != '?' || ReaderPeek(1) != '>') { 1041 nextChar = ReaderRead(); 1042 if (nextChar == -1) 1043 break; 1044 sb.Append((char)nextChar); 1045 } 1046 1047 ReaderSkip("?>".Length); 1048 1049 return new Token(Tokens.XmlProcessingInstruction, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null); 1050 } 1051 1052 Token ReadXmlCommentOrCData(int x, int y) 1053 { 1054 sb.Length = 0; 1055 int nextChar = -1; 1056 1057 if (string.CompareOrdinal(ReaderPeekString("--".Length), "--") == 0) { 1058 ReaderSkip("--".Length); 1059 while ((nextChar = ReaderRead()) != -1) { 1060 sb.Append((char)nextChar); 1061 if (string.CompareOrdinal(ReaderPeekString("-->".Length), "-->") == 0) { 1062 ReaderSkip("-->".Length); 1063 return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null); 1064 } 1065 } 1066 } 1067 1068 if (string.CompareOrdinal(ReaderPeekString("[CDATA[".Length), "[CDATA[") == 0) { 1069 ReaderSkip("[CDATA[".Length); 1070 while ((nextChar = ReaderRead()) != -1) { 1071 sb.Append((char)nextChar); 1072 if (string.CompareOrdinal(ReaderPeekString("]]>".Length), "]]>") == 0) { 1073 ReaderSkip("]]>".Length); 1074 return new Token(Tokens.XmlCData, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null); 1075 } 1076 } 1077 } 1078 1079 return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null); 1080 } 1081 1082 string ReadXmlContent(char ch) 1083 { 1084 sb.Length = 0; 1085 while (true) { 1086 sb.Append(ch); 1087 int next = ReaderPeek(); 1088 1089 if (next == -1 || next == '<') 1090 break; 1091 ch = (char)ReaderRead(); 1092 } 1093 1094 return sb.ToString(); 1095 } 1096 1097 string ReadXmlString(char terminator) 1098 { 1099 char ch = '\0'; 1100 sb.Length = 0; 1101 int nextChar; 1102 while ((nextChar = ReaderRead()) != -1) { 1103 ch = (char)nextChar; 1104 if (ch == terminator) { 1105 break; 1106 } else if (ch == '\n') { 1107 errors.Error(Line, Col, String.Format("No return allowed inside String literal")); 1108 } else { 1109 sb.Append(ch); 1110 } 1111 } 1112 if (ch != terminator) { 1113 errors.Error(Line, Col, String.Format("End of File reached before String terminated ")); 1114 } 1115 return sb.ToString(); 1116 } 1117 1118 string ReadXmlIdent(char ch) 1119 { 1120 sb.Length = 0; 1121 sb.Append(ch); 1122 1123 int peek; 1124 1125 while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) { 1126 sb.Append((char)ReaderRead()); 1127 } 1128 1129 return sb.ToString(); 1130 } 1131 #endregion 1132 1133 char PeekUpperChar() 1134 { 1135 return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture); 1136 } 1137 1138 /// <summary> 1139 /// Skips to the end of the current code block. 1140 /// For this, the lexer must have read the next token AFTER the token opening the 1141 /// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead). 1142 /// After the call, Lexer.LookAhead will be the block-closing token. 1143 /// </summary> 1144 public void SkipCurrentBlock(int targetToken) 1145 { 1146 int lastKind = -1; 1147 int kind = lastToken.kind; 1148 while (kind != Tokens.EOF && 1149 !(lastKind == Tokens.End && kind == targetToken)) 1150 { 1151 lastKind = kind; 1152 NextToken(); 1153 kind = lastToken.kind; 1154 } 1155 } 1156 1157 public void SetInitialContext(SnippetType type) 1158 { 1159 ef.SetContext(type); 1160 } 1161 1162 public VBLexerMemento Export() 1163 { 1164 return new VBLexerMemento() { 1165 Column = Col, 1166 Line = Line, 1167 EncounteredLineContinuation = encounteredLineContinuation, 1168 ExpressionFinder = ef.Export(), 1169 InXmlMode = inXmlMode, 1170 IsAtLineBegin = isAtLineBegin, 1171 LineEnd = lineEnd, 1172 PrevTokenKind = prevToken.kind, 1173 MisreadExclamationMarkAsTypeCharacter = misreadExclamationMarkAsTypeCharacter, 1174 XmlModeInfoStack = new Stack<XmlModeInfo>(xmlModeStack.Select(i => (XmlModeInfo)i.Clone()).Reverse()) 1175 }; 1176 } 1177 1178 LATextReader reader; 1179 int col = 1; 1180 int line = 1; 1181 1182 protected Errors errors = new Errors(); 1183 1184 protected Token lastToken = null; 1185 protected Token curToken = null; 1186 protected Token peekToken = null; 1187 1188 string[] specialCommentTags = null; 1189 protected Hashtable specialCommentHash = null; 1190// List<TagComment> tagComments = new List<TagComment>(); 1191 protected StringBuilder sb = new StringBuilder(); 1192// protected SpecialTracker specialTracker = new SpecialTracker(); 1193 1194 // used for the original value of strings (with escape sequences). 1195 protected StringBuilder originalValue = new StringBuilder(); 1196 1197 public bool SkipAllComments { get; set; } 1198 public bool EvaluateConditionalCompilation { get; set; } 1199 public virtual IDictionary<string, object> ConditionalCompilationSymbols { 1200 get { throw new NotSupportedException(); } 1201 } 1202 1203 protected static IEnumerable<string> GetSymbols (string symbols) 1204 { 1205 if (!string.IsNullOrEmpty(symbols)) { 1206 foreach (string symbol in symbols.Split (';', ' ', '\t')) { 1207 string s = symbol.Trim (); 1208 if (s.Length == 0) 1209 continue; 1210 yield return s; 1211 } 1212 } 1213 } 1214 1215 public void SetConditionalCompilationSymbols (string symbols) 1216 { 1217 throw new NotSupportedException (); 1218 } 1219 1220 protected int Line { 1221 get { 1222 return line; 1223 } 1224 } 1225 protected int Col { 1226 get { 1227 return col; 1228 } 1229 } 1230 1231 protected bool recordRead = false; 1232 protected StringBuilder recordedText = new StringBuilder (); 1233 1234 protected int ReaderRead() 1235 { 1236 int val = reader.Read(); 1237 if (recordRead && val >= 0) 1238 recordedText.Append ((char)val); 1239 if ((val == '\r' && reader.Peek() != '\n') || val == '\n') { 1240 ++line; 1241 col = 1; 1242 LineBreak(); 1243 } else if (val >= 0) { 1244 col++; 1245 } 1246 return val; 1247 } 1248 1249 protected int ReaderPeek() 1250 { 1251 return reader.Peek(); 1252 } 1253 1254 protected int ReaderPeek(int step) 1255 { 1256 return reader.Peek(step); 1257 } 1258 1259 protected void ReaderSkip(int steps) 1260 { 1261 for (int i = 0; i < steps; i++) { 1262 ReaderRead(); 1263 } 1264 } 1265 1266 protected string ReaderPeekString(int length) 1267 { 1268 StringBuilder builder = new StringBuilder(); 1269 1270 for (int i = 0; i < length; i++) { 1271 int peek = ReaderPeek(i); 1272 if (peek != -1) 1273 builder.Append((char)peek); 1274 } 1275 1276 return builder.ToString(); 1277 } 1278 1279 public void SetInitialLocation(TextLocation location) 1280 { 1281 if (lastToken != null || curToken != null || peekToken != null) 1282 throw new InvalidOperationException(); 1283 this.line = location.Line; 1284 this.col = location.Column; 1285 } 1286 1287 public Errors Errors { 1288 get { 1289 return errors; 1290 } 1291 } 1292 1293 /// <summary> 1294 /// Returns the comments that had been read and containing tag key words. 1295 /// </summary> 1296// public List<TagComment> TagComments { 1297// get { 1298// return tagComments; 1299// } 1300// } 1301 1302// public SpecialTracker SpecialTracker { 1303// get { 1304// return specialTracker; 1305// } 1306// } 1307 1308 /// <summary> 1309 /// Special comment tags are tags like TODO, HACK or UNDONE which are read by the lexer and stored in <see cref="TagComments"/>. 1310 /// </summary> 1311 public string[] SpecialCommentTags { 1312 get { 1313 return specialCommentTags; 1314 } 1315 set { 1316 specialCommentTags = value; 1317 specialCommentHash = null; 1318 if (specialCommentTags != null && specialCommentTags.Length > 0) { 1319 specialCommentHash = new Hashtable(); 1320 foreach (string str in specialCommentTags) { 1321 specialCommentHash.Add(str, null); 1322 } 1323 } 1324 } 1325 } 1326 1327 /// <summary> 1328 /// The current Token. <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/> 1329 /// </summary> 1330 public Token Token { 1331 get { 1332// Console.WriteLine("Call to Token"); 1333 return lastToken; 1334 } 1335 } 1336 1337 /// <summary> 1338 /// The next Token (The <see cref="Token"/> after <see cref="NextToken"/> call) . <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/> 1339 /// </summary> 1340 public Token LookAhead { 1341 get { 1342// Console.WriteLine("Call to LookAhead"); 1343 return curToken; 1344 } 1345 } 1346 1347 #region System.IDisposable interface implementation 1348 public virtual void Dispose() 1349 { 1350 reader.Close(); 1351 reader = null; 1352 errors = null; 1353 lastToken = curToken = peekToken = null; 1354 specialCommentHash = null; 1355 sb = originalValue = null; 1356 } 1357 #endregion 1358 1359 /// <summary> 1360 /// Must be called before a peek operation. 1361 /// </summary> 1362 public void StartPeek() 1363 { 1364 peekToken = curToken; 1365 } 1366 1367 /// <summary> 1368 /// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on. 1369 /// </summary> 1370 /// <returns>An <see cref="Token"/> object.</returns> 1371 public Token Peek() 1372 { 1373// Console.WriteLine("Call to Peek"); 1374 if (peekToken.next == null) { 1375 peekToken.next = Next(); 1376 } 1377 peekToken = peekToken.next; 1378 return peekToken; 1379 } 1380 1381 protected static bool IsIdentifierPart(int ch) 1382 { 1383 if (ch == 95) return true; // 95 = '_' 1384 if (ch == -1) return false; 1385 return char.IsLetterOrDigit((char)ch); // accept unicode letters 1386 } 1387 1388 protected static bool IsHex(char digit) 1389 { 1390 return Char.IsDigit(digit) || ('A' <= digit && digit <= 'F') || ('a' <= digit && digit <= 'f'); 1391 } 1392 1393 protected int GetHexNumber(char digit) 1394 { 1395 if (Char.IsDigit(digit)) { 1396 return digit - '0'; 1397 } 1398 if ('A' <= digit && digit <= 'F') { 1399 return digit - 'A' + 0xA; 1400 } 1401 if ('a' <= digit && digit <= 'f') { 1402 return digit - 'a' + 0xA; 1403 } 1404 errors.Error(line, col, String.Format("Invalid hex number '" + digit + "'")); 1405 return 0; 1406 } 1407 protected TextLocation lastLineEnd = new TextLocation(1, 1); 1408 protected TextLocation curLineEnd = new TextLocation(1, 1); 1409 protected void LineBreak () 1410 { 1411 lastLineEnd = curLineEnd; 1412 curLineEnd = new TextLocation (line, col - 1); 1413 } 1414 protected bool HandleLineEnd(char ch) 1415 { 1416 // Handle MS-DOS or MacOS line ends. 1417 if (ch == '\r') { 1418 if (reader.Peek() == '\n') { // MS-DOS line end '\r\n' 1419 ReaderRead(); // LineBreak (); called by ReaderRead (); 1420 return true; 1421 } else { // assume MacOS line end which is '\r' 1422 LineBreak (); 1423 return true; 1424 } 1425 } 1426 if (ch == '\n') { 1427 LineBreak (); 1428 return true; 1429 } 1430 return false; 1431 } 1432 1433 protected void SkipToEndOfLine() 1434 { 1435 int nextChar; 1436 while ((nextChar = reader.Read()) != -1) { 1437 if (nextChar == '\r') { 1438 if (reader.Peek() == '\n') 1439 reader.Read(); 1440 nextChar = '\n'; 1441 } 1442 if (nextChar == '\n') { 1443 ++line; 1444 col = 1; 1445 break; 1446 } 1447 } 1448 } 1449 1450 protected string ReadToEndOfLine() 1451 { 1452 sb.Length = 0; 1453 int nextChar; 1454 while ((nextChar = reader.Read()) != -1) { 1455 char ch = (char)nextChar; 1456 1457 if (nextChar == '\r') { 1458 if (reader.Peek() == '\n') 1459 reader.Read(); 1460 nextChar = '\n'; 1461 } 1462 // Return read string, if EOL is reached 1463 if (nextChar == '\n') { 1464 ++line; 1465 col = 1; 1466 return sb.ToString(); 1467 } 1468 1469 sb.Append(ch); 1470 } 1471 1472 // Got EOF before EOL 1473 string retStr = sb.ToString(); 1474 col += retStr.Length; 1475 return retStr; 1476 } 1477 1478 public event EventHandler<SavepointEventArgs> SavepointReached; 1479 1480 protected virtual void OnSavepointReached(SavepointEventArgs e) 1481 { 1482 if (SavepointReached != null) { 1483 SavepointReached(this, e); 1484 } 1485 } 1486 } 1487}