/NRefactory/ICSharpCode.NRefactory.VB/Lexer/VBLexer.cs

http://github.com/icsharpcode/ILSpy · C# · 1487 lines · 1473 code · 12 blank · 2 comment · 58 complexity · aea4d647444e07fb626ece55c7c34ace MD5 · raw file

  1. // Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt)
  2. // This code is distributed under MIT X11 license (for details please see \doc\license.txt)
  3. using System;
  4. using System.Collections;
  5. using System.Collections.Generic;
  6. using System.Diagnostics;
  7. using System.Globalization;
  8. using System.IO;
  9. using System.Linq;
  10. using System.Text;
  11. using System.Xml;
  12. namespace ICSharpCode.NRefactory.VB.Parser
  13. {
  14. public class VBLexer : IDisposable
  15. {
  16. bool lineEnd = true;
  17. bool isAtLineBegin = false; // TODO: handle line begin, if neccessarry
  18. bool misreadExclamationMarkAsTypeCharacter;
  19. bool encounteredLineContinuation;
  20. ExpressionFinder ef;
  21. bool inXmlMode;
  22. Stack<XmlModeInfo> xmlModeStack = new Stack<XmlModeInfo>();
  23. public VBLexer(TextReader reader)
  24. {
  25. this.reader = new LATextReader(reader);
  26. ef = new ExpressionFinder();
  27. }
  28. public VBLexer(TextReader reader, VBLexerMemento state) : this(reader)
  29. {
  30. SetInitialLocation(new TextLocation(state.Line, state.Column));
  31. lastToken = new Token(state.PrevTokenKind, 0, 0);
  32. ef = new ExpressionFinder(state.ExpressionFinder);
  33. lineEnd = state.LineEnd;
  34. isAtLineBegin = state.IsAtLineBegin;
  35. encounteredLineContinuation = state.EncounteredLineContinuation;
  36. misreadExclamationMarkAsTypeCharacter = state.MisreadExclamationMarkAsTypeCharacter;
  37. xmlModeStack = new Stack<XmlModeInfo>(state.XmlModeInfoStack.Select(i => (XmlModeInfo)i.Clone()).Reverse());
  38. inXmlMode = state.InXmlMode;
  39. }
  40. Token NextInternal()
  41. {
  42. if (misreadExclamationMarkAsTypeCharacter) {
  43. misreadExclamationMarkAsTypeCharacter = false;
  44. return new Token(Tokens.ExclamationMark, Col - 1, Line);
  45. }
  46. unchecked {
  47. while (true) {
  48. TextLocation startLocation = new TextLocation(Line, Col);
  49. int nextChar = ReaderRead();
  50. if (nextChar == -1)
  51. return new Token(Tokens.EOF, Col, Line, string.Empty);
  52. char ch = (char)nextChar;
  53. #region XML mode
  54. CheckXMLState(startLocation);
  55. if (inXmlMode && xmlModeStack.Peek().level <= 0 && !xmlModeStack.Peek().isDocumentStart && !xmlModeStack.Peek().inXmlTag) {
  56. XmlModeInfo info = xmlModeStack.Peek();
  57. int peek = nextChar;
  58. while (true) {
  59. int step = -1;
  60. while (peek != -1 && XmlConvert.IsWhitespaceChar((char)peek)) {
  61. step++;
  62. peek = ReaderPeek(step);
  63. }
  64. if (peek == '<' && (ReaderPeek(step + 1) == '!' || ReaderPeek(step + 1) == '?')) {
  65. char lastCh = '\0';
  66. for (int i = 0; i < step + 2; i++)
  67. lastCh = (char)ReaderRead();
  68. if (lastCh == '!')
  69. return ReadXmlCommentOrCData(Col - 2, Line);
  70. else
  71. return ReadXmlProcessingInstruction(Col - 2, Line);
  72. }
  73. break;
  74. }
  75. inXmlMode = false;
  76. xmlModeStack.Pop();
  77. }
  78. if (inXmlMode) {
  79. XmlModeInfo info = xmlModeStack.Peek();
  80. int x = Col - 1;
  81. int y = Line;
  82. switch (ch) {
  83. case '<':
  84. if (ReaderPeek() == '/') {
  85. ReaderRead();
  86. info.inXmlCloseTag = true;
  87. return new Token(Tokens.XmlOpenEndTag, new TextLocation(y, x), new TextLocation(Line, Col));
  88. }
  89. if (ReaderPeek() == '%' && ReaderPeek(1) == '=') {
  90. inXmlMode = false;
  91. ReaderRead(); ReaderRead();
  92. return new Token(Tokens.XmlStartInlineVB, new TextLocation(y, x), new TextLocation(Line, Col));
  93. }
  94. if (ReaderPeek() == '?') {
  95. ReaderRead();
  96. Token t = ReadXmlProcessingInstruction(x, y);
  97. return t;
  98. }
  99. if (ReaderPeek() == '!') {
  100. ReaderRead();
  101. Token token = ReadXmlCommentOrCData(x, y);
  102. return token;
  103. }
  104. info.level++;
  105. info.isDocumentStart = false;
  106. info.inXmlTag = true;
  107. return new Token(Tokens.XmlOpenTag, x, y);
  108. case '/':
  109. if (ReaderPeek() == '>') {
  110. ReaderRead();
  111. info.inXmlTag = false;
  112. info.level--;
  113. return new Token(Tokens.XmlCloseTagEmptyElement, new TextLocation(y, x), new TextLocation(Line, Col));
  114. }
  115. break;
  116. case '>':
  117. if (info.inXmlCloseTag)
  118. info.level--;
  119. info.inXmlTag = info.inXmlCloseTag = false;
  120. return new Token(Tokens.XmlCloseTag, x, y);
  121. case '=':
  122. return new Token(Tokens.Assign, x, y);
  123. case '\'':
  124. case '"':
  125. string s = ReadXmlString(ch);
  126. return new Token(Tokens.LiteralString, x, y, ch + s + ch, s);
  127. default:
  128. if (info.inXmlCloseTag || info.inXmlTag) {
  129. if (XmlConvert.IsWhitespaceChar(ch))
  130. continue;
  131. return new Token(Tokens.Identifier, x, y, ReadXmlIdent(ch));
  132. } else {
  133. string content = ReadXmlContent(ch);
  134. return new Token(Tokens.XmlContent, startLocation, new TextLocation(Line, Col), content, null);
  135. }
  136. }
  137. #endregion
  138. } else {
  139. #region Standard Mode
  140. if (Char.IsWhiteSpace(ch)) {
  141. if (HandleLineEnd(ch)) {
  142. if (lineEnd) {
  143. // second line end before getting to a token
  144. // -> here was a blank line
  145. // specialTracker.AddEndOfLine(startLocation);
  146. } else {
  147. lineEnd = true;
  148. return new Token(Tokens.EOL, startLocation, new TextLocation(Line, Col), null, null);
  149. }
  150. }
  151. continue;
  152. }
  153. if (ch == '_') {
  154. if (ReaderPeek() == -1) {
  155. errors.Error(Line, Col, String.Format("No EOF expected after _"));
  156. return new Token(Tokens.EOF, Col, Line, string.Empty);
  157. }
  158. if (!Char.IsWhiteSpace((char)ReaderPeek())) {
  159. int x = Col - 1;
  160. int y = Line;
  161. string s = ReadIdent('_');
  162. lineEnd = false;
  163. return new Token(Tokens.Identifier, x, y, s);
  164. }
  165. encounteredLineContinuation = true;
  166. ch = (char)ReaderRead();
  167. bool oldLineEnd = lineEnd;
  168. lineEnd = false;
  169. while (Char.IsWhiteSpace(ch)) {
  170. if (HandleLineEnd(ch)) {
  171. lineEnd = true;
  172. break;
  173. }
  174. if (ReaderPeek() != -1) {
  175. ch = (char)ReaderRead();
  176. } else {
  177. errors.Error(Line, Col, String.Format("No EOF expected after _"));
  178. return new Token(Tokens.EOF, Col, Line, string.Empty);
  179. }
  180. }
  181. if (!lineEnd) {
  182. errors.Error(Line, Col, String.Format("NewLine expected"));
  183. }
  184. lineEnd = oldLineEnd;
  185. continue;
  186. }
  187. if (ch == '#') {
  188. while (Char.IsWhiteSpace((char)ReaderPeek())) {
  189. ReaderRead();
  190. }
  191. if (Char.IsDigit((char)ReaderPeek())) {
  192. int x = Col - 1;
  193. int y = Line;
  194. string s = ReadDate();
  195. DateTime time = new DateTime(1, 1, 1, 0, 0, 0);
  196. try {
  197. time = DateTime.Parse(s, System.Globalization.CultureInfo.InvariantCulture, DateTimeStyles.NoCurrentDateDefault);
  198. } catch (Exception e) {
  199. errors.Error(Line, Col, String.Format("Invalid date time {0}", e));
  200. }
  201. return new Token(Tokens.LiteralDate, x, y, s, time);
  202. } else {
  203. ReadPreprocessorDirective();
  204. continue;
  205. }
  206. }
  207. if (ch == '[') { // Identifier
  208. lineEnd = false;
  209. if (ReaderPeek() == -1) {
  210. errors.Error(Line, Col, String.Format("Identifier expected"));
  211. }
  212. ch = (char)ReaderRead();
  213. if (ch == ']' || Char.IsWhiteSpace(ch)) {
  214. errors.Error(Line, Col, String.Format("Identifier expected"));
  215. }
  216. int x = Col - 1;
  217. int y = Line;
  218. string s = ReadIdent(ch);
  219. if (ReaderPeek() == -1) {
  220. errors.Error(Line, Col, String.Format("']' expected"));
  221. }
  222. ch = (char)ReaderRead();
  223. if (!(ch == ']')) {
  224. errors.Error(Line, Col, String.Format("']' expected"));
  225. }
  226. return new Token(Tokens.Identifier, x, y, s);
  227. }
  228. if (Char.IsLetter(ch)) {
  229. int x = Col - 1;
  230. int y = Line;
  231. char typeCharacter;
  232. string s = ReadIdent(ch, out typeCharacter);
  233. if (typeCharacter == '\0') {
  234. int keyWordToken = Keywords.GetToken(s);
  235. if (keyWordToken >= 0) {
  236. // handle 'REM' comments
  237. if (keyWordToken == Tokens.Rem) {
  238. ReadComment();
  239. if (!lineEnd) {
  240. lineEnd = true;
  241. return new Token(Tokens.EOL, Col, Line, "\n");
  242. }
  243. continue;
  244. }
  245. lineEnd = false;
  246. return new Token(keyWordToken, x, y, s);
  247. }
  248. }
  249. lineEnd = false;
  250. return new Token(Tokens.Identifier, x, y, s);
  251. }
  252. if (Char.IsDigit(ch)) {
  253. lineEnd = false;
  254. return ReadDigit(ch, Col - 1);
  255. }
  256. if (ch == '&') {
  257. lineEnd = false;
  258. if (ReaderPeek() == -1) {
  259. return ReadOperator('&');
  260. }
  261. ch = (char)ReaderPeek();
  262. if (Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'H' || Char.ToUpper(ch, CultureInfo.InvariantCulture) == 'O') {
  263. return ReadDigit('&', Col - 1);
  264. }
  265. return ReadOperator('&');
  266. }
  267. if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {
  268. int x = Col - 1;
  269. int y = Line;
  270. ReadComment();
  271. if (!lineEnd) {
  272. lineEnd = true;
  273. return new Token(Tokens.EOL, x, y, "\n");
  274. }
  275. continue;
  276. }
  277. if (ch == '"') {
  278. lineEnd = false;
  279. int x = Col - 1;
  280. int y = Line;
  281. string s = ReadString();
  282. if (ReaderPeek() != -1 && (ReaderPeek() == 'C' || ReaderPeek() == 'c')) {
  283. ReaderRead();
  284. if (s.Length != 1) {
  285. errors.Error(Line, Col, String.Format("Chars can only have Length 1 "));
  286. }
  287. if (s.Length == 0) {
  288. s = "\0";
  289. }
  290. return new Token(Tokens.LiteralCharacter, x, y, '"' + s + "\"C", s[0]);
  291. }
  292. return new Token(Tokens.LiteralString, x, y, '"' + s + '"', s);
  293. }
  294. if (ch == '%' && ReaderPeek() == '>') {
  295. int x = Col - 1;
  296. int y = Line;
  297. inXmlMode = true;
  298. ReaderRead();
  299. return new Token(Tokens.XmlEndInlineVB, new TextLocation(y, x), new TextLocation(Line, Col));
  300. }
  301. #endregion
  302. if (ch == '<' && (ef.NextTokenIsPotentialStartOfExpression || ef.NextTokenIsStartOfImportsOrAccessExpression)) {
  303. xmlModeStack.Push(new XmlModeInfo(ef.NextTokenIsStartOfImportsOrAccessExpression));
  304. XmlModeInfo info = xmlModeStack.Peek();
  305. int x = Col - 1;
  306. int y = Line;
  307. inXmlMode = true;
  308. if (ReaderPeek() == '/') {
  309. ReaderRead();
  310. info.inXmlCloseTag = true;
  311. return new Token(Tokens.XmlOpenEndTag, new TextLocation(y, x), new TextLocation(Line, Col));
  312. }
  313. // should we allow <%= at start of an expression? not valid with vbc ...
  314. if (ReaderPeek() == '%' && ReaderPeek(1) == '=') {
  315. inXmlMode = false;
  316. ReaderRead(); ReaderRead();
  317. return new Token(Tokens.XmlStartInlineVB, new TextLocation(y, x), new TextLocation(Line, Col));
  318. }
  319. if (ReaderPeek() == '!') {
  320. ReaderRead();
  321. Token t = ReadXmlCommentOrCData(x, y);
  322. return t;
  323. }
  324. if (ReaderPeek() == '?') {
  325. ReaderRead();
  326. Token t = ReadXmlProcessingInstruction(x, y);
  327. info.isDocumentStart = t.val.Trim().StartsWith("xml", StringComparison.OrdinalIgnoreCase);
  328. return t;
  329. }
  330. info.inXmlTag = true;
  331. info.level++;
  332. return new Token(Tokens.XmlOpenTag, x, y);
  333. }
  334. Token token = ReadOperator(ch);
  335. if (token != null) {
  336. lineEnd = false;
  337. return token;
  338. }
  339. }
  340. errors.Error(Line, Col, String.Format("Unknown char({0}) which can't be read", ch));
  341. }
  342. }
  343. }
  344. void CheckXMLState(TextLocation startLocation)
  345. {
  346. if (inXmlMode && !xmlModeStack.Any())
  347. throw new InvalidOperationException("invalid XML stack state at " + startLocation);
  348. }
  349. Token prevToken;
  350. Token Next()
  351. {
  352. Token t = NextInternal();
  353. if (t.kind == Tokens.EOL) {
  354. Debug.Assert(t.next == null); // NextInternal() must return only 1 token
  355. t.next = NextInternal();
  356. Debug.Assert(t.next.next == null);
  357. if (SkipEOL(prevToken.kind, t.next.kind)) {
  358. t = t.next;
  359. }
  360. } else
  361. encounteredLineContinuation = false;
  362. // inform EF only once we're sure it's really a token
  363. // this means we inform it about EOL tokens "1 token too late", but that's not a problem because
  364. // XML literals cannot start immediately after an EOL token
  365. ef.InformToken(t);
  366. if (t.next != null) {
  367. // Next() isn't called again when it returns 2 tokens, so we need to process both tokens
  368. ef.InformToken(t.next);
  369. prevToken = t.next;
  370. } else {
  371. prevToken = t;
  372. }
  373. ef.Advance();
  374. Debug.Assert(t != null);
  375. return t;
  376. }
  377. /// <remarks>see VB language specification 10; pg. 6</remarks>
  378. bool SkipEOL(int prevTokenKind, int nextTokenKind)
  379. {
  380. // exception directly after _
  381. if (encounteredLineContinuation) {
  382. return encounteredLineContinuation = false;
  383. }
  384. // 1st rule
  385. // after a comma (,), open parenthesis ((), open curly brace ({), or open embedded expression (<%=)
  386. if (new[] { Tokens.Comma, Tokens.OpenParenthesis, Tokens.OpenCurlyBrace, Tokens.XmlStartInlineVB }
  387. .Contains(prevTokenKind))
  388. return true;
  389. // 2nd rule
  390. // after a member qualifier (. or .@ or ...), provided that something is being qualified (i.e. is not
  391. // using an implicit With context)
  392. if (new[] { Tokens.Dot, Tokens.DotAt, Tokens.TripleDot }.Contains(prevTokenKind)
  393. && !ef.WasQualifierTokenAtStart)
  394. return true;
  395. // 3rd rule
  396. // before a close parenthesis ()), close curly brace (}), or close embedded expression (%>)
  397. if (new[] { Tokens.CloseParenthesis, Tokens.CloseCurlyBrace, Tokens.XmlEndInlineVB }
  398. .Contains(nextTokenKind))
  399. return true;
  400. // 4th rule
  401. // after a less-than (<) in an attribute context
  402. if (prevTokenKind == Tokens.LessThan && ef.InContext(Context.Attribute))
  403. return true;
  404. // 5th rule
  405. // before a greater-than (>) in an attribute context
  406. if (nextTokenKind == Tokens.GreaterThan && ef.InContext(Context.Attribute))
  407. return true;
  408. // 6th rule
  409. // after a greater-than (>) in a non-file-level attribute context
  410. if (ef.WasNormalAttribute && prevTokenKind == Tokens.GreaterThan)
  411. return true;
  412. // 7th rule
  413. // before and after query operators (Where, Order, Select, etc.)
  414. var queryOperators = new int[] { Tokens.From, Tokens.Aggregate, Tokens.Select, Tokens.Distinct,
  415. Tokens.Where, Tokens.Order, Tokens.By, Tokens.Ascending, Tokens.Descending, Tokens.Take,
  416. Tokens.Skip, Tokens.Let, Tokens.Group, Tokens.Into, Tokens.On, Tokens.While, Tokens.Join };
  417. if (ef.InContext(Context.Query)) {
  418. // Ascending, Descending, Distinct are special
  419. // fixes http://community.sharpdevelop.net/forums/p/12068/32893.aspx#32893
  420. var specialQueryOperators = new int[] { Tokens.Ascending, Tokens.Descending, Tokens.Distinct };
  421. if (specialQueryOperators.Contains(prevTokenKind) && !queryOperators.Contains(nextTokenKind))
  422. return false;
  423. if ((queryOperators.Contains(prevTokenKind) || queryOperators.Contains(nextTokenKind)))
  424. return true;
  425. }
  426. // 8th rule
  427. // after binary operators (+, -, /, *, etc.) in an expression context
  428. if (new[] { Tokens.Plus, Tokens.Minus, Tokens.Div, Tokens.DivInteger, Tokens.Times, Tokens.Mod, Tokens.Power,
  429. Tokens.Assign, Tokens.NotEqual, Tokens.LessThan, Tokens.LessEqual, Tokens.GreaterThan, Tokens.GreaterEqual,
  430. Tokens.Like, Tokens.ConcatString, Tokens.AndAlso, Tokens.OrElse, Tokens.And, Tokens.Or, Tokens.Xor,
  431. Tokens.ShiftLeft, Tokens.ShiftRight }.Contains(prevTokenKind) && ef.CurrentBlock.context == Context.Expression)
  432. return true;
  433. // 9th rule
  434. // after assignment operators (=, :=, +=, -=, etc.) in any context.
  435. if (new[] { Tokens.Assign, Tokens.ColonAssign, Tokens.ConcatStringAssign, Tokens.DivAssign,
  436. Tokens.DivIntegerAssign, Tokens.MinusAssign, Tokens.PlusAssign, Tokens.PowerAssign,
  437. Tokens.ShiftLeftAssign, Tokens.ShiftRightAssign, Tokens.TimesAssign }.Contains(prevTokenKind))
  438. return true;
  439. return false;
  440. }
  441. /// <summary>
  442. /// Reads the next token.
  443. /// </summary>
  444. /// <returns>A <see cref="Token"/> object.</returns>
  445. public Token NextToken()
  446. {
  447. if (curToken == null) { // first call of NextToken()
  448. curToken = Next();
  449. //Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
  450. return curToken;
  451. }
  452. lastToken = curToken;
  453. if (curToken.next == null) {
  454. curToken.next = Next();
  455. }
  456. curToken = curToken.next;
  457. if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
  458. curToken = new Token(Tokens.EOL, curToken.col, curToken.line, string.Empty);
  459. curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, string.Empty);
  460. }
  461. //Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
  462. return curToken;
  463. }
  464. #region VB Readers
  465. string ReadIdent(char ch)
  466. {
  467. char typeCharacter;
  468. return ReadIdent(ch, out typeCharacter);
  469. }
  470. string ReadIdent(char ch, out char typeCharacter)
  471. {
  472. typeCharacter = '\0';
  473. if (ef.ReadXmlIdentifier) {
  474. ef.ReadXmlIdentifier = false;
  475. return ReadXmlIdent(ch);
  476. }
  477. sb.Length = 0;
  478. sb.Append(ch);
  479. int peek;
  480. while ((peek = ReaderPeek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) {
  481. ReaderRead();
  482. sb.Append(ch.ToString());
  483. }
  484. if (peek == -1) {
  485. return sb.ToString();
  486. }
  487. if ("%&@!#$".IndexOf((char)peek) != -1) {
  488. typeCharacter = (char)peek;
  489. ReaderRead();
  490. if (typeCharacter == '!') {
  491. peek = ReaderPeek();
  492. if (peek != -1 && (peek == '_' || peek == '[' || char.IsLetter((char)peek))) {
  493. misreadExclamationMarkAsTypeCharacter = true;
  494. }
  495. }
  496. }
  497. return sb.ToString();
  498. }
  499. [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1818:DoNotConcatenateStringsInsideLoops")]
  500. Token ReadDigit(char ch, int x)
  501. {
  502. sb.Length = 0;
  503. sb.Append(ch);
  504. int y = Line;
  505. string digit = "";
  506. if (ch != '&') {
  507. digit += ch;
  508. }
  509. bool isHex = false;
  510. bool isOct = false;
  511. bool isSingle = false;
  512. bool isDouble = false;
  513. bool isDecimal = false;
  514. if (ReaderPeek() == -1) {
  515. if (ch == '&') {
  516. errors.Error(Line, Col, String.Format("digit expected"));
  517. }
  518. return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0');
  519. }
  520. if (ch == '.') {
  521. if (Char.IsDigit((char)ReaderPeek())) {
  522. isDouble = true; // double is default
  523. if (isHex || isOct) {
  524. errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
  525. }
  526. while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot
  527. digit += (char)ReaderRead();
  528. }
  529. }
  530. } else if (ch == '&' && PeekUpperChar() == 'H') {
  531. const string hex = "0123456789ABCDEF";
  532. sb.Append((char)ReaderRead()); // skip 'H'
  533. while (ReaderPeek() != -1 && hex.IndexOf(PeekUpperChar()) != -1) {
  534. ch = (char)ReaderRead();
  535. sb.Append(ch);
  536. digit += Char.ToUpper(ch, CultureInfo.InvariantCulture);
  537. }
  538. isHex = true;
  539. } else if (ReaderPeek() != -1 && ch == '&' && PeekUpperChar() == 'O') {
  540. const string okt = "01234567";
  541. sb.Append((char)ReaderRead()); // skip 'O'
  542. while (ReaderPeek() != -1 && okt.IndexOf(PeekUpperChar()) != -1) {
  543. ch = (char)ReaderRead();
  544. sb.Append(ch);
  545. digit += Char.ToUpper(ch, CultureInfo.InvariantCulture);
  546. }
  547. isOct = true;
  548. } else {
  549. while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) {
  550. ch = (char)ReaderRead();;
  551. digit += ch;
  552. sb.Append(ch);
  553. }
  554. }
  555. if (digit.Length == 0) {
  556. errors.Error(Line, Col, String.Format("digit expected"));
  557. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
  558. }
  559. if (ReaderPeek() != -1 && "%&SILU".IndexOf(PeekUpperChar()) != -1 || isHex || isOct) {
  560. bool unsigned = false;
  561. if (ReaderPeek() != -1) {
  562. ch = (char)ReaderPeek();
  563. sb.Append(ch);
  564. ch = Char.ToUpper(ch, CultureInfo.InvariantCulture);
  565. unsigned = ch == 'U';
  566. if (unsigned) {
  567. ReaderRead(); // read the U
  568. ch = (char)ReaderPeek();
  569. sb.Append(ch);
  570. ch = Char.ToUpper(ch, CultureInfo.InvariantCulture);
  571. if (ch != 'I' && ch != 'L' && ch != 'S') {
  572. errors.Error(Line, Col, "Invalid type character: U" + ch);
  573. }
  574. }
  575. }
  576. try {
  577. if (isOct) {
  578. ReaderRead();
  579. ulong number = 0L;
  580. for (int i = 0; i < digit.Length; ++i) {
  581. number = number * 8 + digit[i] - '0';
  582. }
  583. if (ch == 'S') {
  584. if (unsigned)
  585. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ushort)number);
  586. else
  587. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (short)number);
  588. } else if (ch == '%' || ch == 'I') {
  589. if (unsigned)
  590. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (uint)number);
  591. else
  592. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (int)number);
  593. } else if (ch == '&' || ch == 'L') {
  594. if (unsigned)
  595. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (ulong)number);
  596. else
  597. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), (long)number);
  598. } else {
  599. if (number > uint.MaxValue) {
  600. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number));
  601. } else {
  602. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number));
  603. }
  604. }
  605. }
  606. if (ch == 'S') {
  607. ReaderRead();
  608. if (unsigned)
  609. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt16.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  610. else
  611. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int16.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  612. } else if (ch == '%' || ch == 'I') {
  613. ReaderRead();
  614. if (unsigned)
  615. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  616. else
  617. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  618. } else if (ch == '&' || ch == 'L') {
  619. ReaderRead();
  620. if (unsigned)
  621. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), UInt64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  622. else
  623. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  624. } else if (isHex) {
  625. ulong number = UInt64.Parse(digit, NumberStyles.HexNumber);
  626. if (number > uint.MaxValue) {
  627. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((long)number));
  628. } else {
  629. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), unchecked((int)number));
  630. }
  631. }
  632. } catch (OverflowException ex) {
  633. errors.Error(Line, Col, ex.Message);
  634. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
  635. } catch (FormatException) {
  636. errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
  637. return new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
  638. }
  639. }
  640. Token nextToken = null; // if we accidently read a 'dot'
  641. if (!isDouble && ReaderPeek() == '.') { // read floating point number
  642. ReaderRead();
  643. if (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) {
  644. isDouble = true; // double is default
  645. if (isHex || isOct) {
  646. errors.Error(Line, Col, String.Format("No hexadecimal or oktadecimal floating point values allowed"));
  647. }
  648. digit += '.';
  649. while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot
  650. digit += (char)ReaderRead();
  651. }
  652. } else {
  653. nextToken = new Token(Tokens.Dot, Col - 1, Line);
  654. }
  655. }
  656. if (ReaderPeek() != -1 && PeekUpperChar() == 'E') { // read exponent
  657. isDouble = true;
  658. digit += (char)ReaderRead();
  659. if (ReaderPeek() != -1 && (ReaderPeek() == '-' || ReaderPeek() == '+')) {
  660. digit += (char)ReaderRead();
  661. }
  662. while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { // read exponent value
  663. digit += (char)ReaderRead();
  664. }
  665. }
  666. if (ReaderPeek() != -1) {
  667. switch (PeekUpperChar()) {
  668. case 'R':
  669. case '#':
  670. ReaderRead();
  671. isDouble = true;
  672. break;
  673. case 'D':
  674. case '@':
  675. ReaderRead();
  676. isDecimal = true;
  677. break;
  678. case 'F':
  679. case '!':
  680. ReaderRead();
  681. isSingle = true;
  682. break;
  683. }
  684. }
  685. try {
  686. if (isSingle) {
  687. return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), Single.Parse(digit, CultureInfo.InvariantCulture));
  688. }
  689. if (isDecimal) {
  690. return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), Decimal.Parse(digit, NumberStyles.Currency | NumberStyles.AllowExponent, CultureInfo.InvariantCulture));
  691. }
  692. if (isDouble) {
  693. return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), Double.Parse(digit, CultureInfo.InvariantCulture));
  694. }
  695. } catch (FormatException) {
  696. errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
  697. if (isSingle)
  698. return new Token(Tokens.LiteralSingle, x, y, sb.ToString(), 0f);
  699. if (isDecimal)
  700. return new Token(Tokens.LiteralDecimal, x, y, sb.ToString(), 0m);
  701. if (isDouble)
  702. return new Token(Tokens.LiteralDouble, x, y, sb.ToString(), 0.0);
  703. }
  704. Token token;
  705. try {
  706. token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int32.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  707. } catch (Exception) {
  708. try {
  709. token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), Int64.Parse(digit, isHex ? NumberStyles.HexNumber : NumberStyles.Number));
  710. } catch (FormatException) {
  711. errors.Error(Line, Col, String.Format("{0} is not a parseable number", digit));
  712. // fallback, when nothing helps :)
  713. token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
  714. } catch (OverflowException) {
  715. errors.Error(Line, Col, String.Format("{0} is too long for a integer literal", digit));
  716. // fallback, when nothing helps :)
  717. token = new Token(Tokens.LiteralInteger, x, y, sb.ToString(), 0);
  718. }
  719. }
  720. token.next = nextToken;
  721. return token;
  722. }
  723. void ReadPreprocessorDirective()
  724. {
  725. TextLocation start = new TextLocation(Line, Col - 1);
  726. string directive = ReadIdent('#');
  727. // TODO : expression parser for PP directives
  728. // needed for proper conversion to e. g. C#
  729. string argument = ReadToEndOfLine();
  730. // this.specialTracker.AddPreprocessingDirective(new PreprocessingDirective(directive, argument.Trim(), start, new AstLocation(start.Line, start.Column + directive.Length + argument.Length)));
  731. }
  732. string ReadDate()
  733. {
  734. char ch = '\0';
  735. sb.Length = 0;
  736. int nextChar;
  737. while ((nextChar = ReaderRead()) != -1) {
  738. ch = (char)nextChar;
  739. if (ch == '#') {
  740. break;
  741. } else if (ch == '\n') {
  742. errors.Error(Line, Col, String.Format("No return allowed inside Date literal"));
  743. } else {
  744. sb.Append(ch);
  745. }
  746. }
  747. if (ch != '#') {
  748. errors.Error(Line, Col, String.Format("End of File reached before Date literal terminated"));
  749. }
  750. return sb.ToString();
  751. }
  752. string ReadString()
  753. {
  754. char ch = '\0';
  755. sb.Length = 0;
  756. int nextChar;
  757. while ((nextChar = ReaderRead()) != -1) {
  758. ch = (char)nextChar;
  759. if (ch == '"') {
  760. if (ReaderPeek() != -1 && ReaderPeek() == '"') {
  761. sb.Append('"');
  762. ReaderRead();
  763. } else {
  764. break;
  765. }
  766. } else if (ch == '\n') {
  767. errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
  768. } else {
  769. sb.Append(ch);
  770. }
  771. }
  772. if (ch != '"') {
  773. errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
  774. }
  775. return sb.ToString();
  776. }
  777. void ReadComment()
  778. {
  779. TextLocation startPos = new TextLocation(Line, Col);
  780. sb.Length = 0;
  781. StringBuilder curWord = specialCommentHash != null ? new StringBuilder() : null;
  782. int missingApostrophes = 2; // no. of ' missing until it is a documentation comment
  783. int nextChar;
  784. while ((nextChar = ReaderRead()) != -1) {
  785. char ch = (char)nextChar;
  786. if (HandleLineEnd(ch)) {
  787. break;
  788. }
  789. sb.Append(ch);
  790. if (missingApostrophes > 0) {
  791. if (ch == '\'' || ch == '\u2018' || ch == '\u2019') {
  792. if (--missingApostrophes == 0) {
  793. // specialTracker.StartComment(CommentType.Documentation, isAtLineBegin, startPos);
  794. sb.Length = 0;
  795. }
  796. } else {
  797. // specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos);
  798. missingApostrophes = 0;
  799. }
  800. }
  801. if (specialCommentHash != null) {
  802. if (Char.IsLetter(ch)) {
  803. curWord.Append(ch);
  804. } else {
  805. string tag = curWord.ToString();
  806. curWord.Length = 0;
  807. if (specialCommentHash.ContainsKey(tag)) {
  808. TextLocation p = new TextLocation(Line, Col);
  809. string comment = ch + ReadToEndOfLine();
  810. // this.TagComments.Add(new TagComment(tag, comment, isAtLineBegin, p, new Location(Col, Line)));
  811. sb.Append(comment);
  812. break;
  813. }
  814. }
  815. }
  816. }
  817. // if (missingApostrophes > 0) {
  818. // specialTracker.StartComment(CommentType.SingleLine, isAtLineBegin, startPos);
  819. // }
  820. // specialTracker.AddString(sb.ToString());
  821. // specialTracker.FinishComment(new Location(Col, Line));
  822. }
  823. Token ReadOperator(char ch)
  824. {
  825. int x = Col - 1;
  826. int y = Line;
  827. switch(ch) {
  828. case '+':
  829. switch (ReaderPeek()) {
  830. case '=':
  831. ReaderRead();
  832. return new Token(Tokens.PlusAssign, x, y);
  833. default:
  834. break;
  835. }
  836. return new Token(Tokens.Plus, x, y);
  837. case '-':
  838. switch (ReaderPeek()) {
  839. case '=':
  840. ReaderRead();
  841. return new Token(Tokens.MinusAssign, x, y);
  842. default:
  843. break;
  844. }
  845. return new Token(Tokens.Minus, x, y);
  846. case '*':
  847. switch (ReaderPeek()) {
  848. case '=':
  849. ReaderRead();
  850. return new Token(Tokens.TimesAssign, x, y);
  851. default:
  852. break;
  853. }
  854. return new Token(Tokens.Times, x, y, "*");
  855. case '/':
  856. switch (ReaderPeek()) {
  857. case '=':
  858. ReaderRead();
  859. return new Token(Tokens.DivAssign, x, y);
  860. default:
  861. break;
  862. }
  863. return new Token(Tokens.Div, x, y);
  864. case '\\':
  865. switch (ReaderPeek()) {
  866. case '=':
  867. ReaderRead();
  868. return new Token(Tokens.DivIntegerAssign, x, y);
  869. default:
  870. break;
  871. }
  872. return new Token(Tokens.DivInteger, x, y);
  873. case '&':
  874. switch (ReaderPeek()) {
  875. case '=':
  876. ReaderRead();
  877. return new Token(Tokens.ConcatStringAssign, x, y);
  878. default:
  879. break;
  880. }
  881. return new Token(Tokens.ConcatString, x, y);
  882. case '^':
  883. switch (ReaderPeek()) {
  884. case '=':
  885. ReaderRead();
  886. return new Token(Tokens.PowerAssign, x, y);
  887. default:
  888. break;
  889. }
  890. return new Token(Tokens.Power, x, y);
  891. case ':':
  892. if (ReaderPeek() == '=') {
  893. ReaderRead();
  894. return new Token(Tokens.ColonAssign, x, y);
  895. }
  896. return new Token(Tokens.Colon, x, y);
  897. case '=':
  898. return new Token(Tokens.Assign, x, y);
  899. case '<':
  900. switch (ReaderPeek()) {
  901. case '=':
  902. ReaderRead();
  903. return new Token(Tokens.LessEqual, x, y);
  904. case '>':
  905. ReaderRead();
  906. return new Token(Tokens.NotEqual, x, y);
  907. case '<':
  908. ReaderRead();
  909. switch (ReaderPeek()) {
  910. case '=':
  911. ReaderRead();
  912. return new Token(Tokens.ShiftLeftAssign, x, y);
  913. default:
  914. break;
  915. }
  916. return new Token(Tokens.ShiftLeft, x, y);
  917. }
  918. return new Token(Tokens.LessThan, x, y);
  919. case '>':
  920. switch (ReaderPeek()) {
  921. case '=':
  922. ReaderRead();
  923. return new Token(Tokens.GreaterEqual, x, y);
  924. case '>':
  925. ReaderRead();
  926. if (ReaderPeek() != -1) {
  927. switch (ReaderPeek()) {
  928. case '=':
  929. ReaderRead();
  930. return new Token(Tokens.ShiftRightAssign, x, y);
  931. default:
  932. break;
  933. }
  934. }
  935. return new Token(Tokens.ShiftRight, x, y);
  936. }
  937. return new Token(Tokens.GreaterThan, x, y);
  938. case ',':
  939. return new Token(Tokens.Comma, x, y);
  940. case '.':
  941. // Prevent OverflowException when Peek returns -1
  942. int tmp = ReaderPeek(); int tmp2 = ReaderPeek(1);
  943. if (tmp > 0) {
  944. if (char.IsDigit((char)tmp))
  945. return ReadDigit('.', Col);
  946. else if ((char)tmp == '@') {
  947. ReaderRead();
  948. return new Token(Tokens.DotAt, x, y);
  949. } else if ((char)tmp == '.' && tmp2 > 0 && (char)tmp2 == '.') {
  950. ReaderRead(); ReaderRead();
  951. return new Token(Tokens.TripleDot, x, y);
  952. }
  953. }
  954. return new Token(Tokens.Dot, x, y);
  955. case '(':
  956. return new Token(Tokens.OpenParenthesis, x, y);
  957. case ')':
  958. return new Token(Tokens.CloseParenthesis, x, y);
  959. case '{':
  960. return new Token(Tokens.OpenCurlyBrace, x, y);
  961. case '}':
  962. return new Token(Tokens.CloseCurlyBrace, x, y);
  963. case '?':
  964. return new Token(Tokens.QuestionMark, x, y);
  965. case '!':
  966. return new Token(Tokens.ExclamationMark, x, y);
  967. }
  968. return null;
  969. }
  970. #endregion
  971. #region XML Readers
  972. Token ReadXmlProcessingInstruction(int x, int y)
  973. {
  974. sb.Length = 0;
  975. int nextChar = -1;
  976. while (ReaderPeek() != '?' || ReaderPeek(1) != '>') {
  977. nextChar = ReaderRead();
  978. if (nextChar == -1)
  979. break;
  980. sb.Append((char)nextChar);
  981. }
  982. ReaderSkip("?>".Length);
  983. return new Token(Tokens.XmlProcessingInstruction, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
  984. }
  985. Token ReadXmlCommentOrCData(int x, int y)
  986. {
  987. sb.Length = 0;
  988. int nextChar = -1;
  989. if (string.CompareOrdinal(ReaderPeekString("--".Length), "--") == 0) {
  990. ReaderSkip("--".Length);
  991. while ((nextChar = ReaderRead()) != -1) {
  992. sb.Append((char)nextChar);
  993. if (string.CompareOrdinal(ReaderPeekString("-->".Length), "-->") == 0) {
  994. ReaderSkip("-->".Length);
  995. return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
  996. }
  997. }
  998. }
  999. if (string.CompareOrdinal(ReaderPeekString("[CDATA[".Length), "[CDATA[") == 0) {
  1000. ReaderSkip("[CDATA[".Length);
  1001. while ((nextChar = ReaderRead()) != -1) {
  1002. sb.Append((char)nextChar);
  1003. if (string.CompareOrdinal(ReaderPeekString("]]>".Length), "]]>") == 0) {
  1004. ReaderSkip("]]>".Length);
  1005. return new Token(Tokens.XmlCData, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
  1006. }
  1007. }
  1008. }
  1009. return new Token(Tokens.XmlComment, new TextLocation(y, x), new TextLocation(Line, Col), sb.ToString(), null);
  1010. }
  1011. string ReadXmlContent(char ch)
  1012. {
  1013. sb.Length = 0;
  1014. while (true) {
  1015. sb.Append(ch);
  1016. int next = ReaderPeek();
  1017. if (next == -1 || next == '<')
  1018. break;
  1019. ch = (char)ReaderRead();
  1020. }
  1021. return sb.ToString();
  1022. }
  1023. string ReadXmlString(char terminator)
  1024. {
  1025. char ch = '\0';
  1026. sb.Length = 0;
  1027. int nextChar;
  1028. while ((nextChar = ReaderRead()) != -1) {
  1029. ch = (char)nextChar;
  1030. if (ch == terminator) {
  1031. break;
  1032. } else if (ch == '\n') {
  1033. errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
  1034. } else {
  1035. sb.Append(ch);
  1036. }
  1037. }
  1038. if (ch != terminator) {
  1039. errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
  1040. }
  1041. return sb.ToString();
  1042. }
  1043. string ReadXmlIdent(char ch)
  1044. {
  1045. sb.Length = 0;
  1046. sb.Append(ch);
  1047. int peek;
  1048. while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) {
  1049. sb.Append((char)ReaderRead());
  1050. }
  1051. return sb.ToString();
  1052. }
  1053. #endregion
  1054. char PeekUpperChar()
  1055. {
  1056. return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture);
  1057. }
  1058. /// <summary>
  1059. /// Skips to the end of the current code block.
  1060. /// For this, the lexer must have read the next token AFTER the token opening the
  1061. /// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).
  1062. /// After the call, Lexer.LookAhead will be the block-closing token.
  1063. /// </summary>
  1064. public void SkipCurrentBlock(int targetToken)
  1065. {
  1066. int lastKind = -1;
  1067. int kind = lastToken.kind;
  1068. while (kind != Tokens.EOF &&
  1069. !(lastKind == Tokens.End && kind == targetToken))
  1070. {
  1071. lastKind = kind;
  1072. NextToken();
  1073. kind = lastToken.kind;
  1074. }
  1075. }
  1076. public void SetInitialContext(SnippetType type)
  1077. {
  1078. ef.SetContext(type);
  1079. }
  1080. public VBLexerMemento Export()
  1081. {
  1082. return new VBLexerMemento() {
  1083. Column = Col,
  1084. Line = Line,
  1085. EncounteredLineContinuation = encounteredLineContinuation,
  1086. ExpressionFinder = ef.Export(),
  1087. InXmlMode = inXmlMode,
  1088. IsAtLineBegin = isAtLineBegin,
  1089. LineEnd = lineEnd,
  1090. PrevTokenKind = prevToken.kind,
  1091. MisreadExclamationMarkAsTypeCharacter = misreadExclamationMarkAsTypeCharacter,
  1092. XmlModeInfoStack = new Stack<XmlModeInfo>(xmlModeStack.Select(i => (XmlModeInfo)i.Clone()).Reverse())
  1093. };
  1094. }
  1095. LATextReader reader;
  1096. int col = 1;
  1097. int line = 1;
  1098. protected Errors errors = new Errors();
  1099. protected Token lastToken = null;
  1100. protected Token curToken = null;
  1101. protected Token peekToken = null;
  1102. string[] specialCommentTags = null;
  1103. protected Hashtable specialCommentHash = null;
  1104. // List<TagComment> tagComments = new List<TagComment>();
  1105. protected StringBuilder sb = new StringBuilder();
  1106. // protected SpecialTracker specialTracker = new SpecialTracker();
  1107. // used for the original value of strings (with escape sequences).
  1108. protected StringBuilder originalValue = new StringBuilder();
  1109. public bool SkipAllComments { get; set; }
  1110. public bool EvaluateConditionalCompilation { get; set; }
  1111. public virtual IDictionary<string, object> ConditionalCompilationSymbols {
  1112. get { throw new NotSupportedException(); }
  1113. }
  1114. protected static IEnumerable<string> GetSymbols (string symbols)
  1115. {
  1116. if (!string.IsNullOrEmpty(symbols)) {
  1117. foreach (string symbol in symbols.Split (';', ' ', '\t')) {
  1118. string s = symbol.Trim ();
  1119. if (s.Length == 0)
  1120. continue;
  1121. yield return s;
  1122. }
  1123. }
  1124. }
  1125. public void SetConditionalCompilationSymbols (string symbols)
  1126. {
  1127. throw new NotSupportedException ();
  1128. }
  1129. protected int Line {
  1130. get {
  1131. return line;
  1132. }
  1133. }
  1134. protected int Col {
  1135. get {
  1136. return col;
  1137. }
  1138. }
  1139. protected bool recordRead = false;
  1140. protected StringBuilder recordedText = new StringBuilder ();
  1141. protected int ReaderRead()
  1142. {
  1143. int val = reader.Read();
  1144. if (recordRead && val >= 0)
  1145. recordedText.Append ((char)val);
  1146. if ((val == '\r' && reader.Peek() != '\n') || val == '\n') {
  1147. ++line;
  1148. col = 1;
  1149. LineBreak();
  1150. } else if (val >= 0) {
  1151. col++;
  1152. }
  1153. return val;
  1154. }
  1155. protected int ReaderPeek()
  1156. {
  1157. return reader.Peek();
  1158. }
  1159. protected int ReaderPeek(int step)
  1160. {
  1161. return reader.Peek(step);
  1162. }
  1163. protected void ReaderSkip(int steps)
  1164. {
  1165. for (int i = 0; i < steps; i++) {
  1166. ReaderRead();
  1167. }
  1168. }
  1169. protected string ReaderPeekString(int length)
  1170. {
  1171. StringBuilder builder = new StringBuilder();
  1172. for (int i = 0; i < length; i++) {
  1173. int peek = ReaderPeek(i);
  1174. if (peek != -1)
  1175. builder.Append((char)peek);
  1176. }
  1177. return builder.ToString();
  1178. }
  1179. public void SetInitialLocation(TextLocation location)
  1180. {
  1181. if (lastToken != null || curToken != null || peekToken != null)
  1182. throw new InvalidOperationException();
  1183. this.line = location.Line;
  1184. this.col = location.Column;
  1185. }
  1186. public Errors Errors {
  1187. get {
  1188. return errors;
  1189. }
  1190. }
  1191. /// <summary>
  1192. /// Returns the comments that had been read and containing tag key words.
  1193. /// </summary>
  1194. // public List<TagComment> TagComments {
  1195. // get {
  1196. // return tagComments;
  1197. // }
  1198. // }
  1199. // public SpecialTracker SpecialTracker {
  1200. // get {
  1201. // return specialTracker;
  1202. // }
  1203. // }
  1204. /// <summary>
  1205. /// Special comment tags are tags like TODO, HACK or UNDONE which are read by the lexer and stored in <see cref="TagComments"/>.
  1206. /// </summary>
  1207. public string[] SpecialCommentTags {
  1208. get {
  1209. return specialCommentTags;
  1210. }
  1211. set {
  1212. specialCommentTags = value;
  1213. specialCommentHash = null;
  1214. if (specialCommentTags != null && specialCommentTags.Length > 0) {
  1215. specialCommentHash = new Hashtable();
  1216. foreach (string str in specialCommentTags) {
  1217. specialCommentHash.Add(str, null);
  1218. }
  1219. }
  1220. }
  1221. }
  1222. /// <summary>
  1223. /// The current Token. <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/>
  1224. /// </summary>
  1225. public Token Token {
  1226. get {
  1227. // Console.WriteLine("Call to Token");
  1228. return lastToken;
  1229. }
  1230. }
  1231. /// <summary>
  1232. /// The next Token (The <see cref="Token"/> after <see cref="NextToken"/> call) . <seealso cref="ICSharpCode.NRefactory.VB.Parser.Token"/>
  1233. /// </summary>
  1234. public Token LookAhead {
  1235. get {
  1236. // Console.WriteLine("Call to LookAhead");
  1237. return curToken;
  1238. }
  1239. }
  1240. #region System.IDisposable interface implementation
  1241. public virtual void Dispose()
  1242. {
  1243. reader.Close();
  1244. reader = null;
  1245. errors = null;
  1246. lastToken = curToken = peekToken = null;
  1247. specialCommentHash = null;
  1248. sb = originalValue = null;
  1249. }
  1250. #endregion
  1251. /// <summary>
  1252. /// Must be called before a peek operation.
  1253. /// </summary>
  1254. public void StartPeek()
  1255. {
  1256. peekToken = curToken;
  1257. }
  1258. /// <summary>
  1259. /// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on.
  1260. /// </summary>
  1261. /// <returns>An <see cref="Token"/> object.</returns>
  1262. public Token Peek()
  1263. {
  1264. // Console.WriteLine("Call to Peek");
  1265. if (peekToken.next == null) {
  1266. peekToken.next = Next();
  1267. }
  1268. peekToken = peekToken.next;
  1269. return peekToken;
  1270. }
  1271. protected static bool IsIdentifierPart(int ch)
  1272. {
  1273. if (ch == 95) return true; // 95 = '_'
  1274. if (ch == -1) return false;
  1275. return char.IsLetterOrDigit((char)ch); // accept unicode letters
  1276. }
  1277. protected static bool IsHex(char digit)
  1278. {
  1279. return Char.IsDigit(digit) || ('A' <= digit && digit <= 'F') || ('a' <= digit && digit <= 'f');
  1280. }
  1281. protected int GetHexNumber(char digit)
  1282. {
  1283. if (Char.IsDigit(digit)) {
  1284. return digit - '0';
  1285. }
  1286. if ('A' <= digit && digit <= 'F') {
  1287. return digit - 'A' + 0xA;
  1288. }
  1289. if ('a' <= digit && digit <= 'f') {
  1290. return digit - 'a' + 0xA;
  1291. }
  1292. errors.Error(line, col, String.Format("Invalid hex number '" + digit + "'"));
  1293. return 0;
  1294. }
  1295. protected TextLocation lastLineEnd = new TextLocation(1, 1);
  1296. protected TextLocation curLineEnd = new TextLocation(1, 1);
  1297. protected void LineBreak ()
  1298. {
  1299. lastLineEnd = curLineEnd;
  1300. curLineEnd = new TextLocation (line, col - 1);
  1301. }
  1302. protected bool HandleLineEnd(char ch)
  1303. {
  1304. // Handle MS-DOS or MacOS line ends.
  1305. if (ch == '\r') {
  1306. if (reader.Peek() == '\n') { // MS-DOS line end '\r\n'
  1307. ReaderRead(); // LineBreak (); called by ReaderRead ();
  1308. return true;
  1309. } else { // assume MacOS line end which is '\r'
  1310. LineBreak ();
  1311. return true;
  1312. }
  1313. }
  1314. if (ch == '\n') {
  1315. LineBreak ();
  1316. return true;
  1317. }
  1318. return false;
  1319. }
  1320. protected void SkipToEndOfLine()
  1321. {
  1322. int nextChar;
  1323. while ((nextChar = reader.Read()) != -1) {
  1324. if (nextChar == '\r') {
  1325. if (reader.Peek() == '\n')
  1326. reader.Read();
  1327. nextChar = '\n';
  1328. }
  1329. if (nextChar == '\n') {
  1330. ++line;
  1331. col = 1;
  1332. break;
  1333. }
  1334. }
  1335. }
  1336. protected string ReadToEndOfLine()
  1337. {
  1338. sb.Length = 0;
  1339. int nextChar;
  1340. while ((nextChar = reader.Read()) != -1) {
  1341. char ch = (char)nextChar;
  1342. if (nextChar == '\r') {
  1343. if (reader.Peek() == '\n')
  1344. reader.Read();
  1345. nextChar = '\n';
  1346. }
  1347. // Return read string, if EOL is reached
  1348. if (nextChar == '\n') {
  1349. ++line;
  1350. col = 1;
  1351. return sb.ToString();
  1352. }
  1353. sb.Append(ch);
  1354. }
  1355. // Got EOF before EOL
  1356. string retStr = sb.ToString();
  1357. col += retStr.Length;
  1358. return retStr;
  1359. }
  1360. public event EventHandler<SavepointEventArgs> SavepointReached;
  1361. protected virtual void OnSavepointReached(SavepointEventArgs e)
  1362. {
  1363. if (SavepointReached != null) {
  1364. SavepointReached(this, e);
  1365. }
  1366. }
  1367. }
  1368. }