PageRenderTime 48ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 0ms

/main/contrib/NRefactory/Project/Src/Lexer/AbstractLexer.cs

https://github.com/jfcantin/monodevelop
C# | 374 lines | 284 code | 39 blank | 51 comment | 61 complexity | 7fc93dd0e8da2297e6bd4a8f24e34a26 MD5 | raw file
  1. // <file>
  2. // <copyright see="prj:///doc/copyright.txt"/>
  3. // <license see="prj:///doc/license.txt"/>
  4. // <owner name="Mike Kr├╝ger" email="mike@icsharpcode.net"/>
  5. // <version>$Revision: 4482 $</version>
  6. // </file>
  7. using System;
  8. using System.Collections;
  9. using System.Collections.Generic;
  10. using System.IO;
  11. using System.Text;
  12. namespace ICSharpCode.OldNRefactory.Parser
  13. {
  14. /// <summary>
  15. /// This is the base class for the C# and VB.NET lexer
  16. /// </summary>
  17. [System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Naming", "CA1708:IdentifiersShouldDifferByMoreThanCase")]
  18. public abstract class AbstractLexer : ILexer
  19. {
  20. TextReader reader;
  21. int col = 1;
  22. int line = 1;
  23. [CLSCompliant(false)]
  24. protected Errors errors = new Errors();
  25. protected Token lastToken = null;
  26. protected Token curToken = null;
  27. protected Token peekToken = null;
  28. string[] specialCommentTags = null;
  29. protected Hashtable specialCommentHash = null;
  30. List<TagComment> tagComments = new List<TagComment>();
  31. protected StringBuilder sb = new StringBuilder();
  32. [CLSCompliant(false)]
  33. protected SpecialTracker specialTracker = new SpecialTracker();
  34. // used for the original value of strings (with escape sequences).
  35. protected StringBuilder originalValue = new StringBuilder();
  36. public bool SkipAllComments { get; set; }
  37. public bool EvaluateConditionalCompilation { get; set; }
  38. public virtual IDictionary<string, object> ConditionalCompilationSymbols {
  39. get { throw new NotSupportedException(); }
  40. }
  41. protected static IEnumerable<string> GetSymbols (string symbols)
  42. {
  43. if (!string.IsNullOrEmpty(symbols)) {
  44. foreach (string symbol in symbols.Split (';', ' ', '\t')) {
  45. string s = symbol.Trim ();
  46. if (s.Length == 0)
  47. continue;
  48. yield return s;
  49. }
  50. }
  51. }
  52. public virtual void SetConditionalCompilationSymbols (string symbols)
  53. {
  54. throw new NotSupportedException ();
  55. }
  56. protected int Line {
  57. get {
  58. return line;
  59. }
  60. }
  61. protected int Col {
  62. get {
  63. return col;
  64. }
  65. }
  66. protected bool recordRead = false;
  67. protected StringBuilder recordedText = new StringBuilder ();
  68. protected int ReaderRead()
  69. {
  70. ++col;
  71. int val = reader.Read();
  72. if (recordRead)
  73. recordedText.Append ((char)val);
  74. if (val == '\r') {
  75. if (reader.Peek() == '\n') {
  76. lineBreakPosition = new Location (col + 2, line);
  77. reader.Read ();
  78. } else {
  79. lineBreakPosition = new Location (col + 1, line);
  80. }
  81. ++line;
  82. col = 1;
  83. LineBreak ();
  84. return '\n';
  85. }
  86. if (val == '\n') {
  87. lineBreakPosition = new Location (col + 1, line);
  88. ++line;
  89. col = 1;
  90. LineBreak ();
  91. }
  92. return val;
  93. }
  94. protected int ReaderPeek()
  95. {
  96. return reader.Peek();
  97. }
  98. public Errors Errors {
  99. get {
  100. return errors;
  101. }
  102. }
  103. /// <summary>
  104. /// Returns the comments that had been read and containing tag key words.
  105. /// </summary>
  106. public List<TagComment> TagComments {
  107. get {
  108. return tagComments;
  109. }
  110. }
  111. public SpecialTracker SpecialTracker {
  112. get {
  113. return specialTracker;
  114. }
  115. }
  116. /// <summary>
  117. /// Special comment tags are tags like TODO, HACK or UNDONE which are read by the lexer and stored in <see cref="TagComments"/>.
  118. /// </summary>
  119. public string[] SpecialCommentTags {
  120. get {
  121. return specialCommentTags;
  122. }
  123. set {
  124. specialCommentTags = value;
  125. specialCommentHash = null;
  126. if (specialCommentTags != null && specialCommentTags.Length > 0) {
  127. specialCommentHash = new Hashtable();
  128. foreach (string str in specialCommentTags) {
  129. specialCommentHash.Add(str, null);
  130. }
  131. }
  132. }
  133. }
  134. /// <summary>
  135. /// The current Token. <seealso cref="ICSharpCode.OldNRefactory.Parser.Token"/>
  136. /// </summary>
  137. public Token Token {
  138. get {
  139. // Console.WriteLine("Call to Token");
  140. return lastToken;
  141. }
  142. }
  143. /// <summary>
  144. /// The next Token (The <see cref="Token"/> after <see cref="NextToken"/> call) . <seealso cref="ICSharpCode.OldNRefactory.Parser.Token"/>
  145. /// </summary>
  146. public Token LookAhead {
  147. get {
  148. // Console.WriteLine("Call to LookAhead");
  149. return curToken;
  150. }
  151. }
  152. /// <summary>
  153. /// Constructor for the abstract lexer class.
  154. /// </summary>
  155. protected AbstractLexer(TextReader reader)
  156. {
  157. this.reader = reader;
  158. }
  159. #region System.IDisposable interface implementation
  160. public virtual void Dispose()
  161. {
  162. reader.Close();
  163. reader = null;
  164. errors = null;
  165. lastToken = curToken = peekToken = null;
  166. specialCommentHash = null;
  167. tagComments = null;
  168. sb = originalValue = null;
  169. }
  170. #endregion
  171. /// <summary>
  172. /// Must be called before a peek operation.
  173. /// </summary>
  174. public void StartPeek()
  175. {
  176. peekToken = curToken;
  177. }
  178. /// <summary>
  179. /// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on.
  180. /// </summary>
  181. /// <returns>An <see cref="Token"/> object.</returns>
  182. public Token Peek()
  183. {
  184. // Console.WriteLine("Call to Peek");
  185. if (peekToken.next == null) {
  186. peekToken.next = Next();
  187. specialTracker.InformToken(peekToken.next.kind);
  188. }
  189. peekToken = peekToken.next;
  190. return peekToken;
  191. }
  192. /// <summary>
  193. /// Reads the next token and gives it back.
  194. /// </summary>
  195. /// <returns>An <see cref="Token"/> object.</returns>
  196. public virtual Token NextToken()
  197. {
  198. if (curToken == null) {
  199. curToken = Next();
  200. specialTracker.InformToken(curToken.kind);
  201. //Console.WriteLine(ICSharpCode.OldNRefactory.Parser.CSharp.Tokens.GetTokenString(curToken.kind) + " -- " + curToken.val + "(" + curToken.kind + ")");
  202. return curToken;
  203. }
  204. lastToken = curToken;
  205. if (curToken.next == null) {
  206. curToken.next = Next();
  207. if (curToken.next != null) {
  208. specialTracker.InformToken(curToken.next.kind);
  209. }
  210. }
  211. curToken = curToken.next;
  212. //Console.WriteLine(ICSharpCode.OldNRefactory.Parser.CSharp.Tokens.GetTokenString(curToken.kind) + " -- " + curToken.val + "(" + curToken.kind + ")");
  213. return curToken;
  214. }
  215. protected abstract Token Next();
  216. protected static bool IsIdentifierPart(int ch)
  217. {
  218. if (ch == 95) return true; // 95 = '_'
  219. if (ch == -1) return false;
  220. return char.IsLetterOrDigit((char)ch); // accept unicode letters
  221. }
  222. protected static bool IsHex(char digit)
  223. {
  224. return Char.IsDigit(digit) || ('A' <= digit && digit <= 'F') || ('a' <= digit && digit <= 'f');
  225. }
  226. protected int GetHexNumber(char digit)
  227. {
  228. if (Char.IsDigit(digit)) {
  229. return digit - '0';
  230. }
  231. if ('A' <= digit && digit <= 'F') {
  232. return digit - 'A' + 0xA;
  233. }
  234. if ('a' <= digit && digit <= 'f') {
  235. return digit - 'a' + 0xA;
  236. }
  237. errors.Error(line, col, String.Format("Invalid hex number '" + digit + "'"));
  238. return 0;
  239. }
  240. protected Location lineBreakPosition = new Location (1, 1);
  241. protected Location lastLineEnd = new Location (1, 1);
  242. protected Location curLineEnd = new Location (1, 1);
  243. protected void LineBreak ()
  244. {
  245. lastLineEnd = curLineEnd;
  246. curLineEnd = new Location (col - 1, line);
  247. }
  248. protected bool HandleLineEnd(char ch)
  249. {
  250. // Handle MS-DOS or MacOS line ends.
  251. if (ch == '\r') {
  252. if (reader.Peek() == '\n') { // MS-DOS line end '\r\n'
  253. ReaderRead(); // LineBreak (); called by ReaderRead ();
  254. return true;
  255. } else { // assume MacOS line end which is '\r'
  256. LineBreak ();
  257. return true;
  258. }
  259. }
  260. if (ch == '\n') {
  261. LineBreak ();
  262. return true;
  263. }
  264. return false;
  265. }
  266. protected void SkipToEndOfLine()
  267. {
  268. int nextChar;
  269. while ((nextChar = reader.Read()) != -1) {
  270. char ch = (char)nextChar;
  271. if (ch == '\r') {
  272. if (reader.Peek() == '\n') {
  273. lineBreakPosition = new Location (col + 2, line);
  274. reader.Read();
  275. } else {
  276. lineBreakPosition = new Location (col + 1, line);
  277. }
  278. ++line;
  279. col = 1;
  280. return;
  281. }
  282. // Return read string, if EOL is reached
  283. if (ch == '\n') {
  284. lineBreakPosition = new Location (col + 1, line);
  285. ++line;
  286. col = 1;
  287. return;
  288. }
  289. }
  290. }
  291. protected string ReadToEndOfLine()
  292. {
  293. return ReadToEndOfLine (true);
  294. }
  295. protected string ReadToEndOfLine(bool resetBuilder)
  296. {
  297. if (resetBuilder)
  298. sb.Length = 0;
  299. int nextChar;
  300. while ((nextChar = reader.Read()) != -1) {
  301. char ch = (char)nextChar;
  302. if (nextChar == '\r') {
  303. if (reader.Peek() == '\n') {
  304. lineBreakPosition = new Location (col + 2, line);
  305. reader.Read();
  306. } else {
  307. lineBreakPosition = new Location (col + 1, line);
  308. }
  309. ++line;
  310. col = 1;
  311. return sb.ToString();
  312. }
  313. // Return read string, if EOL is reached
  314. if (nextChar == '\n') {
  315. lineBreakPosition = new Location (col + 1, line);
  316. ++line;
  317. col = 1;
  318. return sb.ToString();
  319. }
  320. sb.Append(ch);
  321. }
  322. // Got EOF before EOL
  323. string retStr = sb.ToString();
  324. col += retStr.Length;
  325. return retStr;
  326. }
  327. /// <summary>
  328. /// Skips to the end of the current code block.
  329. /// For this, the lexer must have read the next token AFTER the token opening the
  330. /// block (so that Lexer.Token is the block-opening token, not Lexer.LookAhead).
  331. /// After the call, Lexer.LookAhead will be the block-closing token.
  332. /// </summary>
  333. public abstract void SkipCurrentBlock(int targetToken);
  334. }
  335. }