PageRenderTime 31ms CodeModel.GetById 16ms app.highlight 10ms RepoModel.GetById 1ms app.codeStats 1ms

/src/NUnit/UiException/CSharpParser/Lexer.cs

#
C# | 233 lines | 202 code | 8 blank | 23 comment | 0 complexity | 5200a2757dfcd1f04febc05be4e15040 MD5 | raw file
  1// ****************************************************************
  2// This is free software licensed under the NUnit license. You may
  3// obtain a copy of the license at http://nunit.org
  4// ****************************************************************
  5
  6using System;
  7using System.Collections.Generic;
  8using System.Text;
  9
 10namespace NUnit.UiException.CodeFormatters
 11{
 12    /// <summary>
 13    /// Splits a text formatted as C# code into a list of identified tokens.
 14    /// </summary>
 15    public class Lexer
 16    {
 17        /// <summary>
 18        /// Reading position in the current text.
 19        /// </summary>
 20        private int _position;
 21
 22        /// <summary>
 23        /// Text where to fetch tokens.
 24        /// </summary>
 25        private string _text;
 26
 27        /// <summary>
 28        /// Last identified token.
 29        /// </summary>
 30        private InternalToken _token;
 31
 32        /// <summary>
 33        /// Holds pre-defined sequences.
 34        /// </summary>
 35        private TokenDictionary _dictionary;
 36
 37        /// <summary>
 38        /// Builds a new instance of Lexer.
 39        /// </summary>
 40        public Lexer()
 41        {
 42            _position = 0;
 43            _text = "";
 44
 45            _dictionary = new TokenDictionary();
 46            _dictionary.Add("/*", LexerTag.CommentC_Open);
 47            _dictionary.Add("*/", LexerTag.CommentC_Close);
 48            _dictionary.Add("//", LexerTag.CommentCpp);
 49
 50            // Here: definition of one lengthed sequences
 51            _dictionary.Add("\\", LexerTag.Text);
 52            _dictionary.Add(" ", LexerTag.Separator);
 53            _dictionary.Add("\t", LexerTag.Separator);
 54            _dictionary.Add("\r", LexerTag.Separator);
 55            _dictionary.Add(".", LexerTag.Separator);
 56            _dictionary.Add(";", LexerTag.Separator);
 57            _dictionary.Add("[", LexerTag.Separator);
 58            _dictionary.Add("]", LexerTag.Separator);
 59            _dictionary.Add("(", LexerTag.Separator);
 60            _dictionary.Add(")", LexerTag.Separator);
 61            _dictionary.Add("#", LexerTag.Separator);
 62            _dictionary.Add(":", LexerTag.Separator);
 63            _dictionary.Add("<", LexerTag.Separator);
 64            _dictionary.Add(">", LexerTag.Separator);
 65            _dictionary.Add("=", LexerTag.Separator);
 66            _dictionary.Add(",", LexerTag.Separator);
 67            _dictionary.Add("\n", LexerTag.EndOfLine);
 68            _dictionary.Add("'", LexerTag.SingleQuote);
 69            _dictionary.Add("\"", LexerTag.DoubleQuote);
 70
 71            return;
 72        }
 73
 74        public TokenDictionary Dictionary
 75        {
 76            get { return (_dictionary); }
 77        }
 78
 79        /// <summary>
 80        /// Clear all previously defined sequences.
 81        /// </summary>
 82        protected void Clear()
 83        {
 84            _dictionary = new TokenDictionary();
 85
 86            return;
 87        }
 88
 89        /// <summary>
 90        /// Setup the text to be splitted in tokens. 
 91        /// 
 92        /// Client code must call Next() first before getting
 93        /// the first available token (if any).
 94        /// </summary>
 95        public void Parse(string codeCSharp)
 96        {
 97            UiExceptionHelper.CheckNotNull(codeCSharp, "text");
 98
 99            _text = codeCSharp;
100            _position = 0;
101
102            return;
103        }
104
105        /// <summary>
106        /// Gets the token identifed after a call to Next().
107        /// The value may be null if the end of the text has been reached.
108        /// </summary>
109        public LexToken CurrentToken
110        {
111            get { return (_token); }
112        }
113
114        /// <summary>
115        /// Checks whether there are none visited tokens.
116        /// </summary>
117        public bool HasNext()
118        {
119            return (_position < _text.Length);
120        }
121
122        /// <summary>
123        /// Call this method to visit iteratively all tokens in the source text.
124        /// Each time a token has been identifier, the method returns true and the
125        /// identified Token is place under the CurrentToken property.
126        ///   When there is not more token to visit, the method returns false
127        /// and null is set in CurrentToken property.
128        /// </summary>
129        public bool Next()
130        {
131            char c;
132            LexToken token;
133            string prediction;
134            int pos;
135            int count;
136            int prediction_length;
137
138            _token = null;
139
140            if (!HasNext())
141                return (false);
142
143            pos = _position;
144            _token = new InternalToken(pos);
145            prediction_length = _dictionary[0].Text.Length;
146
147            while (pos < _text.Length)
148            {
149                c = _text[pos];
150                _token.AppendsChar(c);
151
152                prediction = "";
153                if (pos + 1 < _text.Length)
154                {
155                    count = Math.Min(prediction_length, _text.Length - pos - 1);
156                    prediction = _text.Substring(pos + 1, count);
157                }
158
159                token = _dictionary.TryMatch(_token.Text, prediction);
160
161                if (token != null)
162                {
163                    _token.SetText(token.Text);
164                    _token.SetIndex(_position);
165                    _token.SetLexerTag(token.Tag);
166                    _position += _token.Text.Length;
167
168                    break;
169                }
170
171                pos++;
172            }
173
174            return (true);
175        }
176
177        #region InternalToken
178
179        class InternalToken :
180            LexToken
181        {
182            /// <summary>
183            /// Builds a concrete instance of LexToken. By default, created instance
184            /// are setup with LexerTag.Text value.
185            /// </summary>
186            /// <param name="startingPosition">The starting startingPosition of this token in the text.</param>
187            public InternalToken(int index)
188            {
189                _tag = LexerTag.Text;
190                _text = "";
191                _start = index;
192
193                return;
194            }
195
196            /// <summary>
197            /// Appends this character to this token.
198            /// </summary>
199            public void AppendsChar(char c)
200            {
201                _text += c;
202            }
203
204            /// <summary>
205            /// Removes the "count" ending character of this token.
206            /// </summary>
207            public void PopChars(int count)
208            {
209                _text = _text.Remove(_text.Length - count);
210            }
211
212            /// <summary>
213            /// Set a new value to the Tag property.
214            /// </summary>
215            public void SetLexerTag(LexerTag tag)
216            {
217                _tag = tag;
218            }
219
220            public void SetText(string text)
221            {
222                _text = text;
223            }
224
225            public void SetIndex(int index)
226            {
227                _start = index;
228            }
229        }
230
231        #endregion
232    }
233}