PageRenderTime 30ms CodeModel.GetById 23ms app.highlight 4ms RepoModel.GetById 1ms app.codeStats 0ms

/Microsoft.Scripting/Runtime/TokenizerService.cs

https://bitbucket.org/stefanrusek/xronos
C# | 107 lines | 41 code | 17 blank | 49 comment | 6 complexity | 2b78d6d7ee90c73073299d91f5f90b09 MD5 | raw file
  1/* ****************************************************************************
  2 *
  3 * Copyright (c) Microsoft Corporation. 
  4 *
  5 * This source code is subject to terms and conditions of the Microsoft Public License. A 
  6 * copy of the license can be found in the License.html file at the root of this distribution. If 
  7 * you cannot locate the  Microsoft Public License, please send an email to 
  8 * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 
  9 * by the terms of the Microsoft Public License.
 10 *
 11 * You must not remove this notice, or any other, from this software.
 12 *
 13 *
 14 * ***************************************************************************/
 15using System; using Microsoft;
 16
 17
 18using System.Collections.Generic;
 19#if CODEPLEX_40
 20using System.Dynamic;
 21#else
 22using Microsoft.Scripting;
 23#endif
 24using System.IO;
 25
 26namespace Microsoft.Scripting.Runtime {
 27    public abstract class TokenizerService {
 28
 29        // static contract:
 30        protected TokenizerService() {
 31        }
 32
 33        public abstract void Initialize(object state, TextReader sourceReader, SourceUnit sourceUnit, SourceLocation initialLocation);
 34
 35        /// <summary>
 36        /// The current internal state of the scanner.
 37        /// </summary>
 38        public abstract object CurrentState { get; }
 39
 40        /// <summary>
 41        /// The current startLocation of the scanner.
 42        /// </summary>
 43        public abstract SourceLocation CurrentPosition { get; }
 44
 45        /// <summary>
 46        /// Move the tokenizer past the next token and return its category.
 47        /// </summary>
 48        /// <returns>The token information associated with the token just scanned.</returns>
 49        public abstract TokenInfo ReadToken();
 50
 51        public abstract bool IsRestartable { get; }
 52        public abstract ErrorSink ErrorSink { get; set; }
 53
 54        /// <summary>
 55        /// Move the tokenizer past the next token.
 56        /// </summary>
 57        /// <returns><c>False</c> if the end of stream has been reached, <c>true</c> otherwise.</returns>
 58        public virtual bool SkipToken() {
 59            return ReadToken().Category != TokenCategory.EndOfStream;
 60        }
 61
 62        // TODO: shouldn't be virutal (JS tokenizer needs to be fixed)
 63        /// <summary>
 64        /// Get all tokens over a block of the stream.
 65        /// </summary>
 66        /// <remarks>
 67        /// <para>
 68        /// The scanner should return full tokens. If startLocation + length lands in the middle of a token, the full token
 69        /// should be returned.
 70        /// </para>
 71        /// </remarks>
 72        /// <param name="countOfChars">The mininum number of characters to process while getting tokens.</param>
 73        /// <returns>A enumeration of tokens.</returns>
 74        public virtual IEnumerable<TokenInfo> ReadTokens(int countOfChars) {
 75            List<TokenInfo> tokens = new List<TokenInfo>();
 76
 77            int start_index = CurrentPosition.Index;
 78
 79            while (CurrentPosition.Index - start_index < countOfChars) {
 80                TokenInfo token = ReadToken();
 81                if (token.Category == TokenCategory.EndOfStream) break;
 82                tokens.Add(token);
 83            }
 84
 85            return tokens;
 86        }
 87
 88        /// <summary>
 89        /// Scan from startLocation to at least startLocation + length.
 90        /// </summary>
 91        /// <param name="countOfChars">The mininum number of characters to process while getting tokens.</param>
 92        /// <remarks>
 93        /// This method is used to determine state at arbitrary startLocation.
 94        /// </remarks>
 95        /// <returns><c>False</c> if the end of stream has been reached, <c>true</c> otherwise.</returns>
 96        public bool SkipTokens(int countOfChars) {
 97            bool eos = false;
 98            int start_index = CurrentPosition.Index;
 99
100            while (CurrentPosition.Index - start_index < countOfChars && (eos = SkipToken())) {
101                ;
102            }
103
104            return eos;
105        }
106    }
107}