xronos /Microsoft.Scripting/Runtime/TokenizerService.cs

Language C# Lines 108
MD5 Hash 2b78d6d7ee90c73073299d91f5f90b09 Estimated Cost $944 (why?)
Repository https://bitbucket.org/stefanrusek/xronos View Raw File
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
/* ****************************************************************************
 *
 * Copyright (c) Microsoft Corporation. 
 *
 * This source code is subject to terms and conditions of the Microsoft Public License. A 
 * copy of the license can be found in the License.html file at the root of this distribution. If 
 * you cannot locate the  Microsoft Public License, please send an email to 
 * dlr@microsoft.com. By using this source code in any fashion, you are agreeing to be bound 
 * by the terms of the Microsoft Public License.
 *
 * You must not remove this notice, or any other, from this software.
 *
 *
 * ***************************************************************************/
using System; using Microsoft;


using System.Collections.Generic;
#if CODEPLEX_40
using System.Dynamic;
#else
using Microsoft.Scripting;
#endif
using System.IO;

namespace Microsoft.Scripting.Runtime {
    public abstract class TokenizerService {

        // static contract:
        protected TokenizerService() {
        }

        public abstract void Initialize(object state, TextReader sourceReader, SourceUnit sourceUnit, SourceLocation initialLocation);

        /// <summary>
        /// The current internal state of the scanner.
        /// </summary>
        public abstract object CurrentState { get; }

        /// <summary>
        /// The current startLocation of the scanner.
        /// </summary>
        public abstract SourceLocation CurrentPosition { get; }

        /// <summary>
        /// Move the tokenizer past the next token and return its category.
        /// </summary>
        /// <returns>The token information associated with the token just scanned.</returns>
        public abstract TokenInfo ReadToken();

        public abstract bool IsRestartable { get; }
        public abstract ErrorSink ErrorSink { get; set; }

        /// <summary>
        /// Move the tokenizer past the next token.
        /// </summary>
        /// <returns><c>False</c> if the end of stream has been reached, <c>true</c> otherwise.</returns>
        public virtual bool SkipToken() {
            return ReadToken().Category != TokenCategory.EndOfStream;
        }

        // TODO: shouldn't be virutal (JS tokenizer needs to be fixed)
        /// <summary>
        /// Get all tokens over a block of the stream.
        /// </summary>
        /// <remarks>
        /// <para>
        /// The scanner should return full tokens. If startLocation + length lands in the middle of a token, the full token
        /// should be returned.
        /// </para>
        /// </remarks>
        /// <param name="countOfChars">The mininum number of characters to process while getting tokens.</param>
        /// <returns>A enumeration of tokens.</returns>
        public virtual IEnumerable<TokenInfo> ReadTokens(int countOfChars) {
            List<TokenInfo> tokens = new List<TokenInfo>();

            int start_index = CurrentPosition.Index;

            while (CurrentPosition.Index - start_index < countOfChars) {
                TokenInfo token = ReadToken();
                if (token.Category == TokenCategory.EndOfStream) break;
                tokens.Add(token);
            }

            return tokens;
        }

        /// <summary>
        /// Scan from startLocation to at least startLocation + length.
        /// </summary>
        /// <param name="countOfChars">The mininum number of characters to process while getting tokens.</param>
        /// <remarks>
        /// This method is used to determine state at arbitrary startLocation.
        /// </remarks>
        /// <returns><c>False</c> if the end of stream has been reached, <c>true</c> otherwise.</returns>
        public bool SkipTokens(int countOfChars) {
            bool eos = false;
            int start_index = CurrentPosition.Index;

            while (CurrentPosition.Index - start_index < countOfChars && (eos = SkipToken())) {
                ;
            }

            return eos;
        }
    }
}
Back to Top