/Tokenizer.h

  1/**
  2 * Copyright (C) 2010 Jakub Wieczorek <fawek@fawek.net>
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a copy
  5 * of this software and associated documentation files (the "Software"), to deal
  6 * in the Software without restriction, including without limitation the rights
  7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  8 * copies of the Software, and to permit persons to whom the Software is
  9 * furnished to do so, subject to the following conditions:
 10 * 
 11 * The above copyright notice and this permission notice shall be included in
 12 * all copies or substantial portions of the Software.
 13 * 
 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 20 * SOFTWARE.
 21 */
 22
 23#ifndef _TOKENIZER_H
 24#define	_TOKENIZER_H
 25
 26#include "Defines.h"
 27
 28#include <istream>
 29#include <string>
 30
 31class Tokenizer {
 32public:
 33    class Token {
 34    public:
 35        enum Type {
 36            None,
 37            Text,
 38            OpenComment,
 39            CloseComment,
 40            OpenVariable,
 41            CloseVariable,
 42            OpenTag,
 43            CloseTag,
 44            EndOfInput
 45        };
 46
 47        Token()
 48            : type(None)
 49        {
 50        }
 51
 52        void clear()
 53        {
 54            type = None;
 55            contents.clear();
 56        }
 57
 58        Type type;
 59        std::string contents;
 60
 61    private:
 62        DISALLOW_COPY_AND_ASSIGN(Token)
 63    };
 64
 65    enum State {
 66        None,
 67        Text,
 68        OpenComment,
 69        CommentText,
 70        CloseComment,
 71        OpenVariable,
 72        VariableExpression,
 73        LiteralInVariableExpression,
 74        CloseVariable,
 75        OpenTag,
 76        TagExpression,
 77        LiteralInTagExpression,
 78        CloseTag,
 79        EndOfInput
 80    };
 81
 82    Tokenizer(std::istream* stream);
 83    ~Tokenizer();
 84
 85    void nextToken(Token& token);
 86
 87    const int& lineNumber() const { return m_lineNumber; }
 88    const int& columnNumber() const { return m_columnNumber; }
 89
 90protected:
 91    inline bool nextCharacter(char& c);
 92    inline bool peekCharacter(char& c);
 93
 94private:
 95    State m_state;
 96    std::istream* m_stream;
 97
 98    int m_lineNumber;
 99    int m_columnNumber;
100};
101
102#endif	/* _TOKENIZER_H */