/Tokenizer.h
C Header | 102 lines | 65 code | 16 blank | 21 comment | 0 complexity | a17685ef149f595b178c3d52b82be2e6 MD5 | raw file
1/** 2 * Copyright (C) 2010 Jakub Wieczorek <fawek@fawek.net> 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a copy 5 * of this software and associated documentation files (the "Software"), to deal 6 * in the Software without restriction, including without limitation the rights 7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 8 * copies of the Software, and to permit persons to whom the Software is 9 * furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 17 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 19 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 20 * SOFTWARE. 21 */ 22 23#ifndef _TOKENIZER_H 24#define _TOKENIZER_H 25 26#include "Defines.h" 27 28#include <istream> 29#include <string> 30 31class Tokenizer { 32public: 33 class Token { 34 public: 35 enum Type { 36 None, 37 Text, 38 OpenComment, 39 CloseComment, 40 OpenVariable, 41 CloseVariable, 42 OpenTag, 43 CloseTag, 44 EndOfInput 45 }; 46 47 Token() 48 : type(None) 49 { 50 } 51 52 void clear() 53 { 54 type = None; 55 contents.clear(); 56 } 57 58 Type type; 59 std::string contents; 60 61 private: 62 DISALLOW_COPY_AND_ASSIGN(Token) 63 }; 64 65 enum State { 66 None, 67 Text, 68 OpenComment, 69 CommentText, 70 CloseComment, 71 OpenVariable, 72 VariableExpression, 73 LiteralInVariableExpression, 74 CloseVariable, 75 OpenTag, 76 TagExpression, 77 LiteralInTagExpression, 78 CloseTag, 79 EndOfInput 80 }; 81 82 Tokenizer(std::istream* stream); 83 ~Tokenizer(); 84 85 void nextToken(Token& token); 86 87 const int& lineNumber() const { return m_lineNumber; } 88 const int& columnNumber() const { return m_columnNumber; } 89 90protected: 91 inline bool nextCharacter(char& c); 92 inline bool peekCharacter(char& c); 93 94private: 95 State m_state; 96 std::istream* m_stream; 97 98 int m_lineNumber; 99 int m_columnNumber; 100}; 101 102#endif /* _TOKENIZER_H */