/Tokenizer.h

http://github.com/fawek/cjango · C Header · 102 lines · 65 code · 16 blank · 21 comment · 0 complexity · a17685ef149f595b178c3d52b82be2e6 MD5 · raw file

  1. /**
  2. * Copyright (C) 2010 Jakub Wieczorek <fawek@fawek.net>
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a copy
  5. * of this software and associated documentation files (the "Software"), to deal
  6. * in the Software without restriction, including without limitation the rights
  7. * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  8. * copies of the Software, and to permit persons to whom the Software is
  9. * furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17. * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20. * SOFTWARE.
  21. */
  22. #ifndef _TOKENIZER_H
  23. #define _TOKENIZER_H
  24. #include "Defines.h"
  25. #include <istream>
  26. #include <string>
  27. class Tokenizer {
  28. public:
  29. class Token {
  30. public:
  31. enum Type {
  32. None,
  33. Text,
  34. OpenComment,
  35. CloseComment,
  36. OpenVariable,
  37. CloseVariable,
  38. OpenTag,
  39. CloseTag,
  40. EndOfInput
  41. };
  42. Token()
  43. : type(None)
  44. {
  45. }
  46. void clear()
  47. {
  48. type = None;
  49. contents.clear();
  50. }
  51. Type type;
  52. std::string contents;
  53. private:
  54. DISALLOW_COPY_AND_ASSIGN(Token)
  55. };
  56. enum State {
  57. None,
  58. Text,
  59. OpenComment,
  60. CommentText,
  61. CloseComment,
  62. OpenVariable,
  63. VariableExpression,
  64. LiteralInVariableExpression,
  65. CloseVariable,
  66. OpenTag,
  67. TagExpression,
  68. LiteralInTagExpression,
  69. CloseTag,
  70. EndOfInput
  71. };
  72. Tokenizer(std::istream* stream);
  73. ~Tokenizer();
  74. void nextToken(Token& token);
  75. const int& lineNumber() const { return m_lineNumber; }
  76. const int& columnNumber() const { return m_columnNumber; }
  77. protected:
  78. inline bool nextCharacter(char& c);
  79. inline bool peekCharacter(char& c);
  80. private:
  81. State m_state;
  82. std::istream* m_stream;
  83. int m_lineNumber;
  84. int m_columnNumber;
  85. };
  86. #endif /* _TOKENIZER_H */