/thirdparty/breakpad/third_party/protobuf/protobuf/src/google/protobuf/io/tokenizer.h

http://github.com/tomahawk-player/tomahawk · C++ Header · 313 lines · 105 code · 55 blank · 153 comment · 0 complexity · 41daeb774d73f97c5a8c961b67c55cd4 MD5 · raw file

  1. // Protocol Buffers - Google's data interchange format
  2. // Copyright 2008 Google Inc. All rights reserved.
  3. // http://code.google.com/p/protobuf/
  4. //
  5. // Redistribution and use in source and binary forms, with or without
  6. // modification, are permitted provided that the following conditions are
  7. // met:
  8. //
  9. // * Redistributions of source code must retain the above copyright
  10. // notice, this list of conditions and the following disclaimer.
  11. // * Redistributions in binary form must reproduce the above
  12. // copyright notice, this list of conditions and the following disclaimer
  13. // in the documentation and/or other materials provided with the
  14. // distribution.
  15. // * Neither the name of Google Inc. nor the names of its
  16. // contributors may be used to endorse or promote products derived from
  17. // this software without specific prior written permission.
  18. //
  19. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30. // Author: kenton@google.com (Kenton Varda)
  31. // Based on original Protocol Buffers design by
  32. // Sanjay Ghemawat, Jeff Dean, and others.
  33. //
  34. // Class for parsing tokenized text from a ZeroCopyInputStream.
  35. #ifndef GOOGLE_PROTOBUF_IO_TOKENIZER_H__
  36. #define GOOGLE_PROTOBUF_IO_TOKENIZER_H__
  37. #include <string>
  38. #include <google/protobuf/stubs/common.h>
  39. namespace google {
  40. namespace protobuf {
  41. namespace io {
  42. class ZeroCopyInputStream; // zero_copy_stream.h
  43. // Defined in this file.
  44. class ErrorCollector;
  45. class Tokenizer;
  46. // Abstract interface for an object which collects the errors that occur
  47. // during parsing. A typical implementation might simply print the errors
  48. // to stdout.
  49. class LIBPROTOBUF_EXPORT ErrorCollector {
  50. public:
  51. inline ErrorCollector() {}
  52. virtual ~ErrorCollector();
  53. // Indicates that there was an error in the input at the given line and
  54. // column numbers. The numbers are zero-based, so you may want to add
  55. // 1 to each before printing them.
  56. virtual void AddError(int line, int column, const string& message) = 0;
  57. // Indicates that there was a warning in the input at the given line and
  58. // column numbers. The numbers are zero-based, so you may want to add
  59. // 1 to each before printing them.
  60. virtual void AddWarning(int line, int column, const string& message) { }
  61. private:
  62. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ErrorCollector);
  63. };
  64. // This class converts a stream of raw text into a stream of tokens for
  65. // the protocol definition parser to parse. The tokens recognized are
  66. // similar to those that make up the C language; see the TokenType enum for
  67. // precise descriptions. Whitespace and comments are skipped. By default,
  68. // C- and C++-style comments are recognized, but other styles can be used by
  69. // calling set_comment_style().
  70. class LIBPROTOBUF_EXPORT Tokenizer {
  71. public:
  72. // Construct a Tokenizer that reads and tokenizes text from the given
  73. // input stream and writes errors to the given error_collector.
  74. // The caller keeps ownership of input and error_collector.
  75. Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector);
  76. ~Tokenizer();
  77. enum TokenType {
  78. TYPE_START, // Next() has not yet been called.
  79. TYPE_END, // End of input reached. "text" is empty.
  80. TYPE_IDENTIFIER, // A sequence of letters, digits, and underscores, not
  81. // starting with a digit. It is an error for a number
  82. // to be followed by an identifier with no space in
  83. // between.
  84. TYPE_INTEGER, // A sequence of digits representing an integer. Normally
  85. // the digits are decimal, but a prefix of "0x" indicates
  86. // a hex number and a leading zero indicates octal, just
  87. // like with C numeric literals. A leading negative sign
  88. // is NOT included in the token; it's up to the parser to
  89. // interpret the unary minus operator on its own.
  90. TYPE_FLOAT, // A floating point literal, with a fractional part and/or
  91. // an exponent. Always in decimal. Again, never
  92. // negative.
  93. TYPE_STRING, // A quoted sequence of escaped characters. Either single
  94. // or double quotes can be used, but they must match.
  95. // A string literal cannot cross a line break.
  96. TYPE_SYMBOL, // Any other printable character, like '!' or '+'.
  97. // Symbols are always a single character, so "!+$%" is
  98. // four tokens.
  99. };
  100. // Structure representing a token read from the token stream.
  101. struct Token {
  102. TokenType type;
  103. string text; // The exact text of the token as it appeared in
  104. // the input. e.g. tokens of TYPE_STRING will still
  105. // be escaped and in quotes.
  106. // "line" and "column" specify the position of the first character of
  107. // the token within the input stream. They are zero-based.
  108. int line;
  109. int column;
  110. int end_column;
  111. };
  112. // Get the current token. This is updated when Next() is called. Before
  113. // the first call to Next(), current() has type TYPE_START and no contents.
  114. const Token& current();
  115. // Return the previous token -- i.e. what current() returned before the
  116. // previous call to Next().
  117. const Token& previous();
  118. // Advance to the next token. Returns false if the end of the input is
  119. // reached.
  120. bool Next();
  121. // Parse helpers ---------------------------------------------------
  122. // Parses a TYPE_FLOAT token. This never fails, so long as the text actually
  123. // comes from a TYPE_FLOAT token parsed by Tokenizer. If it doesn't, the
  124. // result is undefined (possibly an assert failure).
  125. static double ParseFloat(const string& text);
  126. // Parses a TYPE_STRING token. This never fails, so long as the text actually
  127. // comes from a TYPE_STRING token parsed by Tokenizer. If it doesn't, the
  128. // result is undefined (possibly an assert failure).
  129. static void ParseString(const string& text, string* output);
  130. // Identical to ParseString, but appends to output.
  131. static void ParseStringAppend(const string& text, string* output);
  132. // Parses a TYPE_INTEGER token. Returns false if the result would be
  133. // greater than max_value. Otherwise, returns true and sets *output to the
  134. // result. If the text is not from a Token of type TYPE_INTEGER originally
  135. // parsed by a Tokenizer, the result is undefined (possibly an assert
  136. // failure).
  137. static bool ParseInteger(const string& text, uint64 max_value,
  138. uint64* output);
  139. // Options ---------------------------------------------------------
  140. // Set true to allow floats to be suffixed with the letter 'f'. Tokens
  141. // which would otherwise be integers but which have the 'f' suffix will be
  142. // forced to be interpreted as floats. For all other purposes, the 'f' is
  143. // ignored.
  144. void set_allow_f_after_float(bool value) { allow_f_after_float_ = value; }
  145. // Valid values for set_comment_style().
  146. enum CommentStyle {
  147. // Line comments begin with "//", block comments are delimited by "/*" and
  148. // "*/".
  149. CPP_COMMENT_STYLE,
  150. // Line comments begin with "#". No way to write block comments.
  151. SH_COMMENT_STYLE
  152. };
  153. // Sets the comment style.
  154. void set_comment_style(CommentStyle style) { comment_style_ = style; }
  155. // -----------------------------------------------------------------
  156. private:
  157. GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer);
  158. Token current_; // Returned by current().
  159. Token previous_; // Returned by previous().
  160. ZeroCopyInputStream* input_;
  161. ErrorCollector* error_collector_;
  162. char current_char_; // == buffer_[buffer_pos_], updated by NextChar().
  163. const char* buffer_; // Current buffer returned from input_.
  164. int buffer_size_; // Size of buffer_.
  165. int buffer_pos_; // Current position within the buffer.
  166. bool read_error_; // Did we previously encounter a read error?
  167. // Line and column number of current_char_ within the whole input stream.
  168. int line_;
  169. int column_;
  170. // Position in buffer_ where StartToken() was called. If the token
  171. // started in the previous buffer, this is zero, and current_.text already
  172. // contains the part of the token from the previous buffer. If not
  173. // currently parsing a token, this is -1.
  174. int token_start_;
  175. // Options.
  176. bool allow_f_after_float_;
  177. CommentStyle comment_style_;
  178. // Since we count columns we need to interpret tabs somehow. We'll take
  179. // the standard 8-character definition for lack of any way to do better.
  180. static const int kTabWidth = 8;
  181. // -----------------------------------------------------------------
  182. // Helper methods.
  183. // Consume this character and advance to the next one.
  184. void NextChar();
  185. // Read a new buffer from the input.
  186. void Refresh();
  187. // Called when the current character is the first character of a new
  188. // token (not including whitespace or comments).
  189. inline void StartToken();
  190. // Called when the current character is the first character after the
  191. // end of the last token. After this returns, current_.text will
  192. // contain all text consumed since StartToken() was called.
  193. inline void EndToken();
  194. // Convenience method to add an error at the current line and column.
  195. void AddError(const string& message) {
  196. error_collector_->AddError(line_, column_, message);
  197. }
  198. // -----------------------------------------------------------------
  199. // The following four methods are used to consume tokens of specific
  200. // types. They are actually used to consume all characters *after*
  201. // the first, since the calling function consumes the first character
  202. // in order to decide what kind of token is being read.
  203. // Read and consume a string, ending when the given delimiter is
  204. // consumed.
  205. void ConsumeString(char delimiter);
  206. // Read and consume a number, returning TYPE_FLOAT or TYPE_INTEGER
  207. // depending on what was read. This needs to know if the first
  208. // character was a zero in order to correctly recognize hex and octal
  209. // numbers.
  210. // It also needs to know if the first characted was a . to parse floating
  211. // point correctly.
  212. TokenType ConsumeNumber(bool started_with_zero, bool started_with_dot);
  213. // Consume the rest of a line.
  214. void ConsumeLineComment();
  215. // Consume until "*/".
  216. void ConsumeBlockComment();
  217. // -----------------------------------------------------------------
  218. // These helper methods make the parsing code more readable. The
  219. // "character classes" refered to are defined at the top of the .cc file.
  220. // Basically it is a C++ class with one method:
  221. // static bool InClass(char c);
  222. // The method returns true if c is a member of this "class", like "Letter"
  223. // or "Digit".
  224. // Returns true if the current character is of the given character
  225. // class, but does not consume anything.
  226. template<typename CharacterClass>
  227. inline bool LookingAt();
  228. // If the current character is in the given class, consume it and return
  229. // true. Otherwise return false.
  230. // e.g. TryConsumeOne<Letter>()
  231. template<typename CharacterClass>
  232. inline bool TryConsumeOne();
  233. // Like above, but try to consume the specific character indicated.
  234. inline bool TryConsume(char c);
  235. // Consume zero or more of the given character class.
  236. template<typename CharacterClass>
  237. inline void ConsumeZeroOrMore();
  238. // Consume one or more of the given character class or log the given
  239. // error message.
  240. // e.g. ConsumeOneOrMore<Digit>("Expected digits.");
  241. template<typename CharacterClass>
  242. inline void ConsumeOneOrMore(const char* error);
  243. };
  244. // inline methods ====================================================
  245. inline const Tokenizer::Token& Tokenizer::current() {
  246. return current_;
  247. }
  248. inline const Tokenizer::Token& Tokenizer::previous() {
  249. return previous_;
  250. }
  251. inline void Tokenizer::ParseString(const string& text, string* output) {
  252. output->clear();
  253. ParseStringAppend(text, output);
  254. }
  255. } // namespace io
  256. } // namespace protobuf
  257. } // namespace google
  258. #endif // GOOGLE_PROTOBUF_IO_TOKENIZER_H__