/tags/jsdoc_toolkit-2.3.2/jsdoc-toolkit/app/lib/JSDOC/TokenReader.js

http://jsdoc-toolkit.googlecode.com/ · JavaScript · 332 lines · 229 code · 35 blank · 68 comment · 118 complexity · 04c1f96bce1ab497f30c3943f5407890 MD5 · raw file

  1. if (typeof JSDOC == "undefined") JSDOC = {};
  2. /**
  3. @class Search a {@link JSDOC.TextStream} for language tokens.
  4. */
  5. JSDOC.TokenReader = function() {
  6. this.keepDocs = true;
  7. this.keepWhite = false;
  8. this.keepComments = false;
  9. }
  10. /**
  11. @type {JSDOC.Token[]}
  12. */
  13. JSDOC.TokenReader.prototype.tokenize = function(/**JSDOC.TextStream*/stream) {
  14. var tokens = [];
  15. /**@ignore*/ tokens.last = function() { return tokens[tokens.length-1]; }
  16. /**@ignore*/ tokens.lastSym = function() {
  17. for (var i = tokens.length-1; i >= 0; i--) {
  18. if (!(tokens[i].is("WHIT") || tokens[i].is("COMM"))) return tokens[i];
  19. }
  20. }
  21. while (!stream.look().eof) {
  22. if (this.read_mlcomment(stream, tokens)) continue;
  23. if (this.read_slcomment(stream, tokens)) continue;
  24. if (this.read_dbquote(stream, tokens)) continue;
  25. if (this.read_snquote(stream, tokens)) continue;
  26. if (this.read_regx(stream, tokens)) continue;
  27. if (this.read_numb(stream, tokens)) continue;
  28. if (this.read_punc(stream, tokens)) continue;
  29. if (this.read_newline(stream, tokens)) continue;
  30. if (this.read_space(stream, tokens)) continue;
  31. if (this.read_word(stream, tokens)) continue;
  32. // if execution reaches here then an error has happened
  33. tokens.push(new JSDOC.Token(stream.next(), "TOKN", "UNKNOWN_TOKEN"));
  34. }
  35. return tokens;
  36. }
  37. /**
  38. @returns {Boolean} Was the token found?
  39. */
  40. JSDOC.TokenReader.prototype.read_word = function(/**JSDOC.TokenStream*/stream, tokens) {
  41. var found = "";
  42. while (!stream.look().eof && JSDOC.Lang.isWordChar(stream.look())) {
  43. found += stream.next();
  44. }
  45. if (found === "") {
  46. return false;
  47. }
  48. else {
  49. var name;
  50. if ((name = JSDOC.Lang.keyword(found))) tokens.push(new JSDOC.Token(found, "KEYW", name));
  51. else tokens.push(new JSDOC.Token(found, "NAME", "NAME"));
  52. return true;
  53. }
  54. }
  55. /**
  56. @returns {Boolean} Was the token found?
  57. */
  58. JSDOC.TokenReader.prototype.read_punc = function(/**JSDOC.TokenStream*/stream, tokens) {
  59. var found = "";
  60. var name;
  61. while (!stream.look().eof && JSDOC.Lang.punc(found+stream.look())) {
  62. found += stream.next();
  63. }
  64. if (found === "") {
  65. return false;
  66. }
  67. else {
  68. tokens.push(new JSDOC.Token(found, "PUNC", JSDOC.Lang.punc(found)));
  69. return true;
  70. }
  71. }
  72. /**
  73. @returns {Boolean} Was the token found?
  74. */
  75. JSDOC.TokenReader.prototype.read_space = function(/**JSDOC.TokenStream*/stream, tokens) {
  76. var found = "";
  77. while (!stream.look().eof && JSDOC.Lang.isSpace(stream.look())) {
  78. found += stream.next();
  79. }
  80. if (found === "") {
  81. return false;
  82. }
  83. else {
  84. if (this.collapseWhite) found = " ";
  85. if (this.keepWhite) tokens.push(new JSDOC.Token(found, "WHIT", "SPACE"));
  86. return true;
  87. }
  88. }
  89. /**
  90. @returns {Boolean} Was the token found?
  91. */
  92. JSDOC.TokenReader.prototype.read_newline = function(/**JSDOC.TokenStream*/stream, tokens) {
  93. var found = "";
  94. while (!stream.look().eof && JSDOC.Lang.isNewline(stream.look())) {
  95. found += stream.next();
  96. }
  97. if (found === "") {
  98. return false;
  99. }
  100. else {
  101. if (this.collapseWhite) found = "\n";
  102. if (this.keepWhite) tokens.push(new JSDOC.Token(found, "WHIT", "NEWLINE"));
  103. return true;
  104. }
  105. }
  106. /**
  107. @returns {Boolean} Was the token found?
  108. */
  109. JSDOC.TokenReader.prototype.read_mlcomment = function(/**JSDOC.TokenStream*/stream, tokens) {
  110. if (stream.look() == "/" && stream.look(1) == "*") {
  111. var found = stream.next(2);
  112. while (!stream.look().eof && !(stream.look(-1) == "/" && stream.look(-2) == "*")) {
  113. found += stream.next();
  114. }
  115. // to start doclet we allow /** or /*** but not /**/ or /****
  116. if (/^\/\*\*([^\/]|\*[^*])/.test(found) && this.keepDocs) tokens.push(new JSDOC.Token(found, "COMM", "JSDOC"));
  117. else if (this.keepComments) tokens.push(new JSDOC.Token(found, "COMM", "MULTI_LINE_COMM"));
  118. return true;
  119. }
  120. return false;
  121. }
  122. /**
  123. @returns {Boolean} Was the token found?
  124. */
  125. JSDOC.TokenReader.prototype.read_slcomment = function(/**JSDOC.TokenStream*/stream, tokens) {
  126. var found;
  127. if (
  128. (stream.look() == "/" && stream.look(1) == "/" && (found=stream.next(2)))
  129. ||
  130. (stream.look() == "<" && stream.look(1) == "!" && stream.look(2) == "-" && stream.look(3) == "-" && (found=stream.next(4)))
  131. ) {
  132. while (!stream.look().eof && !JSDOC.Lang.isNewline(stream.look())) {
  133. found += stream.next();
  134. }
  135. if (this.keepComments) {
  136. tokens.push(new JSDOC.Token(found, "COMM", "SINGLE_LINE_COMM"));
  137. }
  138. return true;
  139. }
  140. return false;
  141. }
  142. /**
  143. @returns {Boolean} Was the token found?
  144. */
  145. JSDOC.TokenReader.prototype.read_dbquote = function(/**JSDOC.TokenStream*/stream, tokens) {
  146. if (stream.look() == "\"") {
  147. // find terminator
  148. var string = stream.next();
  149. while (!stream.look().eof) {
  150. if (stream.look() == "\\") {
  151. if (JSDOC.Lang.isNewline(stream.look(1))) {
  152. do {
  153. stream.next();
  154. } while (!stream.look().eof && JSDOC.Lang.isNewline(stream.look()));
  155. string += "\\\n";
  156. }
  157. else {
  158. string += stream.next(2);
  159. }
  160. }
  161. else if (stream.look() == "\"") {
  162. string += stream.next();
  163. tokens.push(new JSDOC.Token(string, "STRN", "DOUBLE_QUOTE"));
  164. return true;
  165. }
  166. else {
  167. string += stream.next();
  168. }
  169. }
  170. }
  171. return false; // error! unterminated string
  172. }
  173. /**
  174. @returns {Boolean} Was the token found?
  175. */
  176. JSDOC.TokenReader.prototype.read_snquote = function(/**JSDOC.TokenStream*/stream, tokens) {
  177. if (stream.look() == "'") {
  178. // find terminator
  179. var string = stream.next();
  180. while (!stream.look().eof) {
  181. if (stream.look() == "\\") { // escape sequence
  182. string += stream.next(2);
  183. }
  184. else if (stream.look() == "'") {
  185. string += stream.next();
  186. tokens.push(new JSDOC.Token(string, "STRN", "SINGLE_QUOTE"));
  187. return true;
  188. }
  189. else {
  190. string += stream.next();
  191. }
  192. }
  193. }
  194. return false; // error! unterminated string
  195. }
  196. /**
  197. @returns {Boolean} Was the token found?
  198. */
  199. JSDOC.TokenReader.prototype.read_numb = function(/**JSDOC.TokenStream*/stream, tokens) {
  200. if (stream.look() === "0" && stream.look(1) == "x") {
  201. return this.read_hex(stream, tokens);
  202. }
  203. var found = "";
  204. while (!stream.look().eof && JSDOC.Lang.isNumber(found+stream.look())){
  205. found += stream.next();
  206. }
  207. if (found === "") {
  208. return false;
  209. }
  210. else {
  211. if (/^0[0-7]/.test(found)) tokens.push(new JSDOC.Token(found, "NUMB", "OCTAL"));
  212. else tokens.push(new JSDOC.Token(found, "NUMB", "DECIMAL"));
  213. return true;
  214. }
  215. }
  216. /*t:
  217. requires("../lib/JSDOC/TextStream.js");
  218. requires("../lib/JSDOC/Token.js");
  219. requires("../lib/JSDOC/Lang.js");
  220. plan(3, "testing JSDOC.TokenReader.prototype.read_numb");
  221. //// setup
  222. var src = "function foo(num){while (num+8.0 >= 0x20 && num < 0777){}}";
  223. var tr = new JSDOC.TokenReader();
  224. var tokens = tr.tokenize(new JSDOC.TextStream(src));
  225. var hexToken, octToken, decToken;
  226. for (var i = 0; i < tokens.length; i++) {
  227. if (tokens[i].name == "HEX_DEC") hexToken = tokens[i];
  228. if (tokens[i].name == "OCTAL") octToken = tokens[i];
  229. if (tokens[i].name == "DECIMAL") decToken = tokens[i];
  230. }
  231. ////
  232. is(decToken.data, "8.0", "decimal number is found in source.");
  233. is(hexToken.data, "0x20", "hexdec number is found in source (issue #99).");
  234. is(octToken.data, "0777", "octal number is found in source.");
  235. */
  236. /**
  237. @returns {Boolean} Was the token found?
  238. */
  239. JSDOC.TokenReader.prototype.read_hex = function(/**JSDOC.TokenStream*/stream, tokens) {
  240. var found = stream.next(2);
  241. while (!stream.look().eof) {
  242. if (JSDOC.Lang.isHexDec(found) && !JSDOC.Lang.isHexDec(found+stream.look())) { // done
  243. tokens.push(new JSDOC.Token(found, "NUMB", "HEX_DEC"));
  244. return true;
  245. }
  246. else {
  247. found += stream.next();
  248. }
  249. }
  250. return false;
  251. }
  252. /**
  253. @returns {Boolean} Was the token found?
  254. */
  255. JSDOC.TokenReader.prototype.read_regx = function(/**JSDOC.TokenStream*/stream, tokens) {
  256. var last;
  257. if (
  258. stream.look() == "/"
  259. &&
  260. (
  261. (
  262. !(last = tokens.lastSym()) // there is no last, the regex is the first symbol
  263. ||
  264. (
  265. !last.is("NUMB")
  266. && !last.is("NAME")
  267. && !last.is("RIGHT_PAREN")
  268. && !last.is("RIGHT_BRACKET")
  269. )
  270. )
  271. )
  272. ) {
  273. var regex = stream.next();
  274. while (!stream.look().eof) {
  275. if (stream.look() == "\\") { // escape sequence
  276. regex += stream.next(2);
  277. }
  278. else if (stream.look() == "/") {
  279. regex += stream.next();
  280. while (/[gmi]/.test(stream.look())) {
  281. regex += stream.next();
  282. }
  283. tokens.push(new JSDOC.Token(regex, "REGX", "REGX"));
  284. return true;
  285. }
  286. else {
  287. regex += stream.next();
  288. }
  289. }
  290. // error: unterminated regex
  291. }
  292. return false;
  293. }