/tools/Ruby/lib/ruby/1.8/irb/ruby-token.rb

http://github.com/agross/netopenspace · Ruby · 273 lines · 233 code · 25 blank · 15 comment · 12 complexity · 68f114be2b0ed2c9b5673c7326e5889b MD5 · raw file

  1. #
  2. # irb/ruby-token.rb - ruby tokens
  3. # $Release Version: 0.9.5$
  4. # $Revision: 11708 $
  5. # $Date: 2007-02-13 08:01:19 +0900 (Tue, 13 Feb 2007) $
  6. # by Keiju ISHITSUKA(keiju@ruby-lang.org)
  7. #
  8. # --
  9. #
  10. #
  11. #
  12. module RubyToken
  13. EXPR_BEG = :EXPR_BEG
  14. EXPR_MID = :EXPR_MID
  15. EXPR_END = :EXPR_END
  16. EXPR_ARG = :EXPR_ARG
  17. EXPR_FNAME = :EXPR_FNAME
  18. EXPR_DOT = :EXPR_DOT
  19. EXPR_CLASS = :EXPR_CLASS
  20. # for ruby 1.4X
  21. if !defined?(Symbol)
  22. Symbol = Integer
  23. end
  24. class Token
  25. def initialize(seek, line_no, char_no)
  26. @seek = seek
  27. @line_no = line_no
  28. @char_no = char_no
  29. end
  30. attr :seek
  31. attr :line_no
  32. attr :char_no
  33. end
  34. class TkNode < Token
  35. def initialize(seek, line_no, char_no)
  36. super
  37. end
  38. attr :node
  39. end
  40. class TkId < Token
  41. def initialize(seek, line_no, char_no, name)
  42. super(seek, line_no, char_no)
  43. @name = name
  44. end
  45. attr :name
  46. end
  47. class TkVal < Token
  48. def initialize(seek, line_no, char_no, value = nil)
  49. super(seek, line_no, char_no)
  50. @value = value
  51. end
  52. attr :value
  53. end
  54. class TkOp < Token
  55. attr :name, true
  56. end
  57. class TkOPASGN < TkOp
  58. def initialize(seek, line_no, char_no, op)
  59. super(seek, line_no, char_no)
  60. op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
  61. @op = op
  62. end
  63. attr :op
  64. end
  65. class TkUnknownChar < Token
  66. def initialize(seek, line_no, char_no, id)
  67. super(seek, line_no, char_no)
  68. @name = name
  69. end
  70. attr :name
  71. end
  72. class TkError < Token
  73. end
  74. def Token(token, value = nil)
  75. case token
  76. when String
  77. if (tk = TkReading2Token[token]).nil?
  78. IRB.fail TkReading2TokenNoKey, token
  79. end
  80. tk = Token(tk[0], value)
  81. if tk.kind_of?(TkOp)
  82. tk.name = token
  83. end
  84. return tk
  85. when Symbol
  86. if (tk = TkSymbol2Token[token]).nil?
  87. IRB.fail TkSymbol2TokenNoKey, token
  88. end
  89. return Token(tk[0], value)
  90. else
  91. if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
  92. token.new(@prev_seek, @prev_line_no, @prev_char_no)
  93. else
  94. token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
  95. end
  96. end
  97. end
  98. TokenDefinitions = [
  99. [:TkCLASS, TkId, "class", EXPR_CLASS],
  100. [:TkMODULE, TkId, "module", EXPR_BEG],
  101. [:TkDEF, TkId, "def", EXPR_FNAME],
  102. [:TkUNDEF, TkId, "undef", EXPR_FNAME],
  103. [:TkBEGIN, TkId, "begin", EXPR_BEG],
  104. [:TkRESCUE, TkId, "rescue", EXPR_MID],
  105. [:TkENSURE, TkId, "ensure", EXPR_BEG],
  106. [:TkEND, TkId, "end", EXPR_END],
  107. [:TkIF, TkId, "if", EXPR_BEG, :TkIF_MOD],
  108. [:TkUNLESS, TkId, "unless", EXPR_BEG, :TkUNLESS_MOD],
  109. [:TkTHEN, TkId, "then", EXPR_BEG],
  110. [:TkELSIF, TkId, "elsif", EXPR_BEG],
  111. [:TkELSE, TkId, "else", EXPR_BEG],
  112. [:TkCASE, TkId, "case", EXPR_BEG],
  113. [:TkWHEN, TkId, "when", EXPR_BEG],
  114. [:TkWHILE, TkId, "while", EXPR_BEG, :TkWHILE_MOD],
  115. [:TkUNTIL, TkId, "until", EXPR_BEG, :TkUNTIL_MOD],
  116. [:TkFOR, TkId, "for", EXPR_BEG],
  117. [:TkBREAK, TkId, "break", EXPR_END],
  118. [:TkNEXT, TkId, "next", EXPR_END],
  119. [:TkREDO, TkId, "redo", EXPR_END],
  120. [:TkRETRY, TkId, "retry", EXPR_END],
  121. [:TkIN, TkId, "in", EXPR_BEG],
  122. [:TkDO, TkId, "do", EXPR_BEG],
  123. [:TkRETURN, TkId, "return", EXPR_MID],
  124. [:TkYIELD, TkId, "yield", EXPR_END],
  125. [:TkSUPER, TkId, "super", EXPR_END],
  126. [:TkSELF, TkId, "self", EXPR_END],
  127. [:TkNIL, TkId, "nil", EXPR_END],
  128. [:TkTRUE, TkId, "true", EXPR_END],
  129. [:TkFALSE, TkId, "false", EXPR_END],
  130. [:TkAND, TkId, "and", EXPR_BEG],
  131. [:TkOR, TkId, "or", EXPR_BEG],
  132. [:TkNOT, TkId, "not", EXPR_BEG],
  133. [:TkIF_MOD, TkId],
  134. [:TkUNLESS_MOD, TkId],
  135. [:TkWHILE_MOD, TkId],
  136. [:TkUNTIL_MOD, TkId],
  137. [:TkALIAS, TkId, "alias", EXPR_FNAME],
  138. [:TkDEFINED, TkId, "defined?", EXPR_END],
  139. [:TklBEGIN, TkId, "BEGIN", EXPR_END],
  140. [:TklEND, TkId, "END", EXPR_END],
  141. [:Tk__LINE__, TkId, "__LINE__", EXPR_END],
  142. [:Tk__FILE__, TkId, "__FILE__", EXPR_END],
  143. [:TkIDENTIFIER, TkId],
  144. [:TkFID, TkId],
  145. [:TkGVAR, TkId],
  146. [:TkCVAR, TkId],
  147. [:TkIVAR, TkId],
  148. [:TkCONSTANT, TkId],
  149. [:TkINTEGER, TkVal],
  150. [:TkFLOAT, TkVal],
  151. [:TkSTRING, TkVal],
  152. [:TkXSTRING, TkVal],
  153. [:TkREGEXP, TkVal],
  154. [:TkSYMBOL, TkVal],
  155. [:TkDSTRING, TkNode],
  156. [:TkDXSTRING, TkNode],
  157. [:TkDREGEXP, TkNode],
  158. [:TkNTH_REF, TkNode],
  159. [:TkBACK_REF, TkNode],
  160. [:TkUPLUS, TkOp, "+@"],
  161. [:TkUMINUS, TkOp, "-@"],
  162. [:TkPOW, TkOp, "**"],
  163. [:TkCMP, TkOp, "<=>"],
  164. [:TkEQ, TkOp, "=="],
  165. [:TkEQQ, TkOp, "==="],
  166. [:TkNEQ, TkOp, "!="],
  167. [:TkGEQ, TkOp, ">="],
  168. [:TkLEQ, TkOp, "<="],
  169. [:TkANDOP, TkOp, "&&"],
  170. [:TkOROP, TkOp, "||"],
  171. [:TkMATCH, TkOp, "=~"],
  172. [:TkNMATCH, TkOp, "!~"],
  173. [:TkDOT2, TkOp, ".."],
  174. [:TkDOT3, TkOp, "..."],
  175. [:TkAREF, TkOp, "[]"],
  176. [:TkASET, TkOp, "[]="],
  177. [:TkLSHFT, TkOp, "<<"],
  178. [:TkRSHFT, TkOp, ">>"],
  179. [:TkCOLON2, TkOp],
  180. [:TkCOLON3, TkOp],
  181. # [:OPASGN, TkOp], # +=, -= etc. #
  182. [:TkASSOC, TkOp, "=>"],
  183. [:TkQUESTION, TkOp, "?"], #?
  184. [:TkCOLON, TkOp, ":"], #:
  185. [:TkfLPAREN], # func( #
  186. [:TkfLBRACK], # func[ #
  187. [:TkfLBRACE], # func{ #
  188. [:TkSTAR], # *arg
  189. [:TkAMPER], # &arg #
  190. [:TkSYMBEG], # :SYMBOL
  191. [:TkGT, TkOp, ">"],
  192. [:TkLT, TkOp, "<"],
  193. [:TkPLUS, TkOp, "+"],
  194. [:TkMINUS, TkOp, "-"],
  195. [:TkMULT, TkOp, "*"],
  196. [:TkDIV, TkOp, "/"],
  197. [:TkMOD, TkOp, "%"],
  198. [:TkBITOR, TkOp, "|"],
  199. [:TkBITXOR, TkOp, "^"],
  200. [:TkBITAND, TkOp, "&"],
  201. [:TkBITNOT, TkOp, "~"],
  202. [:TkNOTOP, TkOp, "!"],
  203. [:TkBACKQUOTE, TkOp, "`"],
  204. [:TkASSIGN, Token, "="],
  205. [:TkDOT, Token, "."],
  206. [:TkLPAREN, Token, "("], #(exp)
  207. [:TkLBRACK, Token, "["], #[arry]
  208. [:TkLBRACE, Token, "{"], #{hash}
  209. [:TkRPAREN, Token, ")"],
  210. [:TkRBRACK, Token, "]"],
  211. [:TkRBRACE, Token, "}"],
  212. [:TkCOMMA, Token, ","],
  213. [:TkSEMICOLON, Token, ";"],
  214. [:TkCOMMENT],
  215. [:TkRD_COMMENT],
  216. [:TkSPACE],
  217. [:TkNL],
  218. [:TkEND_OF_SCRIPT],
  219. [:TkBACKSLASH, TkUnknownChar, "\\"],
  220. [:TkAT, TkUnknownChar, "@"],
  221. [:TkDOLLAR, TkUnknownChar, "$"],
  222. ]
  223. # {reading => token_class}
  224. # {reading => [token_class, *opt]}
  225. TkReading2Token = {}
  226. TkSymbol2Token = {}
  227. def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
  228. token_n = token_n.id2name if token_n.kind_of?(Symbol)
  229. if RubyToken.const_defined?(token_n)
  230. IRB.fail AlreadyDefinedToken, token_n
  231. end
  232. token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
  233. if reading
  234. if TkReading2Token[reading]
  235. IRB.fail TkReading2TokenDuplicateError, token_n, reading
  236. end
  237. if opts.empty?
  238. TkReading2Token[reading] = [token_c]
  239. else
  240. TkReading2Token[reading] = [token_c].concat(opts)
  241. end
  242. end
  243. TkSymbol2Token[token_n.intern] = token_c
  244. end
  245. for defs in TokenDefinitions
  246. def_token(*defs)
  247. end
  248. end