PageRenderTime 95ms CodeModel.GetById 38ms app.highlight 54ms RepoModel.GetById 1ms app.codeStats 0ms

/tools/Ruby/lib/ruby/1.8/irb/ruby-token.rb

http://github.com/agross/netopenspace
Ruby | 273 lines | 233 code | 25 blank | 15 comment | 12 complexity | 68f114be2b0ed2c9b5673c7326e5889b MD5 | raw file
  1#
  2#   irb/ruby-token.rb - ruby tokens 
  3#   	$Release Version: 0.9.5$
  4#   	$Revision: 11708 $
  5#   	$Date: 2007-02-13 08:01:19 +0900 (Tue, 13 Feb 2007) $
  6#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
  7#
  8# --
  9#
 10#   
 11#
 12module RubyToken
 13  EXPR_BEG = :EXPR_BEG
 14  EXPR_MID = :EXPR_MID
 15  EXPR_END = :EXPR_END
 16  EXPR_ARG = :EXPR_ARG
 17  EXPR_FNAME = :EXPR_FNAME
 18  EXPR_DOT = :EXPR_DOT
 19  EXPR_CLASS = :EXPR_CLASS
 20
 21  # for ruby 1.4X
 22  if !defined?(Symbol)
 23    Symbol = Integer
 24  end
 25  
 26  class Token
 27    def initialize(seek, line_no, char_no)
 28      @seek = seek
 29      @line_no = line_no
 30      @char_no = char_no
 31    end
 32    attr :seek
 33    attr :line_no
 34    attr :char_no
 35  end
 36
 37  class TkNode < Token
 38    def initialize(seek, line_no, char_no)
 39      super
 40    end
 41    attr :node
 42  end
 43
 44  class TkId < Token
 45    def initialize(seek, line_no, char_no, name)
 46      super(seek, line_no, char_no)
 47      @name = name
 48    end
 49    attr :name
 50  end
 51
 52  class TkVal < Token
 53    def initialize(seek, line_no, char_no, value = nil)
 54      super(seek, line_no, char_no)
 55      @value = value
 56    end
 57    attr :value
 58  end
 59
 60  class TkOp < Token
 61    attr :name, true
 62  end
 63
 64  class TkOPASGN < TkOp
 65    def initialize(seek, line_no, char_no, op)
 66      super(seek, line_no, char_no)
 67      op = TkReading2Token[op][0] unless op.kind_of?(Symbol)
 68      @op = op
 69    end
 70    attr :op
 71  end
 72
 73  class TkUnknownChar < Token
 74    def initialize(seek, line_no, char_no, id)
 75      super(seek, line_no, char_no)
 76      @name = name
 77    end
 78    attr :name
 79  end
 80
 81  class TkError < Token
 82  end
 83
 84  def Token(token, value = nil)
 85    case token
 86    when String
 87      if (tk = TkReading2Token[token]).nil?
 88	IRB.fail TkReading2TokenNoKey, token
 89      end
 90      tk = Token(tk[0], value) 
 91      if tk.kind_of?(TkOp)
 92	tk.name = token
 93      end
 94      return tk
 95    when Symbol
 96      if (tk = TkSymbol2Token[token]).nil?
 97	IRB.fail TkSymbol2TokenNoKey, token
 98      end
 99      return Token(tk[0], value) 
100    else 
101      if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty?
102	token.new(@prev_seek, @prev_line_no, @prev_char_no)
103      else
104	token.new(@prev_seek, @prev_line_no, @prev_char_no, value)
105      end
106    end
107  end
108
109  TokenDefinitions = [
110    [:TkCLASS,      TkId,  "class",  EXPR_CLASS],
111    [:TkMODULE,     TkId,  "module", EXPR_BEG],
112    [:TkDEF,	    TkId,  "def",    EXPR_FNAME],
113    [:TkUNDEF,      TkId,  "undef",  EXPR_FNAME],
114    [:TkBEGIN,      TkId,  "begin",  EXPR_BEG],
115    [:TkRESCUE,     TkId,  "rescue", EXPR_MID],
116    [:TkENSURE,     TkId,  "ensure", EXPR_BEG],
117    [:TkEND,	    TkId,  "end",    EXPR_END],
118    [:TkIF,         TkId,  "if",     EXPR_BEG, :TkIF_MOD],
119    [:TkUNLESS,     TkId,  "unless", EXPR_BEG, :TkUNLESS_MOD],
120    [:TkTHEN,	    TkId,  "then",   EXPR_BEG],
121    [:TkELSIF,      TkId,  "elsif",  EXPR_BEG],
122    [:TkELSE,	    TkId,  "else",   EXPR_BEG],
123    [:TkCASE,	    TkId,  "case",   EXPR_BEG],
124    [:TkWHEN,	    TkId,  "when",   EXPR_BEG],
125    [:TkWHILE,      TkId,  "while",  EXPR_BEG, :TkWHILE_MOD],
126    [:TkUNTIL,      TkId,  "until",  EXPR_BEG, :TkUNTIL_MOD],
127    [:TkFOR,	    TkId,  "for",    EXPR_BEG],
128    [:TkBREAK,      TkId,  "break",  EXPR_END],
129    [:TkNEXT,	    TkId,  "next",   EXPR_END],
130    [:TkREDO,	    TkId,  "redo",   EXPR_END],
131    [:TkRETRY,      TkId,  "retry",  EXPR_END],
132    [:TkIN,	    TkId,  "in",     EXPR_BEG],
133    [:TkDO,	    TkId,  "do",     EXPR_BEG],
134    [:TkRETURN,     TkId,  "return", EXPR_MID],
135    [:TkYIELD,      TkId,  "yield",  EXPR_END],
136    [:TkSUPER,      TkId,  "super",  EXPR_END],
137    [:TkSELF,	    TkId,  "self",   EXPR_END],
138    [:TkNIL, 	    TkId,  "nil",    EXPR_END],
139    [:TkTRUE,	    TkId,  "true",   EXPR_END],
140    [:TkFALSE,      TkId,  "false",  EXPR_END],
141    [:TkAND,	    TkId,  "and",    EXPR_BEG],
142    [:TkOR, 	    TkId,  "or",     EXPR_BEG],
143    [:TkNOT,	    TkId,  "not",    EXPR_BEG],
144    [:TkIF_MOD,     TkId],
145    [:TkUNLESS_MOD, TkId],
146    [:TkWHILE_MOD,  TkId],
147    [:TkUNTIL_MOD,  TkId],
148    [:TkALIAS,      TkId,  "alias",    EXPR_FNAME],
149    [:TkDEFINED,    TkId,  "defined?", EXPR_END],
150    [:TklBEGIN,     TkId,  "BEGIN",    EXPR_END],
151    [:TklEND,	    TkId,  "END",      EXPR_END],
152    [:Tk__LINE__,   TkId,  "__LINE__", EXPR_END],
153    [:Tk__FILE__,   TkId,  "__FILE__", EXPR_END],
154
155    [:TkIDENTIFIER, TkId],
156    [:TkFID,	    TkId],
157    [:TkGVAR,	    TkId],
158    [:TkCVAR,	    TkId],
159    [:TkIVAR,	    TkId],
160    [:TkCONSTANT,   TkId],
161
162    [:TkINTEGER,    TkVal],
163    [:TkFLOAT,      TkVal],
164    [:TkSTRING,     TkVal],
165    [:TkXSTRING,    TkVal],
166    [:TkREGEXP,     TkVal],
167    [:TkSYMBOL,     TkVal],
168
169    [:TkDSTRING,    TkNode],
170    [:TkDXSTRING,   TkNode],
171    [:TkDREGEXP,    TkNode],
172    [:TkNTH_REF,    TkNode],
173    [:TkBACK_REF,   TkNode],
174
175    [:TkUPLUS,      TkOp,   "+@"],
176    [:TkUMINUS,     TkOp,   "-@"],
177    [:TkPOW,	    TkOp,   "**"],
178    [:TkCMP,	    TkOp,   "<=>"],
179    [:TkEQ,	    TkOp,   "=="],
180    [:TkEQQ,	    TkOp,   "==="],
181    [:TkNEQ,	    TkOp,   "!="],
182    [:TkGEQ,	    TkOp,   ">="],
183    [:TkLEQ,	    TkOp,   "<="],
184    [:TkANDOP,      TkOp,   "&&"],
185    [:TkOROP,	    TkOp,   "||"],
186    [:TkMATCH,      TkOp,   "=~"],
187    [:TkNMATCH,     TkOp,   "!~"],
188    [:TkDOT2,	    TkOp,   ".."],
189    [:TkDOT3,	    TkOp,   "..."],
190    [:TkAREF,	    TkOp,   "[]"],
191    [:TkASET,	    TkOp,   "[]="],
192    [:TkLSHFT,      TkOp,   "<<"],
193    [:TkRSHFT,      TkOp,   ">>"],
194    [:TkCOLON2,     TkOp],
195    [:TkCOLON3,     TkOp],
196#   [:OPASGN,	    TkOp],               # +=, -=  etc. #
197    [:TkASSOC,      TkOp,   "=>"],
198    [:TkQUESTION,   TkOp,   "?"],	 #?
199    [:TkCOLON,      TkOp,   ":"],        #:
200    
201    [:TkfLPAREN],         # func( #
202    [:TkfLBRACK],         # func[ #
203    [:TkfLBRACE],         # func{ #
204    [:TkSTAR],            # *arg
205    [:TkAMPER],           # &arg #
206    [:TkSYMBEG],          # :SYMBOL
207
208    [:TkGT,	    TkOp,   ">"],
209    [:TkLT,	    TkOp,   "<"],
210    [:TkPLUS,	    TkOp,   "+"],
211    [:TkMINUS,      TkOp,   "-"],
212    [:TkMULT,	    TkOp,   "*"],
213    [:TkDIV,	    TkOp,   "/"],
214    [:TkMOD,	    TkOp,   "%"],
215    [:TkBITOR,      TkOp,   "|"],
216    [:TkBITXOR,     TkOp,   "^"],
217    [:TkBITAND,     TkOp,   "&"],
218    [:TkBITNOT,     TkOp,   "~"],
219    [:TkNOTOP,      TkOp,   "!"],
220
221    [:TkBACKQUOTE,  TkOp,   "`"],
222
223    [:TkASSIGN,     Token,  "="],
224    [:TkDOT,	    Token,  "."],
225    [:TkLPAREN,     Token,  "("],  #(exp)
226    [:TkLBRACK,     Token,  "["],  #[arry]
227    [:TkLBRACE,     Token,  "{"],  #{hash}
228    [:TkRPAREN,     Token,  ")"],
229    [:TkRBRACK,     Token,  "]"],
230    [:TkRBRACE,     Token,  "}"],
231    [:TkCOMMA,      Token,  ","],
232    [:TkSEMICOLON,  Token,  ";"],
233
234    [:TkCOMMENT],
235    [:TkRD_COMMENT],
236    [:TkSPACE],
237    [:TkNL],
238    [:TkEND_OF_SCRIPT],
239
240    [:TkBACKSLASH,  TkUnknownChar,  "\\"],
241    [:TkAT,	    TkUnknownChar,  "@"],
242    [:TkDOLLAR,     TkUnknownChar,  "$"],
243  ]
244
245  # {reading => token_class}
246  # {reading => [token_class, *opt]}
247  TkReading2Token = {}
248  TkSymbol2Token = {}
249
250  def RubyToken.def_token(token_n, super_token = Token, reading = nil, *opts)
251    token_n = token_n.id2name if token_n.kind_of?(Symbol)
252    if RubyToken.const_defined?(token_n)
253      IRB.fail AlreadyDefinedToken, token_n
254    end
255    token_c = eval("class #{token_n} < #{super_token}; end; #{token_n}")
256    
257    if reading
258      if TkReading2Token[reading]
259	IRB.fail TkReading2TokenDuplicateError, token_n, reading
260      end
261      if opts.empty?
262	TkReading2Token[reading] = [token_c]
263      else
264	TkReading2Token[reading] = [token_c].concat(opts)
265      end
266    end
267    TkSymbol2Token[token_n.intern] = token_c
268  end
269
270  for defs in TokenDefinitions
271    def_token(*defs)
272  end
273end