/lib/twostroke/tokens.rb

https://github.com/filp/twostroke · Ruby · 89 lines · 85 code · 4 blank · 0 comment · 6 complexity · d379a4384104c57a6b193527edc923bb MD5 · raw file

  1. module Twostroke
  2. class Lexer
  3. RESERVED = %w(function var if instanceof in else for while do this return throw typeof try catch finally void null new delete switch case break continue default true false)
  4. TOKENS = [
  5. [ :MULTI_COMMENT, %r{/\*.*?\*/} ],
  6. [ :SINGLE_COMMENT, /\/\/.*?$/ ],
  7. [ :WHITESPACE, /\s+/ ],
  8. [ :NUMBER, /((?<oct>0[0-7]+)|(?<hex>0x[A-Fa-f0-9]+)|(?<to_f>(\d+(\.?\d*([eE][+-]?\d+)?)?|\.\d+([eE][+-]?\d+)?)))/, ->m do
  9. method, number = m.names.zip(m.captures).select { |k,v| v }.first
  10. n = number.send method
  11. if (n % 1).zero?
  12. n.to_i
  13. else
  14. n
  15. end
  16. end ],
  17. *RESERVED.map do |w|
  18. [ w.upcase.intern, /#{w}(?=[^a-zA-Z_0-9])/ ]
  19. end,
  20. [ :BAREWORD, /[a-zA-Z_\$][\$a-zA-Z_0-9]*/, ->m { m[0] } ],
  21. [ :STRING, /(["'])((\\.|[^\1])*?[^\1\\]?)\1/, ->m do
  22. m[2].gsub(/\\([bfnrt])/) { |m|
  23. case m[1]
  24. when "b"; "\b"
  25. when "n"; "\n"
  26. when "f"; "\f"
  27. when "r"; "\r"
  28. when "t"; "\t"
  29. end
  30. }
  31. .gsub(/\\([0-6]{1,3})/) { |m| m[1].to_i(7).chr }
  32. .gsub(/\\x([a-f0-9]{2})/i) { |m| m[1].to_i(16).chr }
  33. .gsub(/\\u([a-f0-9]{4})/i) { |m| m[1].to_i(16).chr }
  34. .gsub(/\\(.)/) { |m| m[1] }
  35. end ],
  36. [ :REGEXP, %r{/(?<src>(\\.|[^\1])*?[^\1\\]?)/(?<opts>[gim]+)?}, ->m { [m[:src], m[:opts]] } ],
  37. [ :OPEN_PAREN, /\(/ ],
  38. [ :CLOSE_PAREN, /\)/ ],
  39. [ :OPEN_BRACKET, /\[/ ],
  40. [ :CLOSE_BRACKET, /\]/ ],
  41. [ :OPEN_BRACE, /\{/ ],
  42. [ :CLOSE_BRACE, /\}/ ],
  43. [ :MEMBER_ACCESS, /\./ ],
  44. [ :INCREMENT, /\+\+/ ],
  45. [ :DECREMENT, /--/ ],
  46. [ :PLUS, /\+/ ],
  47. [ :MINUS, /-/ ],
  48. [ :ASTERISK, /\*/ ],
  49. [ :SLASH, /\// ],
  50. [ :MOD, /%/ ],
  51. [ :QUESTION, /\?/ ],
  52. [ :COMMA, /,/ ],
  53. [ :SEMICOLON, /;/ ],
  54. [ :COLON, /:/ ],
  55. [ :AND, /&&/ ],
  56. [ :AMPERSAND, /&/ ],
  57. [ :OR, /\|\|/ ],
  58. [ :PIPE, /\|/ ],
  59. [ :TRIPLE_EQUALS, /===/ ],
  60. [ :DOUBLE_EQUALS, /==/ ],
  61. [ :EQUALS, /=/ ],
  62. [ :NOT_DOUBLE_EQUALS, /!==/ ],
  63. [ :NOT_EQUALS, /!=/ ],
  64. [ :NOT, /!/ ],
  65. [ :TILDE, /~/ ],
  66. [ :CARET, /\^/ ],
  67. [ :LEFT_SHIFT, /<</ ],
  68. [ :RIGHT_TRIPLE_SHIFT, />>>/ ],
  69. [ :RIGHT_SHIFT, />>/ ],
  70. [ :LTE, /<=/ ],
  71. [ :GTE, />=/ ],
  72. [ :LT, /</ ],
  73. [ :GT, />/ ],
  74. ].map do |a|
  75. [a[0], Regexp.new("\\A#{a[1].source}", Regexp::MULTILINE), a[2]]
  76. end
  77. end
  78. end