/scalate-jruby/src/main/resources/haml-3.0.25/lib/sass/script/lexer.rb

http://github.com/scalate/scalate · Ruby · 337 lines · 240 code · 37 blank · 60 comment · 33 complexity · 134a3b05be0e749f9b41280bdc15ed9d MD5 · raw file

  1. require 'sass/scss/rx'
  2. require 'strscan'
  3. module Sass
  4. module Script
  5. # The lexical analyzer for SassScript.
  6. # It takes a raw string and converts it to individual tokens
  7. # that are easier to parse.
  8. class Lexer
  9. include Sass::SCSS::RX
  10. # A struct containing information about an individual token.
  11. #
  12. # `type`: \[`Symbol`\]
  13. # : The type of token.
  14. #
  15. # `value`: \[`Object`\]
  16. # : The Ruby object corresponding to the value of the token.
  17. #
  18. # `line`: \[`Fixnum`\]
  19. # : The line of the source file on which the token appears.
  20. #
  21. # `offset`: \[`Fixnum`\]
  22. # : The number of bytes into the line the SassScript token appeared.
  23. #
  24. # `pos`: \[`Fixnum`\]
  25. # : The scanner position at which the SassScript token appeared.
  26. Token = Struct.new(:type, :value, :line, :offset, :pos)
  27. # The line number of the lexer's current position.
  28. #
  29. # @return [Fixnum]
  30. attr_reader :line
  31. # The number of bytes into the current line
  32. # of the lexer's current position.
  33. #
  34. # @return [Fixnum]
  35. attr_reader :offset
  36. # A hash from operator strings to the corresponding token types.
  37. OPERATORS = {
  38. '+' => :plus,
  39. '-' => :minus,
  40. '*' => :times,
  41. '/' => :div,
  42. '%' => :mod,
  43. '=' => :single_eq,
  44. ':' => :colon,
  45. '(' => :lparen,
  46. ')' => :rparen,
  47. ',' => :comma,
  48. 'and' => :and,
  49. 'or' => :or,
  50. 'not' => :not,
  51. '==' => :eq,
  52. '!=' => :neq,
  53. '>=' => :gte,
  54. '<=' => :lte,
  55. '>' => :gt,
  56. '<' => :lt,
  57. '#{' => :begin_interpolation,
  58. '}' => :end_interpolation,
  59. ';' => :semicolon,
  60. '{' => :lcurly,
  61. }
  62. OPERATORS_REVERSE = Haml::Util.map_hash(OPERATORS) {|k, v| [v, k]}
  63. TOKEN_NAMES = Haml::Util.map_hash(OPERATORS_REVERSE) {|k, v| [k, v.inspect]}.merge({
  64. :const => "variable (e.g. $foo)",
  65. :ident => "identifier (e.g. middle)",
  66. :bool => "boolean (e.g. true, false)",
  67. })
  68. # A list of operator strings ordered with longer names first
  69. # so that `>` and `<` don't clobber `>=` and `<=`.
  70. OP_NAMES = OPERATORS.keys.sort_by {|o| -o.size}
  71. # A sub-list of {OP_NAMES} that only includes operators
  72. # with identifier names.
  73. IDENT_OP_NAMES = OP_NAMES.select {|k, v| k =~ /^\w+/}
  74. # A hash of regular expressions that are used for tokenizing.
  75. REGULAR_EXPRESSIONS = {
  76. :whitespace => /\s+/,
  77. :comment => COMMENT,
  78. :single_line_comment => SINGLE_LINE_COMMENT,
  79. :variable => /([!\$])(#{IDENT})/,
  80. :ident => /(#{IDENT})(\()?/,
  81. :number => /(-)?(?:(\d*\.\d+)|(\d+))([a-zA-Z%]+)?/,
  82. :color => HEXCOLOR,
  83. :bool => /(true|false)\b/,
  84. :ident_op => %r{(#{Regexp.union(*IDENT_OP_NAMES.map{|s| Regexp.new(Regexp.escape(s) + "(?!#{NMCHAR}|\Z)")})})},
  85. :op => %r{(#{Regexp.union(*OP_NAMES)})},
  86. }
  87. class << self
  88. private
  89. def string_re(open, close)
  90. /#{open}((?:\\.|\#(?!\{)|[^#{close}\\#])*)(#{close}|#\{)/
  91. end
  92. end
  93. # A hash of regular expressions that are used for tokenizing strings.
  94. #
  95. # The key is a `[Symbol, Boolean]` pair.
  96. # The symbol represents which style of quotation to use,
  97. # while the boolean represents whether or not the string
  98. # is following an interpolated segment.
  99. STRING_REGULAR_EXPRESSIONS = {
  100. [:double, false] => string_re('"', '"'),
  101. [:single, false] => string_re("'", "'"),
  102. [:double, true] => string_re('', '"'),
  103. [:single, true] => string_re('', "'"),
  104. [:uri, false] => /url\(#{W}(#{URLCHAR}*?)(#{W}\)|#\{)/,
  105. [:uri, true] => /(#{URLCHAR}*?)(#{W}\)|#\{)/,
  106. }
  107. # @param str [String, StringScanner] The source text to lex
  108. # @param line [Fixnum] The line on which the SassScript appears.
  109. # Used for error reporting
  110. # @param offset [Fixnum] The number of characters in on which the SassScript appears.
  111. # Used for error reporting
  112. # @param options [{Symbol => Object}] An options hash;
  113. # see {file:SASS_REFERENCE.md#sass_options the Sass options documentation}
  114. def initialize(str, line, offset, options)
  115. @scanner = str.is_a?(StringScanner) ? str : StringScanner.new(str)
  116. @line = line
  117. @offset = offset
  118. @options = options
  119. @interpolation_stack = []
  120. @prev = nil
  121. end
  122. # Moves the lexer forward one token.
  123. #
  124. # @return [Token] The token that was moved past
  125. def next
  126. @tok ||= read_token
  127. @tok, tok = nil, @tok
  128. @prev = tok
  129. return tok
  130. end
  131. # Returns whether or not there's whitespace before the next token.
  132. #
  133. # @return [Boolean]
  134. def whitespace?(tok = @tok)
  135. if tok
  136. @scanner.string[0...tok.pos] =~ /\s\Z/
  137. else
  138. @scanner.string[@scanner.pos, 1] =~ /^\s/ ||
  139. @scanner.string[@scanner.pos - 1, 1] =~ /\s\Z/
  140. end
  141. end
  142. # Returns the next token without moving the lexer forward.
  143. #
  144. # @return [Token] The next token
  145. def peek
  146. @tok ||= read_token
  147. end
  148. # Rewinds the underlying StringScanner
  149. # to before the token returned by \{#peek}.
  150. def unpeek!
  151. @scanner.pos = @tok.pos if @tok
  152. end
  153. # @return [Boolean] Whether or not there's more source text to lex.
  154. def done?
  155. whitespace unless after_interpolation? && @interpolation_stack.last
  156. @scanner.eos? && @tok.nil?
  157. end
  158. # @return [Boolean] Whether or not the last token lexed was `:end_interpolation`.
  159. def after_interpolation?
  160. @prev && @prev.type == :end_interpolation
  161. end
  162. # Raise an error to the effect that `name` was expected in the input stream
  163. # and wasn't found.
  164. #
  165. # This calls \{#unpeek!} to rewind the scanner to immediately after
  166. # the last returned token.
  167. #
  168. # @param name [String] The name of the entity that was expected but not found
  169. # @raise [Sass::SyntaxError]
  170. def expected!(name)
  171. unpeek!
  172. Sass::SCSS::Parser.expected(@scanner, name, @line)
  173. end
  174. # Records all non-comment text the lexer consumes within the block
  175. # and returns it as a string.
  176. #
  177. # @yield A block in which text is recorded
  178. # @return [String]
  179. def str
  180. old_pos = @tok ? @tok.pos : @scanner.pos
  181. yield
  182. new_pos = @tok ? @tok.pos : @scanner.pos
  183. @scanner.string[old_pos...new_pos]
  184. end
  185. private
  186. def read_token
  187. return if done?
  188. return unless value = token
  189. type, val, size = value
  190. size ||= @scanner.matched_size
  191. val.line = @line if val.is_a?(Script::Node)
  192. Token.new(type, val, @line,
  193. current_position - size, @scanner.pos - size)
  194. end
  195. def whitespace
  196. nil while scan(REGULAR_EXPRESSIONS[:whitespace]) ||
  197. scan(REGULAR_EXPRESSIONS[:comment]) ||
  198. scan(REGULAR_EXPRESSIONS[:single_line_comment])
  199. end
  200. def token
  201. if after_interpolation? && (interp_type = @interpolation_stack.pop)
  202. return string(interp_type, true)
  203. end
  204. variable || string(:double, false) || string(:single, false) || number ||
  205. color || bool || string(:uri, false) || raw(UNICODERANGE) ||
  206. special_fun || ident_op || ident || op
  207. end
  208. def variable
  209. _variable(REGULAR_EXPRESSIONS[:variable])
  210. end
  211. def _variable(rx)
  212. line = @line
  213. offset = @offset
  214. return unless scan(rx)
  215. if @scanner[1] == '!' && @scanner[2] != 'important'
  216. Script.var_warning(@scanner[2], line, offset + 1, @options[:filename])
  217. end
  218. [:const, @scanner[2]]
  219. end
  220. def ident
  221. return unless scan(REGULAR_EXPRESSIONS[:ident])
  222. [@scanner[2] ? :funcall : :ident, @scanner[1]]
  223. end
  224. def string(re, open)
  225. return unless scan(STRING_REGULAR_EXPRESSIONS[[re, open]])
  226. if @scanner[2] == '#{' #'
  227. @scanner.pos -= 2 # Don't actually consume the #{
  228. @interpolation_stack << re
  229. end
  230. str =
  231. if re == :uri
  232. Script::String.new("#{'url(' unless open}#{@scanner[1]}#{')' unless @scanner[2] == '#{'}")
  233. else
  234. Script::String.new(@scanner[1].gsub(/\\(['"]|\#\{)/, '\1'), :string)
  235. end
  236. [:string, str]
  237. end
  238. def number
  239. return unless scan(REGULAR_EXPRESSIONS[:number])
  240. value = @scanner[2] ? @scanner[2].to_f : @scanner[3].to_i
  241. value = -value if @scanner[1]
  242. [:number, Script::Number.new(value, Array(@scanner[4]))]
  243. end
  244. def color
  245. return unless s = scan(REGULAR_EXPRESSIONS[:color])
  246. raise Sass::SyntaxError.new(<<MESSAGE.rstrip) unless s.size == 4 || s.size == 7
  247. Colors must have either three or six digits: '#{s}'
  248. MESSAGE
  249. value = s.scan(/^#(..?)(..?)(..?)$/).first.
  250. map {|num| num.ljust(2, num).to_i(16)}
  251. [:color, Script::Color.new(value)]
  252. end
  253. def bool
  254. return unless s = scan(REGULAR_EXPRESSIONS[:bool])
  255. [:bool, Script::Bool.new(s == 'true')]
  256. end
  257. def special_fun
  258. return unless str1 = scan(/((-[\w-]+-)?calc|expression|progid:[a-z\.]*)\(/i)
  259. str2, _ = Haml::Shared.balance(@scanner, ?(, ?), 1)
  260. c = str2.count("\n")
  261. old_line = @line
  262. old_offset = @offset
  263. @line += c
  264. @offset = (c == 0 ? @offset + str2.size : str2[/\n(.*)/, 1].size)
  265. [:special_fun,
  266. Haml::Util.merge_adjacent_strings(
  267. [str1] + Sass::Engine.parse_interp(str2, old_line, old_offset, @options)),
  268. str1.size + str2.size]
  269. end
  270. def ident_op
  271. return unless op = scan(REGULAR_EXPRESSIONS[:ident_op])
  272. [OPERATORS[op]]
  273. end
  274. def op
  275. return unless op = scan(REGULAR_EXPRESSIONS[:op])
  276. @interpolation_stack << nil if op == :begin_interpolation
  277. [OPERATORS[op]]
  278. end
  279. def raw(rx)
  280. return unless val = scan(rx)
  281. [:raw, val]
  282. end
  283. def scan(re)
  284. return unless str = @scanner.scan(re)
  285. c = str.count("\n")
  286. @line += c
  287. @offset = (c == 0 ? @offset + str.size : str[/\n(.*)/, 1].size)
  288. str
  289. end
  290. def current_position
  291. @offset + 1
  292. end
  293. end
  294. end
  295. end