Class Sass::Script::Lexer
In: lib/sass/script/lexer.rb
Parent: Object
Haml::Util Engine Color SyntaxError UnitConversionError StandardError AbstractSequence CommaSequence Sequence SimpleSequence Simple Parent Universal Class SelectorPseudoClass Id Pseudo Attribute Interpolation Element Node Operation Literal UnaryOperation StringInterpolation Funcall Interpolation Variable Lexer CssLexer Number Bool String Parser Parser CssParser EvaluationContext SassParser StaticParser CssParser Node DebugNode IfNode CommentNode ForNode PropNode MixinNode DirectiveNode VariableNode RootNode ExtendNode WarnNode WhileNode RuleNode MixinDefNode Enumerable ImportNode Merb::BootLoader MerbBootLoader Repl CSS Environment Rack StalenessChecker lib/sass/repl.rb lib/sass/css.rb lib/sass/environment.rb lib/sass/error.rb lib/sass/engine.rb lib/sass/selector/simple_sequence.rb lib/sass/selector/abstract_sequence.rb lib/sass/selector/sequence.rb lib/sass/selector/comma_sequence.rb lib/sass/selector/simple.rb lib/sass/selector.rb Selector lib/sass/script/css_parser.rb lib/sass/script/lexer.rb lib/sass/script/color.rb lib/sass/script/string.rb lib/sass/script/unary_operation.rb lib/sass/script/variable.rb lib/sass/script/funcall.rb lib/sass/script/string_interpolation.rb lib/sass/script/operation.rb lib/sass/script/bool.rb lib/sass/script/parser.rb lib/sass/script/node.rb lib/sass/script/literal.rb lib/sass/script/interpolation.rb lib/sass/script/css_lexer.rb lib/sass/script/number.rb lib/sass/script/functions.rb Functions Script lib/sass/scss/sass_parser.rb lib/sass/scss/static_parser.rb lib/sass/scss/parser.rb lib/sass/scss/css_parser.rb ScriptLexer ScriptParser RX SCSS Files Callbacks lib/sass/tree/while_node.rb lib/sass/tree/if_node.rb lib/sass/tree/mixin_def_node.rb lib/sass/tree/debug_node.rb lib/sass/tree/root_node.rb lib/sass/tree/for_node.rb lib/sass/tree/import_node.rb lib/sass/tree/prop_node.rb lib/sass/tree/node.rb lib/sass/tree/comment_node.rb lib/sass/tree/extend_node.rb lib/sass/tree/mixin_node.rb lib/sass/tree/warn_node.rb lib/sass/tree/directive_node.rb lib/sass/tree/rule_node.rb lib/sass/tree/variable_node.rb Tree lib/sass/plugin/rack.rb lib/sass/plugin/staleness_checker.rb lib/sass/plugin/merb.rb Plugin Sass dot/m_63_0.png

The lexical analyzer for SassScript. It takes a raw string and converts it to individual tokens that are easier to parse.

Methods

_variable   after_interpolation?   done?   expected!   ident   new   next   peek   read_token   str   string   string_re   token   unpeek!   variable   whitespace   whitespace?  

Included Modules

Sass::SCSS::RX

Constants

Token = Struct.new(:type, :value, :line, :offset, :pos)   A struct containing information about an individual token.

`type`: \[`Symbol`\] : The type of token.

`value`: \[`Object`\] : The Ruby object corresponding to the value of the token.

`line`: \[`Fixnum`\] : The line of the source file on which the token appears.

`offset`: \[`Fixnum`\] : The number of bytes into the line the SassScript token appeared.

`pos`: \[`Fixnum`\] : The scanner position at which the SassScript token appeared.

OPERATORS = { '+' => :plus, '-' => :minus, '*' => :times, '/' => :div, '%' => :mod, '=' => :single_eq, ':' => :colon, '(' => :lparen, ')' => :rparen, ',' => :comma, 'and' => :and, 'or' => :or, 'not' => :not, '==' => :eq, '!=' => :neq, '>=' => :gte, '<=' => :lte, '>' => :gt, '<' => :lt, '#{' => :begin_interpolation, '}' => :end_interpolation, ';' => :semicolon, '{' => :lcurly, }   A hash from operator strings to the corresponding token types.
OPERATORS_REVERSE = Haml::Util.map_hash(OPERATORS) {|k, v| [v, k]}
TOKEN_NAMES = Haml::Util.map_hash(OPERATORS_REVERSE) {|k, v| [k, v.inspect]}.merge({ :const => "variable (e.g. $foo)", :ident => "identifier (e.g. middle)", :bool => "boolean (e.g. true, false)", })
OP_NAMES = OPERATORS.keys.sort_by {|o| -o.size}   A list of operator strings ordered with longer names first so that `>` and `<` don‘t clobber `>=` and `<=`.
IDENT_OP_NAMES = OP_NAMES.select {|k, v| k =~ /^\w+/}   A sub-list of {OP_NAMES} that only includes operators with identifier names.
REGULAR_EXPRESSIONS = { :whitespace => /\s+/, :comment => COMMENT, :single_line_comment => SINGLE_LINE_COMMENT, :variable => /([!\$])(#{IDENT})/, :ident => /(#{IDENT})(\()?/, :number => /(-)?(?:(\d*\.\d+)|(\d+))([a-zA-Z%]+)?/, :color => HEXCOLOR, :bool => /(true|false)\b/, :ident_op => %r{(#{Regexp.union(*IDENT_OP_NAMES.map{|s| Regexp.new(Regexp.escape(s) + "(?!#{NMCHAR}|$)")})})}, :op => %r{(#{Regexp.union(*OP_NAMES)})}, }   A hash of regular expressions that are used for tokenizing.
STRING_REGULAR_EXPRESSIONS = { [:double, false] => string_re('"', '"'), [:single, false] => string_re("'", "'"), [:double, true] => string_re('', '"'), [:single, true] => string_re('', "'"), [:uri, false] => /url\(#{W}(#{URLCHAR}*?)(#{W}\)|#\{)/, [:uri, true] => /(#{URLCHAR}*?)(#{W}\)|#\{)/, }   A hash of regular expressions that are used for tokenizing strings.

The key is a `[Symbol, Boolean]` pair. The symbol represents which style of quotation to use, while the boolean represents whether or not the string is following an interpolated segment.

Attributes

line  [R]  The line number of the lexer‘s current position.

@return [Fixnum]

offset  [R]  The number of bytes into the current line of the lexer‘s current position.

@return [Fixnum]

Public Class methods

@param str [String, StringScanner] The source text to lex @param line [Fixnum] The line on which the SassScript appears.

  Used for error reporting

@param offset [Fixnum] The number of characters in on which the SassScript appears.

  Used for error reporting

@param options [{Symbol => Object}] An options hash;

  see {file:SASS_REFERENCE.md#sass_options the Sass options documentation}

[Source]

     # File lib/sass/script/lexer.rb, line 128
128:       def initialize(str, line, offset, options)
129:         @scanner = str.is_a?(StringScanner) ? str : StringScanner.new(str)
130:         @line = line
131:         @offset = offset
132:         @options = options
133:         @interpolation_stack = []
134:         @prev = nil
135:       end

Private Class methods

[Source]

     # File lib/sass/script/lexer.rb, line 101
101:         def string_re(open, close)
102:           /#{open}((?:\\.|\#(?!\{)|[^#{close}\\#])*)(#{close}|#\{)/
103:         end

Public Instance methods

@return [Boolean] Whether or not the last token lexed was `:end_interpolation`.

[Source]

     # File lib/sass/script/lexer.rb, line 179
179:       def after_interpolation?
180:         @prev && @prev.type == :end_interpolation
181:       end

@return [Boolean] Whether or not there‘s more source text to lex.

[Source]

     # File lib/sass/script/lexer.rb, line 173
173:       def done?
174:         whitespace unless after_interpolation? && @interpolation_stack.last
175:         @scanner.eos? && @tok.nil?
176:       end

Raise an error to the effect that `name` was expected in the input stream and wasn‘t found.

This calls \{unpeek!} to rewind the scanner to immediately after the last returned token.

@param name [String] The name of the entity that was expected but not found @raise [Sass::SyntaxError]

[Source]

     # File lib/sass/script/lexer.rb, line 191
191:       def expected!(name)
192:         unpeek!
193:         Sass::SCSS::Parser.expected(@scanner, name, @line)
194:       end

Moves the lexer forward one token.

@return [Token] The token that was moved past

[Source]

     # File lib/sass/script/lexer.rb, line 140
140:       def next
141:         @tok ||= read_token
142:         @tok, tok = nil, @tok
143:         @prev = tok
144:         return tok
145:       end

Returns the next token without moving the lexer forward.

@return [Token] The next token

[Source]

     # File lib/sass/script/lexer.rb, line 162
162:       def peek
163:         @tok ||= read_token
164:       end

Records all non-comment text the lexer consumes within the block and returns it as a string.

@yield A block in which text is recorded @return [String]

[Source]

     # File lib/sass/script/lexer.rb, line 201
201:       def str
202:         old_pos = @tok ? @tok.pos : @scanner.pos
203:         yield
204:         new_pos = @tok ? @tok.pos : @scanner.pos
205:         @scanner.string[old_pos...new_pos]
206:       end

Rewinds the underlying StringScanner to before the token returned by \{peek}.

[Source]

     # File lib/sass/script/lexer.rb, line 168
168:       def unpeek!
169:         @scanner.pos = @tok.pos if @tok
170:       end

Returns whether or not there‘s whitespace before the next token.

@return [Boolean]

[Source]

     # File lib/sass/script/lexer.rb, line 150
150:       def whitespace?(tok = @tok)
151:         if tok
152:           @scanner.string[0...tok.pos] =~ /\s\Z/
153:         else
154:           @scanner.string[@scanner.pos, 1] =~ /^\s/ ||
155:             @scanner.string[@scanner.pos - 1, 1] =~ /\s\Z/
156:         end
157:       end

Private Instance methods

[Source]

     # File lib/sass/script/lexer.rb, line 241
241:       def _variable(rx)
242:         line = @line
243:         offset = @offset
244:         return unless scan(rx)
245:         if @scanner[1] == '!' && @scanner[2] != 'important'
246:           Script.var_warning(@scanner[2], line, offset + 1, @options[:filename])
247:         end
248: 
249:         [:const, @scanner[2]]
250:       end

[Source]

     # File lib/sass/script/lexer.rb, line 252
252:       def ident
253:         return unless scan(REGULAR_EXPRESSIONS[:ident])
254:         [@scanner[2] ? :funcall : :ident, @scanner[1]]
255:       end

[Source]

     # File lib/sass/script/lexer.rb, line 210
210:       def read_token
211:         return if done?
212:         return unless value = token
213:         type, val, size = value
214:         size ||= @scanner.matched_size
215: 
216:         val.line = @line if val.is_a?(Script::Node)
217:         Token.new(type, val, @line,
218:           current_position - size, @scanner.pos - size)
219:       end

[Source]

     # File lib/sass/script/lexer.rb, line 257
257:       def string(re, open)
258:         return unless scan(STRING_REGULAR_EXPRESSIONS[[re, open]])
259:         if @scanner[2] == '#{' #'
260:           @scanner.pos -= 2 # Don't actually consume the #{
261:           @interpolation_stack << re
262:         end
263:         str =
264:           if re == :uri
265:             Script::String.new("#{'url(' unless open}#{@scanner[1]}#{')' unless @scanner[2] == '#{'}")
266:           else
267:             Script::String.new(@scanner[1].gsub(/\\(['"]|\#\{)/, '\1'), :string)
268:           end
269:         [:string, str]
270:       end
271: 
272:       def number
273:         return unless scan(REGULAR_EXPRESSIONS[:number])
274:         value = @scanner[2] ? @scanner[2].to_f : @scanner[3].to_i
275:         value = -value if @scanner[1]
276:         [:number, Script::Number.new(value, Array(@scanner[4]))]
277:       end
278: 
279:       def color
280:         return unless s = scan(REGULAR_EXPRESSIONS[:color])
281:         raise Sass::SyntaxError.new(<<MESSAGE) unless s.size == 4 || s.size == 7
282: Colors must have either three or six digits: '#{s}'
283: MESSAGE
284:         value = s.scan(/^#(..?)(..?)(..?)$/).first.
285:           map {|num| num.ljust(2, num).to_i(16)}
286:         [:color, Script::Color.new(value)]
287:       end
288: 
289:       def bool
290:         return unless s = scan(REGULAR_EXPRESSIONS[:bool])
291:         [:bool, Script::Bool.new(s == 'true')]
292:       end
293: 
294:       def special_fun
295:         return unless str1 = scan(/((-[\w-]+-)?calc|expression|progid:[a-z\.]*)\(/i)
296:         str2, _ = Haml::Shared.balance(@scanner, ?(, ?), 1)
297:         c = str2.count("\n")
298:         old_line = @line
299:         old_offset = @offset
300:         @line += c
301:         @offset = (c == 0 ? @offset + str2.size : str2[/\n(.*)/, 1].size)
302:         [:special_fun,
303:           Haml::Util.merge_adjacent_strings(
304:             [str1] + Sass::Engine.parse_interp(str2, old_line, old_offset, @options)),
305:           str1.size + str2.size]
306:       end
307: 
308:       def ident_op
309:         return unless op = scan(REGULAR_EXPRESSIONS[:ident_op])
310:         [OPERATORS[op]]
311:       end
312: 
313:       def op
314:         return unless op = scan(REGULAR_EXPRESSIONS[:op])
315:         @interpolation_stack << nil if op == :begin_interpolation
316:         [OPERATORS[op]]
317:       end
318: 
319:       def raw(rx)
320:         return unless val = scan(rx)
321:         [:raw, val]
322:       end
323: 
324:       def scan(re)
325:         return unless str = @scanner.scan(re)
326:         c = str.count("\n")
327:         @line += c
328:         @offset = (c == 0 ? @offset + str.size : str[/\n(.*)/, 1].size)
329:         str
330:       end
331: 
332:       def current_position
333:         @offset + 1
334:       end
335:     end
336:   end
337: end
338: 

[Source]

     # File lib/sass/script/lexer.rb, line 227
227:       def token
228:         if after_interpolation? && (interp_type = @interpolation_stack.pop)
229:           return string(interp_type, true)
230:         end
231: 
232:         variable || string(:double, false) || string(:single, false) || number ||
233:           color || bool || string(:uri, false) || raw(UNICODERANGE) ||
234:           special_fun || ident_op || ident || op
235:       end

[Source]

     # File lib/sass/script/lexer.rb, line 237
237:       def variable
238:         _variable(REGULAR_EXPRESSIONS[:variable])
239:       end

[Source]

     # File lib/sass/script/lexer.rb, line 221
221:       def whitespace
222:         nil while scan(REGULAR_EXPRESSIONS[:whitespace]) ||
223:           scan(REGULAR_EXPRESSIONS[:comment]) ||
224:           scan(REGULAR_EXPRESSIONS[:single_line_comment])
225:       end

[Validate]