diff --git a/lib/lrama/lexer.rb b/lib/lrama/lexer.rb index e22eb235..bb89801b 100644 --- a/lib/lrama/lexer.rb +++ b/lib/lrama/lexer.rb @@ -1,85 +1,12 @@ require "strscan" require "lrama/report" +require "lrama/lexer/token" module Lrama # Lexer for parse.y class Lexer include Lrama::Report::Duration - # s_value is semantic value - Token = Struct.new(:type, :s_value, :alias, keyword_init: true) do - Type = Struct.new(:id, :name, keyword_init: true) - - attr_accessor :line, :column, :referred - # For User_code - attr_accessor :references - - def to_s - "#{super} line: #{line}, column: #{column}" - end - - def referred_by?(string) - [self.s_value, self.alias].include?(string) - end - - def ==(other) - self.class == other.class && self.type == other.type && self.s_value == other.s_value - end - - def numberize_references(lhs, rhs) - self.references.map! {|ref| - ref_name = ref[1] - if ref_name.is_a?(String) && ref_name != '$' - value = - if lhs.referred_by?(ref_name) - '$' - else - rhs.find_index {|token| token.referred_by?(ref_name) } + 1 - end - [ref[0], value, ref[2], ref[3], ref[4]] - else - ref - end - } - end - - @i = 0 - @types = [] - - def self.define_type(name) - type = Type.new(id: @i, name: name.to_s) - const_set(name, type) - @types << type - @i += 1 - end - - # Token types - define_type(:P_expect) # %expect - define_type(:P_define) # %define - define_type(:P_printer) # %printer - define_type(:P_error_token) # %error-token - define_type(:P_lex_param) # %lex-param - define_type(:P_parse_param) # %parse-param - define_type(:P_initial_action) # %initial-action - define_type(:P_union) # %union - define_type(:P_token) # %token - define_type(:P_type) # %type - define_type(:P_nonassoc) # %nonassoc - define_type(:P_left) # %left - define_type(:P_right) # %right - define_type(:P_prec) # %prec - define_type(:User_code) # { ... } - define_type(:Tag) # - define_type(:Number) # 0 - define_type(:Ident_Colon) # k_if:, k_if : (spaces can be there) - define_type(:Ident) # api.pure, tNUMBER - define_type(:Named_Ref) # [foo] - define_type(:Semicolon) # ; - define_type(:Bar) # | - define_type(:String) # "str" - define_type(:Char) # '+' - end - # States # # See: https://www.gnu.org/software/bison/manual/html_node/Grammar-Outline.html diff --git a/lib/lrama/lexer/token.rb b/lib/lrama/lexer/token.rb new file mode 100644 index 00000000..29ce48b2 --- /dev/null +++ b/lib/lrama/lexer/token.rb @@ -0,0 +1,76 @@ +module Lrama + class Lexer + class Token < Struct.new(:type, :s_value, :alias, keyword_init: true) + Type = Struct.new(:id, :name, keyword_init: true) + + attr_accessor :line, :column, :referred + # For User_code + attr_accessor :references + + def to_s + "#{super} line: #{line}, column: #{column}" + end + + def referred_by?(string) + [self.s_value, self.alias].include?(string) + end + + def ==(other) + self.class == other.class && self.type == other.type && self.s_value == other.s_value + end + + def numberize_references(lhs, rhs) + self.references.map! {|ref| + ref_name = ref[1] + if ref_name.is_a?(::String) && ref_name != '$' + value = + if lhs.referred_by?(ref_name) + '$' + else + rhs.find_index {|token| token.referred_by?(ref_name) } + 1 + end + [ref[0], value, ref[2], ref[3], ref[4]] + else + ref + end + } + end + + @i = 0 + @types = [] + + def self.define_type(name) + type = Type.new(id: @i, name: name.to_s) + const_set(name, type) + @types << type + @i += 1 + end + + # Token types + define_type(:P_expect) # %expect + define_type(:P_define) # %define + define_type(:P_printer) # %printer + define_type(:P_error_token) # %error-token + define_type(:P_lex_param) # %lex-param + define_type(:P_parse_param) # %parse-param + define_type(:P_initial_action) # %initial-action + define_type(:P_union) # %union + define_type(:P_token) # %token + define_type(:P_type) # %type + define_type(:P_nonassoc) # %nonassoc + define_type(:P_left) # %left + define_type(:P_right) # %right + define_type(:P_prec) # %prec + define_type(:User_code) # { ... } + define_type(:Tag) # + define_type(:Number) # 0 + define_type(:Ident_Colon) # k_if:, k_if : (spaces can be there) + define_type(:Ident) # api.pure, tNUMBER + define_type(:Named_Ref) # [foo] + define_type(:Semicolon) # ; + define_type(:Bar) # | + define_type(:String) # "str" + define_type(:Char) # '+' + end + end +end