Refactor lrama/lexer.rb

lib/lrama/lexer.rb

@@ -1,85 +1,12 @@
 require "strscan"
 require "lrama/report"
+require "lrama/lexer/token"
 
 module Lrama
   # Lexer for parse.y
   class Lexer
     include Lrama::Report::Duration
 
-    # s_value is semantic value
-    Token = Struct.new(:type, :s_value, :alias, keyword_init: true) do
-      Type = Struct.new(:id, :name, keyword_init: true)
-
-      attr_accessor :line, :column, :referred
-      # For User_code
-      attr_accessor :references
-
-      def to_s
-        "#{super} line: #{line}, column: #{column}"
-      end
-
-      def referred_by?(string)
-        [self.s_value, self.alias].include?(string)
-      end
-
-      def ==(other)
-        self.class == other.class && self.type == other.type && self.s_value == other.s_value
-      end
-
-      def numberize_references(lhs, rhs)
-        self.references.map! {|ref|
-          ref_name = ref[1]
-          if ref_name.is_a?(String) && ref_name != '$'
-            value =
-              if lhs.referred_by?(ref_name)
-                '$'
-              else
-                rhs.find_index {|token| token.referred_by?(ref_name) } + 1
-              end
-            [ref[0], value, ref[2], ref[3], ref[4]]
-          else
-            ref
-          end
-        }
-      end
-
-      @i = 0
-      @types = []
-
-      def self.define_type(name)
-        type = Type.new(id: @i, name: name.to_s)
-        const_set(name, type)
-        @types << type
-        @i += 1
-      end
-
-      # Token types
-      define_type(:P_expect)         # %expect
-      define_type(:P_define)         # %define
-      define_type(:P_printer)        # %printer
-      define_type(:P_error_token)    # %error-token
-      define_type(:P_lex_param)      # %lex-param
-      define_type(:P_parse_param)    # %parse-param
-      define_type(:P_initial_action) # %initial-action
-      define_type(:P_union)          # %union
-      define_type(:P_token)          # %token
-      define_type(:P_type)           # %type
-      define_type(:P_nonassoc)       # %nonassoc
-      define_type(:P_left)           # %left
-      define_type(:P_right)          # %right
-      define_type(:P_prec)           # %prec
-      define_type(:User_code)        # { ... }
-      define_type(:Tag)              # <int>
-      define_type(:Number)           # 0
-      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
-      define_type(:Ident)            # api.pure, tNUMBER
-      define_type(:Named_Ref)        # [foo]
-      define_type(:Semicolon)        # ;
-      define_type(:Bar)              # |
-      define_type(:String)           # "str"
-      define_type(:Char)             # '+'
-    end
-
     # States
     #
     # See: https://www.gnu.org/software/bison/manual/html_node/Grammar-Outline.html

lib/lrama/lexer/token.rb

@@ -0,0 +1,76 @@
+module Lrama
+  class Lexer
+    class Token < Struct.new(:type, :s_value, :alias, keyword_init: true)
+      Type = Struct.new(:id, :name, keyword_init: true)
+
+      attr_accessor :line, :column, :referred
+      # For User_code
+      attr_accessor :references
+
+      def to_s
+        "#{super} line: #{line}, column: #{column}"
+      end
+
+      def referred_by?(string)
+        [self.s_value, self.alias].include?(string)
+      end
+
+      def ==(other)
+        self.class == other.class && self.type == other.type && self.s_value == other.s_value
+      end
+
+      def numberize_references(lhs, rhs)
+        self.references.map! {|ref|
+          ref_name = ref[1]
+          if ref_name.is_a?(::String) && ref_name != '$'
+            value =
+              if lhs.referred_by?(ref_name)
+                '$'
+              else
+                rhs.find_index {|token| token.referred_by?(ref_name) } + 1
+              end
+            [ref[0], value, ref[2], ref[3], ref[4]]
+          else
+            ref
+          end
+        }
+      end
+
+      @i = 0
+      @types = []
+
+      def self.define_type(name)
+        type = Type.new(id: @i, name: name.to_s)
+        const_set(name, type)
+        @types << type
+        @i += 1
+      end
+
+      # Token types
+      define_type(:P_expect)         # %expect
+      define_type(:P_define)         # %define
+      define_type(:P_printer)        # %printer
+      define_type(:P_error_token)    # %error-token
+      define_type(:P_lex_param)      # %lex-param
+      define_type(:P_parse_param)    # %parse-param
+      define_type(:P_initial_action) # %initial-action
+      define_type(:P_union)          # %union
+      define_type(:P_token)          # %token
+      define_type(:P_type)           # %type
+      define_type(:P_nonassoc)       # %nonassoc
+      define_type(:P_left)           # %left
+      define_type(:P_right)          # %right
+      define_type(:P_prec)           # %prec
+      define_type(:User_code)        # { ... }
+      define_type(:Tag)              # <int>
+      define_type(:Number)           # 0
+      define_type(:Ident_Colon)      # k_if:, k_if  : (spaces can be there)
+      define_type(:Ident)            # api.pure, tNUMBER
+      define_type(:Named_Ref)        # [foo]
+      define_type(:Semicolon)        # ;
+      define_type(:Bar)              # |
+      define_type(:String)           # "str"
+      define_type(:Char)             # '+'
+    end
+  end
+end