Merge branch 'master' into chore

.github/workflows/test.yaml

@@ -20,6 +20,7 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
+      - run: flex --help
       - run: bundle install
       - run: bundle exec rspec
   test-windows:
@@ -34,6 +35,8 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
+      - run: choco install winflexbison
+      - run: win_flex --help
       - run: bundle install
       - run: bundle exec rspec
   check-misc:
@@ -65,6 +68,8 @@ jobs:
       fail-fast: false
       matrix:
         ruby: ['head']
+    env:
+      INSTALL_STEEP: 'true'
     steps:
       - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1

Gemfile

@@ -5,8 +5,13 @@ gemspec
 gem "pry"
 gem "racc", "1.7.3"
 gem "rake"
-gem "rbs", require: false
 gem "rspec"
 gem "simplecov", require: false
 gem "stackprof", platforms: [:ruby] # stackprof doesn't support Windows
-gem "steep", require: false
+
+# Recent steep requires Ruby >= 3.0.0.
+# Then skip install on some CI jobs.
+if !ENV['GITHUB_ACTION'] || ENV['INSTALL_STEEP'] == 'true'
+  gem "rbs", "3.3.0", require: false
+  gem "steep", "1.6.0", require: false
+end

Steepfile

@@ -6,8 +6,12 @@ target :lib do
 
   check "lib/lrama/bitmap.rb"
   check "lib/lrama/digraph.rb"
+  check "lib/lrama/grammar/code.rb"
+  check "lib/lrama/grammar/code/printer_code.rb"
   check "lib/lrama/grammar/counter.rb"
   check "lib/lrama/grammar/percent_code.rb"
+  check "lib/lrama/grammar/precedence.rb"
+  check "lib/lrama/grammar/printer.rb"
   check "lib/lrama/grammar/reference.rb"
   check "lib/lrama/grammar/rule_builder.rb"
   check "lib/lrama/lexer/token/char.rb"

lib/lrama/grammar.rb

@@ -373,10 +373,10 @@ def append_special_symbols
     #
     def normalize_rules
       # 1. Add $accept rule to the top of rules
-      accept = find_symbol_by_s_value!("$accept")
-      eof = find_symbol_by_number!(0)
+      accept = @accept_symbol
+      eof = @eof_symbol
       lineno = @rule_builders.first ? @rule_builders.first.line : 0
-      @rules << Rule.new(id: @rule_counter.increment, lhs: accept, _rhs: [@rule_builders.first.lhs, eof], token_code: nil, lineno: lineno)
+      @rules << Rule.new(id: @rule_counter.increment, _lhs: accept.id, _rhs: [@rule_builders.first.lhs, eof.id], token_code: nil, lineno: lineno)
 
       setup_rules
 
@@ -387,12 +387,12 @@ def normalize_rules
         end
 
         builder.rules.each do |rule|
-          add_nterm(id: rule.lhs)
+          add_nterm(id: rule._lhs)
           @rules << rule
         end
 
         builder.midrule_action_rules.each do |rule|
-          add_nterm(id: rule.lhs)
+          add_nterm(id: rule._lhs)
         end
       end
     end
@@ -405,8 +405,6 @@ def collect_symbols
           add_term(id: s)
         when Lrama::Lexer::Token
           # skip
-        when Symbol
-          # skip
         else
           raise "Unknown class: #{s}"
         end
@@ -488,7 +486,7 @@ def fill_symbol_number
 
     def replace_token_with_symbol
       @rules.each do |rule|
-        rule.lhs = token_to_symbol(rule.lhs)
+        rule.lhs = token_to_symbol(rule._lhs) if rule._lhs
 
         rule.rhs = rule._rhs.map do |t|
           token_to_symbol(t)
@@ -500,8 +498,6 @@ def token_to_symbol(token)
       case token
       when Lrama::Lexer::Token
         find_symbol_by_id!(token)
-      when Symbol
-        token
       else
         raise "Unknown class: #{token}"
       end

lib/lrama/grammar/parameterizing_rules/builder/list.rb

@@ -8,9 +8,9 @@ def build
 
             rules = []
             list_token = Lrama::Lexer::Token::Ident.new(s_value: "list_#{@token.s_value}")
-            rules << Rule.new(id: @rule_counter.increment, lhs: @lhs, _rhs: [list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: list_token, _rhs: [], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: list_token, _rhs: [list_token, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: @lhs, _rhs: [list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: list_token, _rhs: [], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: list_token, _rhs: [list_token, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
             rules
           end
         end

lib/lrama/grammar/parameterizing_rules/builder/nonempty_list.rb

@@ -8,9 +8,9 @@ def build
 
             rules = []
             nonempty_list_token = Lrama::Lexer::Token::Ident.new(s_value: "nonempty_list_#{@token.s_value}")
-            rules << Rule.new(id: @rule_counter.increment, lhs: @lhs, _rhs: [nonempty_list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: nonempty_list_token, _rhs: [@token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: nonempty_list_token, _rhs: [nonempty_list_token, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: @lhs, _rhs: [nonempty_list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: nonempty_list_token, _rhs: [@token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: nonempty_list_token, _rhs: [nonempty_list_token, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
             rules
           end
         end

lib/lrama/grammar/parameterizing_rules/builder/option.rb

@@ -8,9 +8,9 @@ def build
 
             rules = []
             option_token = Lrama::Lexer::Token::Ident.new(s_value: "option_#{@token.s_value}")
-            rules << Rule.new(id: @rule_counter.increment, lhs: @lhs, _rhs: [option_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: option_token, _rhs: [], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: option_token, _rhs: [@token], token_code: @ser_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: @lhs, _rhs: [option_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: option_token, _rhs: [], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: option_token, _rhs: [@token], token_code: @ser_code, precedence_sym: @precedence_sym, lineno: @line)
             rules
           end
         end

lib/lrama/grammar/parameterizing_rules/builder/separated_list.rb

@@ -15,10 +15,10 @@ def build
 
             rules = []
             separated_list_token = Lrama::Lexer::Token::Ident.new(s_value: "separated_list_#{@token.s_value}")
-            rules << Rule.new(id: @rule_counter.increment, lhs: @lhs, _rhs: [separated_list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: separated_list_token, _rhs: [], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: separated_list_token, _rhs: [@token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: separated_list_token, _rhs: [separated_list_token, @separator, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: @lhs, _rhs: [separated_list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: separated_list_token, _rhs: [], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: separated_list_token, _rhs: [@token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: separated_list_token, _rhs: [separated_list_token, @separator, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
             rules
           end
         end

lib/lrama/grammar/parameterizing_rules/builder/separated_nonempty_list.rb

@@ -15,9 +15,9 @@ def build
 
             rules = []
             separated_list_token = Lrama::Lexer::Token::Ident.new(s_value: "separated_nonempty_list_#{@token.s_value}")
-            rules << Rule.new(id: @rule_counter.increment, lhs: @lhs, _rhs: [separated_list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: separated_list_token, _rhs: [@token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
-            rules << Rule.new(id: @rule_counter.increment, lhs: separated_list_token, _rhs: [separated_list_token, @separator, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: @lhs, _rhs: [separated_list_token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: separated_list_token, _rhs: [@token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
+            rules << Rule.new(id: @rule_counter.increment, _lhs: separated_list_token, _rhs: [separated_list_token, @separator, @token], token_code: @user_code, precedence_sym: @precedence_sym, lineno: @line)
             rules
           end
         end

lib/lrama/grammar/rule.rb

@@ -1,7 +1,7 @@
 module Lrama
   class Grammar
     # _rhs holds original RHS element. Use rhs to refer to Symbol.
-    class Rule < Struct.new(:id, :lhs, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true)
+    class Rule < Struct.new(:id, :_lhs, :lhs, :_rhs, :rhs, :token_code, :position_in_original_rule_rhs, :nullable, :precedence_sym, :lineno, keyword_init: true)
       attr_accessor :original_rule
 
       def ==(other)

lib/lrama/grammar/rule_builder.rb

@@ -87,7 +87,7 @@ def build_rules
           @midrule_action_rules = []
         else
           rule = Rule.new(
-            id: @rule_counter.increment, lhs: lhs, _rhs: tokens, token_code: user_code,
+            id: @rule_counter.increment, _lhs: lhs, _rhs: tokens, token_code: user_code,
             position_in_original_rule_rhs: @position_in_original_rule_rhs, precedence_sym: precedence_sym, lineno: line
           )
           @rules = [rule]

lib/lrama/lexer.rb

@@ -71,8 +71,8 @@ def lex_token
           # noop
         when @scanner.scan(/\/\*/)
           lex_comment
-        when @scanner.scan(/\/\/.*?\n/)
-          newline
+        when @scanner.scan(/\/\/.*(?<newline>\n)?/)
+          newline if @scanner[:newline]
         else
           break
         end
@@ -141,7 +141,11 @@ def lex_c_code
         when @scanner.scan(/'.*?'/)
           code += %Q(#{@scanner.matched})
         else
-          code += @scanner.getch
+          if @scanner.scan(/[^\"'\{\}\n#{@end_symbol}]+/)
+            code += @scanner.matched
+          else
+            code += @scanner.getch
+          end
         end
       end
       raise ParseError, "Unexpected code: #{code}."

rbs_collection.lock.yaml

@@ -2,7 +2,7 @@
 sources:
 - type: git
   name: ruby/gem_rbs_collection
-  revision: 2de2d4535caba275f3b8533684aab110d921f553
+  revision: 25286c51a19927f28623aee3cd36655f902399ba
   remote: https://github.com/ruby/gem_rbs_collection.git
   repo_dir: gems
 path: ".gem_rbs_collection"
@@ -15,6 +15,10 @@ gems:
   version: '0'
   source:
     type: stdlib
+- name: forwardable
+  version: '0'
+  source:
+    type: stdlib
 - name: rake
   version: '13.0'
   source:

rbs_collection.yaml

@@ -20,3 +20,4 @@ gems:
   # It's unnecessary if you don't use rbs as a library.
   - name: rbs
     ignore: true
+  - name: forwardable

sig/lrama/grammar/code.rbs

@@ -0,0 +1,24 @@
+module Lrama
+  class Grammar
+    class Code
+      extend Forwardable
+
+      attr_accessor type: untyped
+      attr_accessor token_code: untyped
+
+      # delegated
+      def s_value: -> String
+      def line: -> Integer
+      def column: -> untyped
+      def references: -> untyped
+
+      def initialize: (?type: untyped, ?token_code: untyped) -> void
+
+      def translated_code: () -> String
+
+      private
+
+      def reference_to_c: (untyped ref) -> untyped
+    end
+  end
+end

sig/lrama/grammar/code/printer_code.rbs

@@ -0,0 +1,15 @@
+module Lrama
+  class Grammar
+    class Code
+      class PrinterCode < Code
+        @tag: untyped
+        def initialize: (?type: untyped, ?token_code: untyped, ?tag: untyped) -> void
+
+        private
+
+        # ref: Lrama::Grammar::Code.token_code.references
+        def reference_to_c: (untyped ref) -> untyped
+      end
+    end
+  end
+end

sig/lrama/grammar/precedence.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Precedence
+      include Comparable
+      attr_accessor type: Symbol
+      attr_accessor precedence: Integer
+
+      def <=>: (Precedence other) -> Integer
+    end
+  end
+end

sig/lrama/grammar/printer.rbs

@@ -0,0 +1,11 @@
+module Lrama
+  class Grammar
+    class Printer
+      attr_accessor ident_or_tags: Array[Lexer::Token::Ident|Lexer::Token::Tag]
+      attr_accessor token_code: Grammar::Code
+      attr_accessor lineno: Integer
+
+      def translated_code: (Lexer::Token member) -> String
+    end
+  end
+end

sig/lrama/grammar/rule.rbs

@@ -4,7 +4,7 @@ module Lrama
       attr_accessor original_rule: Rule
 
       def initialize: (
-        ?id: untyped, ?lhs: untyped, ?_rhs: untyped, ?rhs: untyped,
+        ?id: untyped, ?_lhs: untyped, ?lhs: untyped, ?_rhs: untyped, ?rhs: untyped,
         ?token_code: untyped, ?position_in_original_rule_rhs: untyped, ?nullable: untyped,
         ?precedence_sym: untyped, ?lineno: untyped
       ) -> void

sig/lrama/grammar/rule_builder.rbs

@@ -7,26 +7,35 @@ module Lrama
       attr_reader user_code: Lexer::Token::UserCode?
       attr_reader precedence_sym: Lexer::Token?
 
+      @rule_counter: Counter
+      @midrule_action_counter: Counter
+      @position_in_original_rule_rhs: Integer?
+      @skip_preprocess_references: bool
       @user_code: Lexer::Token::UserCode?
+      @rule_builders_for_derived_rules: Array[RuleBuilder]
+      @rules: Array[Rule]
+      @replaced_rhs: Array[Lexer::Token]
+      @parameterizing_rules: Array[Rule]
+      @midrule_action_rules: Array[Rule]
 
       def initialize: (Counter rule_counter, Counter midrule_action_counter, ?Integer position_in_original_rule_rhs, ?skip_preprocess_references: bool) -> void
       def add_rhs: (Lexer::Token rhs) -> void
       def user_code=: (Lexer::Token::UserCode user_code) -> void
       def precedence_sym=: (Lexer::Token user_code) -> void
       def complete_input: () -> void
       def setup_rules: () -> void
-      def preprocess_references: () -> void
+      def parameterizing_rules: () -> Array[Rule]
       def midrule_action_rules: () -> Array[Rule]
-      def rhs_with_new_tokens: () -> Array[Lexer::Token]
+      def rules: () -> Array[Rule]
 
       private
 
       def freeze_rhs: () -> void
+      def preprocess_references: () -> void
+      def build_rules: () -> void
       def process_rhs: () -> void
       def numberize_references: () -> void
-      def setup_references: () -> void
       def flush_user_code: () -> void
-      def build_rules: () -> void
     end
   end
 end