From b7b46eeb0c0e0c3fe948ae1b546c400469b1162d Mon Sep 17 00:00:00 2001
From: tompng <tomoyapenguin@gmail.com>
Date: Sat, 7 Jan 2023 04:07:16 +0900
Subject: [PATCH] Add nesting parser tests, fix some existing tests

---
 test/irb/test_nesting_parser.rb | 299 ++++++++++++++++++++++++++++++++
 test/irb/test_ruby_lex.rb       |  86 ++++-----
 2 files changed, 334 insertions(+), 51 deletions(-)
 create mode 100644 test/irb/test_nesting_parser.rb

diff --git a/test/irb/test_nesting_parser.rb b/test/irb/test_nesting_parser.rb
new file mode 100644
index 000000000..2edaea35f
--- /dev/null
+++ b/test/irb/test_nesting_parser.rb
@@ -0,0 +1,299 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'irb'
+require 'rubygems'
+
+require_relative "helper"
+
+module TestIRB
+  class TestNestingParser < TestCase
+    def setup
+      save_encodings
+    end
+
+    def teardown
+      restore_encodings
+    end
+
+    def parse_by_line(code)
+      IRB::NestingParser.parse_by_line(RubyLex.ripper_lex_without_warning(code))
+    end
+
+    def test_scan
+      code = <<~'EOS'
+        class A
+          def f
+            if true
+              tap do
+                {
+                  x: "
+                    #{p(1, 2, 3
+      EOS
+      opens = IRB::NestingParser.scan(RubyLex.ripper_lex_without_warning(code))
+      assert_equal(%w[class def if do { " #{ (], opens.map(&:tok))
+    end
+
+    def test_parse_by_line
+      code = <<~EOS
+        (((((1+2
+        ).to_s())).tap do (((
+      EOS
+      _tokens, prev_opens, next_opens, min_depth = parse_by_line(code).last
+      assert_equal(%w[( ( ( ( (], prev_opens.map(&:tok))
+      assert_equal(%w[( ( do ( ( (], next_opens.map(&:tok))
+      assert_equal(2, min_depth)
+    end
+
+    def test_ruby_syntax
+      code = <<~'EOS'
+        class A
+          1 if 2
+          1 while 2
+          1 until 2
+          1 unless 2
+          1 rescue 2
+          begin; rescue; ensure; end
+          tap do; rescue; ensure; end
+          class B; end
+          module C; end
+          def f; end
+          def `; end
+          def f() = 1
+          %(); %w[]; %q(); %r{}; %i[]
+          "#{1}"; ''; /#{1}/; `#{1}`
+          :sym; :"sym"; :+; :`; :if
+          [1, 2, 3]
+          { x: 1, y: 2 }
+          (a, (*b, c), d), e = 1, 2, 3
+          ->(a){}; ->(a) do end
+          -> a = -> b = :do do end do end
+          if 1; elsif 2; else; end
+          unless 1; end
+          while 1; end
+          until 1; end
+          for i in j; end
+          case 1; when 2; end
+          puts(1, 2, 3)
+          loop{|i|}
+          loop do |i| end
+        end
+      EOS
+      line_results = parse_by_line(code)
+      line_results[1...-1].each {|result| assert_equal(['class'], result[2].map(&:tok)) }
+    end
+
+    def test_multiline_string
+      code = <<~EOS
+        "
+        aaa
+        bbb
+        "
+        <<A
+        aaa
+        bbb
+        A
+      EOS
+      line_results = parse_by_line(code)
+      string_content_line, string_opens = line_results[1]
+      assert_equal("\naaa\nbbb\n", string_content_line.map(&:first).map(&:tok).join)
+      assert_equal("aaa\n", string_content_line.map(&:last).join)
+      assert_equal(['"'], string_opens.map(&:tok))
+      heredoc_content_line, heredoc_opens = line_results[6]
+      assert_equal("aaa\nbbb\n", heredoc_content_line.map(&:first).map(&:tok).join)
+      assert_equal("bbb\n", heredoc_content_line.map(&:last).join)
+      assert_equal(['<<A'], heredoc_opens.map(&:tok))
+      _line, _prev_opens, next_opens, _min_depth = line_results.last
+      assert_equal([], next_opens)
+    end
+
+    def test_backslash_continued_nested_symbol
+      code = <<~'EOS'
+        x = <<A, :\
+          heredoc #{
+            here
+          }
+        A
+        =begin
+        embdoc
+        =end
+        # comment
+
+        if # this is symbol :if
+        while
+      EOS
+      line_results = parse_by_line(code)
+      assert_equal(%w[: <<A #{], line_results[2][2].map(&:tok))
+      assert_equal(%w[while], line_results.last[2].map(&:tok))
+    end
+
+    def test_oneliner_def
+      code = <<~EOC
+        if true
+          # normal oneliner def
+          def f = 1
+          def f() = 1
+          def f(*) = 1
+          # keyword, backtick, op
+          def * = 1
+          def ` = 1
+          def if = 1
+          def *() = 1
+          def `() = 1
+          def if() = 1
+          # oneliner def with receiver
+          def a.* = 1
+          def $a.* = 1
+          def @a.` = 1
+          def A.` = 1
+          def ((a;b;c)).*() = 1
+          def ((a;b;c)).if() = 1
+          def ((a;b;c)).end() = 1
+          # multiline oneliner def
+          def f =
+          1
+          def f()
+          =
+          1
+          # oneliner def with comment and embdoc
+          def # comment
+        =begin
+        embdoc
+        =end
+            ((a;b;c))
+            . # comment
+        =begin
+        embdoc
+        =end
+            f (*) # comment
+        =begin
+        embdoc
+        =end
+          =
+          1
+          # nested oneliner def
+          def f(x = def f() = 1) = def f() = 1
+      EOC
+      _tokens, _prev_opens, next_opens, min_depth = parse_by_line(code).last
+      assert_equal(['if'], next_opens.map(&:tok))
+      assert_equal(1, min_depth)
+    end
+
+    def test_heredoc_embexpr
+      code = <<~'EOS'
+        <<A+<<B+<<C+(<<D+(<<E)
+          #{
+            <<~F+"#{<<~G}
+            #{
+              here
+            }
+            F
+            G
+            "
+          }
+        A
+        B
+        C
+        D
+        E
+        )
+      EOS
+      line_results = parse_by_line(code)
+      last_opens = line_results.last[-2]
+      assert_equal([], last_opens)
+      _tokens, _prev_opens, next_opens, _min_depth = line_results[4]
+      assert_equal(%w[( <<E <<D <<C <<B <<A #{ " <<~G <<~F #{], next_opens.map(&:tok))
+    end
+
+    def test_for_in
+      code = <<~EOS
+        for i in j
+          here
+        end
+        for i in j do
+          here
+        end
+        for i in
+          j do
+          here
+        end
+        for
+          # comment
+          i in j do
+          here
+        end
+        for (a;b;c).d in (a;b;c) do
+          here
+        end
+        for i in :in + :do do
+          here
+        end
+        for i in -> do end do
+          here
+        end
+      EOS
+      line_results = parse_by_line(code).select { |tokens,| tokens.map(&:last).include? 'here' }
+      assert_equal(7, line_results.size)
+      line_results.each do |_tokens, _prev_opens, next_opens, _min_depth|
+        assert_equal(['for'], next_opens.map(&:tok))
+      end
+    end
+
+    def test_while_until
+      base_code = <<~'EOS'
+        while_or_until true
+          here
+        end
+        while_or_until a < c
+          here
+        end
+        while_or_until true do
+          here
+        end
+        while_or_until
+          # comment
+          (a + b) <
+          # comment
+          c do
+          here
+        end
+        while_or_until :\
+          do do
+          here
+        end
+        while_or_until def do; end == :do do
+          here
+        end
+        while_or_until -> do end do
+          here
+        end
+      EOS
+      %w[while until].each do |keyword|
+        code = base_code.gsub('while_or_until', keyword)
+        line_results = parse_by_line(code).select { |tokens,| tokens.map(&:last).include? 'here' }
+        assert_equal(7, line_results.size)
+        line_results.each do |_tokens, _prev_opens, next_opens, _min_depth|
+          assert_equal([keyword], next_opens.map(&:tok) )
+        end
+      end
+    end
+
+    def test_case_in
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        pend 'This test requires ruby version that supports case-in syntax'
+      end
+      code = <<~EOS
+        case 1
+        in 1
+          here
+        in
+          2
+          here
+        end
+      EOS
+      line_results = parse_by_line(code).select { |tokens,| tokens.map(&:last).include?('here') }
+      assert_equal(2, line_results.size)
+      line_results.each do |_tokens, _prev_opens, next_opens, _min_depth|
+        assert_equal(['in'], next_opens.map(&:tok))
+      end
+    end
+  end
+end
diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb
index e2a471c1b..745b7ecc3 100644
--- a/test/irb/test_ruby_lex.rb
+++ b/test/irb/test_ruby_lex.rb
@@ -95,8 +95,11 @@ def assert_code_block_open(lines, expected, local_variables: [])
 
     def check_state(lines, local_variables: [])
       context = build_context(local_variables)
+      tokens = RubyLex.ripper_lex_without_warning(lines.join("\n"), context: context)
+      opens = IRB::NestingParser.scan(tokens)
       ruby_lex = RubyLex.new(context)
-      _ltype, indent, _continue, code_block_open = ruby_lex.check_code_state(lines.join("\n"))
+      indent, _nesting_level = ruby_lex.calc_nesting_depth(opens)
+      code_block_open = !opens.empty? || ruby_lex.process_continue(tokens)
       [indent, code_block_open]
     end
 
@@ -164,9 +167,9 @@ def test_multiple_braces_in_a_line
         Row.new(%q(    ]), 4, 4),
         Row.new(%q(  ]), 2, 2),
         Row.new(%q(]), 0, 0),
-        Row.new(%q([<<FOO]), 0, 0),
+        Row.new(%q([<<FOO]), nil, 0),
         Row.new(%q(hello), 0, 0),
-        Row.new(%q(FOO), nil, 0),
+        Row.new(%q(FOO), 0, 0),
       ]
 
       lines = []
@@ -489,12 +492,12 @@ def test_corresponding_syntax_to_keyword_for_with_do
       end
     end
 
-    def test_corresponding_syntax_to_keyword_in
+    def test_typing_incomplete_include_interpreted_as_keyword_in
       input_with_correct_indents = [
         Row.new(%q(module E), nil, 2, 1),
         Row.new(%q(end), 0, 0, 0),
         Row.new(%q(class A), nil, 2, 1),
-        Row.new(%q(  in), nil, 4, 1)
+        Row.new(%q(  in), nil, 2, 1) # scenario typing `include E`
       ]
 
       lines = []
@@ -575,11 +578,19 @@ def test_local_variables_dependent_code
     end
 
     def test_heredoc_with_indent
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        pend 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
       input_with_correct_indents = [
-        Row.new(%q(<<~Q), 0, 0, 0),
-        Row.new(%q({), 0, 0, 0),
-        Row.new(%q(  #), 2, 0, 0),
-        Row.new(%q(}), 0, 0, 0)
+        Row.new(%q(<<~Q+<<~R), nil, 0, 0),
+        Row.new(%q(a), 0, 0, 0),
+        Row.new(%q(a), 0, 0, 0),
+        Row.new(%q(  b), 2, 2, 0),
+        Row.new(%q(  b), 2, 2, 0),
+        Row.new(%q(  Q), 0, 2, 0),
+        Row.new(%q(    c), 4, 4, 0),
+        Row.new(%q(    c), 4, 4, 0),
+        Row.new(%q(    R), 0, 0, 0),
       ]
 
       lines = []
@@ -592,8 +603,8 @@ def test_heredoc_with_indent
 
     def test_oneliner_def_in_multiple_lines
       input_with_correct_indents = [
-        Row.new(%q(def a()=[), nil, 4, 2),
-        Row.new(%q(  1,), nil, 4, 1),
+        Row.new(%q(def a()=[), nil, 2, 1),
+        Row.new(%q(  1,), nil, 2, 1),
         Row.new(%q(].), 0, 0, 0),
         Row.new(%q(to_s), nil, 0, 0),
       ]
@@ -609,7 +620,7 @@ def test_oneliner_def_in_multiple_lines
     def test_broken_heredoc
       input_with_correct_indents = [
         Row.new(%q(def foo), nil, 2, 1),
-        Row.new(%q(  <<~Q), 2, 2, 1),
+        Row.new(%q(  <<~Q), nil, 2, 1),
         Row.new(%q(  Qend), 2, 2, 1),
       ]
 
@@ -621,6 +632,15 @@ def test_broken_heredoc
       end
     end
 
+    def test_heredoc_keep_indent_spaces
+      (1..4).each do |indent|
+        row = Row.new(' ' * indent, indent, [2, indent].max, 1)
+        lines = ['def foo', '  <<~Q', row.content]
+        assert_row_indenting(lines, row)
+        assert_nesting_level(lines, row.nesting_level)
+      end
+    end
+
     PromptRow = Struct.new(:prompt, :content)
 
     class MockIO_DynamicPrompt
@@ -746,10 +766,9 @@ def test_unterminated_code
     end
 
     def test_unterminated_heredoc_string_literal
-      context = build_context
       ['<<A;<<B', "<<A;<<B\n", "%W[\#{<<A;<<B", "%W[\#{<<A;<<B\n"].each do |code|
         tokens = RubyLex.ripper_lex_without_warning(code)
-        string_literal = RubyLex.new(context).check_string_literal(tokens)
+        string_literal = IRB::NestingParser.scan(tokens).last
         assert_equal('<<A', string_literal&.tok)
       end
     end
@@ -779,43 +798,8 @@ def test_corresponding_token_depth_with_heredoc_and_embdoc
       [reference_code, code_with_heredoc, code_with_embdoc].each do |code|
         lex = RubyLex.new(context)
         lines = code.lines
-        lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code))
-        assert_equal 2, lex.check_corresponding_token_depth(lines, lines.size)
-      end
-    end
-
-    def test_find_prev_spaces_with_multiline_literal
-      lex = RubyLex.new(build_context)
-      reference_code = <<~EOC.chomp
-        if true
-          1
-          hello
-          1
-          world
-        end
-      EOC
-      code_with_percent_string = <<~EOC.chomp
-        if true
-          %w[
-            hello
-          ]
-          world
-        end
-      EOC
-      code_with_quoted_string = <<~EOC.chomp
-        if true
-          '
-            hello
-          '
-          world
-        end
-      EOC
-      context = build_context
-      [reference_code, code_with_percent_string, code_with_quoted_string].each do |code|
-        lex = RubyLex.new(context)
-        lex.instance_variable_set('@tokens', RubyLex.ripper_lex_without_warning(code))
-        prev_spaces = (1..code.lines.size).map { |index| lex.find_prev_spaces index }
-        assert_equal [0, 2, 2, 2, 2, 0], prev_spaces
+        tokens = RubyLex.ripper_lex_without_warning(code)
+        assert_equal(2, lex.check_corresponding_token_depth(tokens, lines, lines.size - 1))
       end
     end