From 649aba28f43534c729087238d01362a1f01bcc4a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 09:57:39 -0400 Subject: [PATCH 001/208] [ruby/yarp] Add Node#copy and MutationVisitor https://github.com/ruby/yarp/commit/3693091661 --- lib/yarp.rb | 1 + lib/yarp/yarp.gemspec | 1 + .../lib/yarp/mutation_visitor.rb.erb | 19 +++++++++++++++++++ yarp/templates/lib/yarp/node.rb.erb | 9 +++++++++ yarp/templates/template.rb | 1 + 5 files changed, 31 insertions(+) create mode 100644 yarp/templates/lib/yarp/mutation_visitor.rb.erb diff --git a/lib/yarp.rb b/lib/yarp.rb index 15217e80f490a7..a3b0c3b0742fa5 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -507,6 +507,7 @@ def self.parse_serialize_file(filepath) end require_relative "yarp/lex_compat" +require_relative "yarp/mutation_visitor" require_relative "yarp/node" require_relative "yarp/ripper_compat" require_relative "yarp/serialize" diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index db69bcb451b8d7..70b6c7ebaa374f 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -61,6 +61,7 @@ Gem::Specification.new do |spec| "lib/yarp.rb", "lib/yarp/ffi.rb", "lib/yarp/lex_compat.rb", + "lib/yarp/mutation_visitor.rb", "lib/yarp/node.rb", "lib/yarp/pack.rb", "lib/yarp/ripper_compat.rb", diff --git a/yarp/templates/lib/yarp/mutation_visitor.rb.erb b/yarp/templates/lib/yarp/mutation_visitor.rb.erb new file mode 100644 index 00000000000000..c88cabbdbb030c --- /dev/null +++ b/yarp/templates/lib/yarp/mutation_visitor.rb.erb @@ -0,0 +1,19 @@ +module YARP + # This visitor walks through the tree and copies each node as it is being + # visited. This is useful for consumers that want to mutate the tree, as you + # can change subtrees in place without effecting the rest of the tree. + class MutationVisitor < BasicVisitor + <%- nodes.each_with_index do |node, index| -%> +<%= "\n" if index != 0 -%> + # Copy a <%= node.name %> node + def visit_<%= node.human %>(node) + <%- params = node.params.select { |param| [NodeParam, OptionalNodeParam, NodeListParam].include?(param.class) } -%> + <%- if params.any? -%> + node.copy(<%= params.map { |param| "#{param.name}: visit(node.#{param.name})" }.join(", ") %>) + <%- else -%> + node.copy + <%- end -%> + end + <%- end -%> + end +end diff --git a/yarp/templates/lib/yarp/node.rb.erb b/yarp/templates/lib/yarp/node.rb.erb index c1f14f537af9ec..9bd9eb54e1d6ac 100644 --- a/yarp/templates/lib/yarp/node.rb.erb +++ b/yarp/templates/lib/yarp/node.rb.erb @@ -48,6 +48,15 @@ module YARP }.compact.join(", ") %>] end + # def copy: (**params) -> <%= node.name %> + def copy(**params) + <%= node.name %>.new( + <%- (node.params.map(&:name) + ["location"]).map do |name| -%> + <%= name %>: params.fetch(:<%= name %>) { self.<%= name %> }, + <%- end -%> + ) + end + # def deconstruct: () -> Array[nil | Node] alias deconstruct child_nodes diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index 8b8a175172ffea..e34d5b1a7be6ea 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -315,6 +315,7 @@ def locals "java/org/yarp/Loader.java", "java/org/yarp/Nodes.java", "java/org/yarp/AbstractNodeVisitor.java", + "lib/yarp/mutation_visitor.rb", "lib/yarp/node.rb", "lib/yarp/serialize.rb", "src/node.c", From a38ca45b6570ab378890664c9e8683f0c16cdd78 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 10:49:00 -0400 Subject: [PATCH 002/208] [ruby/yarp] Split up AndWriteNode, OrWriteNode, OperatorWriteNode https://github.com/ruby/yarp/commit/6d3b3b5776 --- test/yarp/location_test.rb | 105 ++- test/yarp/snapshots/blocks.txt | 8 +- test/yarp/snapshots/boolean_operators.txt | 24 +- test/yarp/snapshots/defined.txt | 8 +- .../seattlerb/bug_op_asgn_rescue.txt | 8 +- .../seattlerb/const_2_op_asgn_or2.txt | 18 +- .../seattlerb/const_3_op_asgn_or.txt | 12 +- .../seattlerb/const_op_asgn_and1.txt | 12 +- .../seattlerb/const_op_asgn_and2.txt | 12 +- .../snapshots/seattlerb/const_op_asgn_or.txt | 18 +- .../seattlerb/messy_op_asgn_lineno.txt | 18 +- .../seattlerb/op_asgn_command_call.txt | 8 +- ..._asgn_primary_colon_const_command_call.txt | 18 +- .../seattlerb/parse_line_defn_complex.txt | 8 +- .../seattlerb/parse_line_op_asgn.txt | 8 +- .../unparser/corpus/literal/assignment.txt | 16 +- .../unparser/corpus/literal/opasgn.txt | 72 +- .../unparser/corpus/literal/send.txt | 8 +- .../snapshots/whitequark/const_op_asgn.txt | 72 +- .../yarp/snapshots/whitequark/op_asgn_cmd.txt | 28 +- .../whitequark/rescue_mod_op_assign.txt | 10 +- .../snapshots/whitequark/ruby_bug_12402.txt | 56 +- .../snapshots/whitequark/ruby_bug_12669.txt | 40 +- .../snapshots/whitequark/var_and_asgn.txt | 8 +- .../yarp/snapshots/whitequark/var_op_asgn.txt | 32 +- .../snapshots/whitequark/var_op_asgn_cmd.txt | 10 +- .../yarp/snapshots/whitequark/var_or_asgn.txt | 8 +- yarp/config.yml | 302 ++++++-- yarp/yarp.c | 681 +++++++++++++++--- 29 files changed, 1174 insertions(+), 454 deletions(-) diff --git a/test/yarp/location_test.rb b/test/yarp/location_test.rb index 703c8e24f087ce..c7f9c0091bc420 100644 --- a/test/yarp/location_test.rb +++ b/test/yarp/location_test.rb @@ -17,16 +17,6 @@ def test_AndNode assert_location(AndNode, "foo && bar") end - def test_AndWriteNode - assert_location(AndWriteNode, "foo &&= bar") - assert_location(AndWriteNode, "foo = 1; foo &&= bar", 9...20) - assert_location(AndWriteNode, "@@foo &&= bar") - assert_location(AndWriteNode, "Parent::Child &&= bar") - assert_location(AndWriteNode, "Foo &&= bar") - assert_location(AndWriteNode, "$foo &&= bar") - assert_location(AndWriteNode, "@foo &&= bar") - end - def test_ArgumentsNode assert_location(ArgumentsNode, "foo(bar, baz, qux)", 4...17, &:arguments) end @@ -209,6 +199,18 @@ def test_ClassNode assert_location(ClassNode, "class Foo < Bar end") end + def test_ClassVariableAndWriteNode + assert_location(ClassVariableAndWriteNode, "@@foo &&= bar") + end + + def test_ClassVariableOperatorWriteNode + assert_location(ClassVariableOperatorWriteNode, "@@foo += bar") + end + + def test_ClassVariableOrWriteNode + assert_location(ClassVariableOrWriteNode, "@@foo ||= bar") + end + def test_ClassVariableReadNode assert_location(ClassVariableReadNode, "@@foo") end @@ -217,18 +219,42 @@ def test_ClassVariableWriteNode assert_location(ClassVariableWriteNode, "@@foo = bar") end + def test_ConstantPathAndWriteNode + assert_location(ConstantPathAndWriteNode, "Parent::Child &&= bar") + end + def test_ConstantPathNode assert_location(ConstantPathNode, "Foo::Bar") assert_location(ConstantPathNode, "::Foo") assert_location(ConstantPathNode, "::Foo::Bar") end + def test_ConstantPathOperatorWriteNode + assert_location(ConstantPathOperatorWriteNode, "Parent::Child += bar") + end + + def test_ConstantPathOrWriteNode + assert_location(ConstantPathOrWriteNode, "Parent::Child ||= bar") + end + def test_ConstantPathWriteNode assert_location(ConstantPathWriteNode, "Foo::Bar = baz") assert_location(ConstantPathWriteNode, "::Foo = bar") assert_location(ConstantPathWriteNode, "::Foo::Bar = baz") end + def test_ConstantAndWriteNode + assert_location(ConstantAndWriteNode, "Foo &&= bar") + end + + def test_ConstantOperatorWriteNode + assert_location(ConstantOperatorWriteNode, "Foo += bar") + end + + def test_ConstantOrWriteNode + assert_location(ConstantOrWriteNode, "Foo ||= bar") + end + def test_ConstantReadNode assert_location(ConstantReadNode, "Foo") assert_location(ConstantReadNode, "Foo::Bar", 5...8, &:child) @@ -302,6 +328,18 @@ def test_ForwardingSuperNode assert_location(ForwardingSuperNode, "super {}") end + def test_GlobalVariableAndWriteNode + assert_location(GlobalVariableAndWriteNode, "$foo &&= bar") + end + + def test_GlobalVariableOperatorWriteNode + assert_location(GlobalVariableOperatorWriteNode, "$foo += bar") + end + + def test_GlobalVariableOrWriteNode + assert_location(GlobalVariableOrWriteNode, "$foo ||= bar") + end + def test_GlobalVariableReadNode assert_location(GlobalVariableReadNode, "$foo") end @@ -336,6 +374,18 @@ def test_InNode end end + def test_InstanceVariableAndWriteNode + assert_location(InstanceVariableAndWriteNode, "@foo &&= bar") + end + + def test_InstanceVariableOperatorWriteNode + assert_location(InstanceVariableOperatorWriteNode, "@foo += bar") + end + + def test_InstanceVariableOrWriteNode + assert_location(InstanceVariableOrWriteNode, "@foo ||= bar") + end + def test_InstanceVariableReadNode assert_location(InstanceVariableReadNode, "@foo") end @@ -402,6 +452,21 @@ def test_LambdaNode assert_location(LambdaNode, "-> do foo end") end + def test_LocalVariableAndWriteNode + assert_location(LocalVariableAndWriteNode, "foo &&= bar") + assert_location(LocalVariableAndWriteNode, "foo = 1; foo &&= bar", 9...20) + end + + def test_LocalVariableOperatorWriteNode + assert_location(LocalVariableOperatorWriteNode, "foo += bar") + assert_location(LocalVariableOperatorWriteNode, "foo = 1; foo += bar", 9...19) + end + + def test_LocalVariableOrWriteNode + assert_location(LocalVariableOrWriteNode, "foo ||= bar") + assert_location(LocalVariableOrWriteNode, "foo = 1; foo ||= bar", 9...20) + end + def test_LocalVariableReadNode assert_location(LocalVariableReadNode, "foo = 1; foo", 9...12) end @@ -445,16 +510,6 @@ def test_NumberedReferenceReadNode assert_location(NumberedReferenceReadNode, "$1") end - def test_OperatorWriteNode - assert_location(OperatorWriteNode, "@@foo += bar") - assert_location(OperatorWriteNode, "Parent::Child += bar") - assert_location(OperatorWriteNode, "Foo += bar") - assert_location(OperatorWriteNode, "$foo += bar") - assert_location(OperatorWriteNode, "@foo += bar") - assert_location(OperatorWriteNode, "foo += bar") - assert_location(OperatorWriteNode, "foo = 1; foo += bar", 9...19) - end - def test_OptionalParameterNode assert_location(OptionalParameterNode, "def foo(bar = nil); end", 8...17) do |node| node.parameters.optionals.first @@ -466,16 +521,6 @@ def test_OrNode assert_location(OrNode, "foo or bar") end - def test_OrWriteNode - assert_location(OrWriteNode, "@@foo ||= bar") - assert_location(OrWriteNode, "Parent::Child ||= bar") - assert_location(OrWriteNode, "Foo ||= bar") - assert_location(OrWriteNode, "$foo ||= bar") - assert_location(OrWriteNode, "@foo ||= bar") - assert_location(OrWriteNode, "foo ||= bar") - assert_location(OrWriteNode, "foo = 1; foo ||= bar", 9...20) - end - def test_ParametersNode assert_location(ParametersNode, "def foo(bar, baz); end", 8...16, &:parameters) end diff --git a/test/yarp/snapshots/blocks.txt b/test/yarp/snapshots/blocks.txt index 089b63b89ca539..8f836208ca81b9 100644 --- a/test/yarp/snapshots/blocks.txt +++ b/test/yarp/snapshots/blocks.txt @@ -88,11 +88,13 @@ ProgramNode(0...402)( (61...62) ), StatementsNode(63...72)( - [OperatorWriteNode(63...72)( - LocalVariableWriteNode(63...67)(:memo, 0, nil, (63...67), nil), + [LocalVariableOperatorWriteNode(63...72)( + (63...67), (68...70), + LocalVariableReadNode(71...72)(:x, 0), + :memo, :+, - LocalVariableReadNode(71...72)(:x, 0) + 0 )] ), (51...52), diff --git a/test/yarp/snapshots/boolean_operators.txt b/test/yarp/snapshots/boolean_operators.txt index c6cfc311e560a8..f1fc5cd4b29472 100644 --- a/test/yarp/snapshots/boolean_operators.txt +++ b/test/yarp/snapshots/boolean_operators.txt @@ -1,21 +1,27 @@ ProgramNode(0...24)( [:a], StatementsNode(0...24)( - [AndWriteNode(0...7)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableAndWriteNode(0...7)( + (0...1), + (2...5), CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), - (2...5) + :a, + 0 ), - OperatorWriteNode(9...15)( - LocalVariableWriteNode(9...10)(:a, 0, nil, (9...10), nil), + LocalVariableOperatorWriteNode(9...15)( + (9...10), (11...13), + CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "b"), + :a, :+, - CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "b") + 0 ), - OrWriteNode(17...24)( - LocalVariableWriteNode(17...18)(:a, 0, nil, (17...18), nil), + LocalVariableOrWriteNode(17...24)( + (17...18), + (19...22), CallNode(23...24)(nil, nil, (23...24), nil, nil, nil, nil, 2, "b"), - (19...22) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/defined.txt b/test/yarp/snapshots/defined.txt index 6c2cad656e0feb..1d4b5a5502e876 100644 --- a/test/yarp/snapshots/defined.txt +++ b/test/yarp/snapshots/defined.txt @@ -8,11 +8,13 @@ ProgramNode(0...78)( ), DefinedNode(27...43)( (35...36), - OperatorWriteNode(36...42)( - LocalVariableWriteNode(36...37)(:x, 0, nil, (36...37), nil), + LocalVariableOperatorWriteNode(36...42)( + (36...37), (38...40), + IntegerNode(41...42)(), + :x, :%, - IntegerNode(41...42)() + 0 ), (42...43), (27...35) diff --git a/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt b/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt index c672ef6b79bcae..2bd4f430e3549f 100644 --- a/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt +++ b/test/yarp/snapshots/seattlerb/bug_op_asgn_rescue.txt @@ -1,14 +1,16 @@ ProgramNode(0...18)( [:a], StatementsNode(0...18)( - [OrWriteNode(0...18)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOrWriteNode(0...18)( + (0...1), + (2...5), RescueModifierNode(6...18)( CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), (8...14), NilNode(15...18)() ), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt b/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt index 7405ad8e5d9d22..48a4d61f5247b7 100644 --- a/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt +++ b/test/yarp/snapshots/seattlerb/const_2_op_asgn_or2.txt @@ -1,18 +1,14 @@ ProgramNode(0...12)( [], StatementsNode(0...12)( - [OrWriteNode(0...12)( - ConstantPathWriteNode(0...6)( - ConstantPathNode(0...6)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - ConstantReadNode(5...6)(), - (3...5) - ), - nil, - nil + [ConstantPathOrWriteNode(0...12)( + ConstantPathNode(0...6)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + ConstantReadNode(5...6)(), + (3...5) ), - IntegerNode(11...12)(), - (7...10) + (7...10), + IntegerNode(11...12)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt b/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt index 838287c59fa9f7..327e0a96431f1a 100644 --- a/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt +++ b/test/yarp/snapshots/seattlerb/const_3_op_asgn_or.txt @@ -1,14 +1,10 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [OrWriteNode(0...9)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), - IntegerNode(8...9)(), - (4...7) + [ConstantPathOrWriteNode(0...9)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + (4...7), + IntegerNode(8...9)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt index 2fdcd97205fd82..1e7dce022689a5 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_and1.txt @@ -1,15 +1,11 @@ ProgramNode(0...8)( [], StatementsNode(0...8)( - [OperatorWriteNode(0...8)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), + [ConstantPathOperatorWriteNode(0...8)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), (4...6), - :&, - IntegerNode(7...8)() + IntegerNode(7...8)(), + :& )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt index 275aa8238d4735..8c9d3ac353a61a 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_and2.txt @@ -1,14 +1,10 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [AndWriteNode(0...9)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), - IntegerNode(8...9)(), - (4...7) + [ConstantPathAndWriteNode(0...9)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + (4...7), + IntegerNode(8...9)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt b/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt index c14175c4e3a8a2..d812fe5a55e675 100644 --- a/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt +++ b/test/yarp/snapshots/seattlerb/const_op_asgn_or.txt @@ -1,18 +1,14 @@ ProgramNode(0...10)( [], StatementsNode(0...10)( - [OrWriteNode(0...10)( - ConstantPathWriteNode(0...4)( - ConstantPathNode(0...4)( - ConstantReadNode(0...1)(), - ConstantReadNode(3...4)(), - (1...3) - ), - nil, - nil + [ConstantPathOrWriteNode(0...10)( + ConstantPathNode(0...4)( + ConstantReadNode(0...1)(), + ConstantReadNode(3...4)(), + (1...3) ), - IntegerNode(9...10)(), - (5...8) + (5...8), + IntegerNode(9...10)() )] ) ) diff --git a/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt b/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt index 8c678e890d92d8..71a28870bd2240 100644 --- a/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt +++ b/test/yarp/snapshots/seattlerb/messy_op_asgn_lineno.txt @@ -9,18 +9,13 @@ ProgramNode(0...15)( ArgumentsNode(2...15)( [ParenthesesNode(2...15)( StatementsNode(3...14)( - [OperatorWriteNode(3...14)( - ConstantPathWriteNode(3...7)( - ConstantPathNode(3...7)( - ConstantReadNode(3...4)(), - ConstantReadNode(6...7)(), - (4...6) - ), - nil, - nil + [ConstantPathOperatorWriteNode(3...14)( + ConstantPathNode(3...7)( + ConstantReadNode(3...4)(), + ConstantReadNode(6...7)(), + (4...6) ), (8...10), - :*, CallNode(11...14)( nil, nil, @@ -43,7 +38,8 @@ ProgramNode(0...15)( nil, 0, "d" - ) + ), + :* )] ), (2...3), diff --git a/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt b/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt index deb7e17066b389..cf092dd5de2cfc 100644 --- a/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/op_asgn_command_call.txt @@ -1,8 +1,9 @@ ProgramNode(0...11)( [:a], StatementsNode(0...11)( - [OrWriteNode(0...11)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOrWriteNode(0...11)( + (0...1), + (2...5), CallNode(6...11)( CallNode(6...7)(nil, nil, (6...7), nil, nil, nil, nil, 2, "b"), (7...8), @@ -14,7 +15,8 @@ ProgramNode(0...11)( 0, "c" ), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt b/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt index d071a45442fe3c..0cd3775202dff6 100644 --- a/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt +++ b/test/yarp/snapshots/seattlerb/op_asgn_primary_colon_const_command_call.txt @@ -1,18 +1,13 @@ ProgramNode(0...11)( [], StatementsNode(0...11)( - [OperatorWriteNode(0...11)( - ConstantPathWriteNode(0...4)( - ConstantPathNode(0...4)( - ConstantReadNode(0...1)(), - ConstantReadNode(3...4)(), - (1...3) - ), - nil, - nil + [ConstantPathOperatorWriteNode(0...11)( + ConstantPathNode(0...4)( + ConstantReadNode(0...1)(), + ConstantReadNode(3...4)(), + (1...3) ), (5...7), - :*, CallNode(8...11)( nil, nil, @@ -25,7 +20,8 @@ ProgramNode(0...11)( nil, 0, "c" - ) + ), + :* )] ) ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt b/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt index 5d4820bef3cf7f..170b9bc81e321e 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_defn_complex.txt @@ -25,11 +25,13 @@ ProgramNode(0...40)( 0, "p" ), - OperatorWriteNode(18...24)( - LocalVariableWriteNode(18...19)(:y, 0, nil, (18...19), nil), + LocalVariableOperatorWriteNode(18...24)( + (18...19), (20...22), + IntegerNode(23...24)(), + :y, :*, - IntegerNode(23...24)() + 0 ), ReturnNode(27...35)( (27...33), diff --git a/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt b/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt index d8bd9891848f16..d3aa84c9869457 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_op_asgn.txt @@ -1,11 +1,13 @@ ProgramNode(6...34)( [:foo], StatementsNode(6...34)( - [OperatorWriteNode(6...24)( - LocalVariableWriteNode(6...9)(:foo, 0, nil, (6...9), nil), + [LocalVariableOperatorWriteNode(6...24)( + (6...9), (10...12), + CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "bar"), + :foo, :+, - CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "bar") + 0 ), CallNode(31...34)(nil, nil, (31...34), nil, nil, nil, nil, 2, "baz")] ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index ffaedf4f1834b5..71f7e8974bc187 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -610,10 +610,10 @@ ProgramNode(0...704)( ), (543...546) ), - OrWriteNode(551...561)( - InstanceVariableWriteNode(551...553)((551...553), nil, nil), - StringNode(558...561)((558...560), (560...560), (560...561), ""), - (554...557) + InstanceVariableOrWriteNode(551...561)( + (551...553), + (554...557), + StringNode(558...561)((558...560), (560...560), (560...561), "") ), LocalVariableWriteNode(562...576)( :x, @@ -703,16 +703,16 @@ ProgramNode(0...704)( ), (665...668) ), - OrWriteNode(687...704)( - InstanceVariableWriteNode(687...689)((687...689), nil, nil), + InstanceVariableOrWriteNode(687...704)( + (687...689), + (690...693), InterpolatedStringNode(694...704)( (694...704), [StringNode(705...707)(nil, (705...707), nil, " "), EmbeddedStatementsNode(707...710)((707...709), nil, (709...710)), StringNode(710...711)(nil, (710...711), nil, "\n")], (711...719) - ), - (690...693) + ) )] ) ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt b/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt index 980459766311ca..15d621cbf6de63 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/opasgn.txt @@ -1,53 +1,69 @@ ProgramNode(0...233)( [:a, :h], StatementsNode(0...233)( - [OperatorWriteNode(0...6)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOperatorWriteNode(0...6)( + (0...1), (2...4), + IntegerNode(5...6)(), + :a, :+, - IntegerNode(5...6)() + 0 ), - OperatorWriteNode(7...13)( - LocalVariableWriteNode(7...8)(:a, 0, nil, (7...8), nil), + LocalVariableOperatorWriteNode(7...13)( + (7...8), (9...11), + IntegerNode(12...13)(), + :a, :-, - IntegerNode(12...13)() + 0 ), - OperatorWriteNode(14...21)( - LocalVariableWriteNode(14...15)(:a, 0, nil, (14...15), nil), + LocalVariableOperatorWriteNode(14...21)( + (14...15), (16...19), + IntegerNode(20...21)(), + :a, :**, - IntegerNode(20...21)() + 0 ), - OperatorWriteNode(22...28)( - LocalVariableWriteNode(22...23)(:a, 0, nil, (22...23), nil), + LocalVariableOperatorWriteNode(22...28)( + (22...23), (24...26), + IntegerNode(27...28)(), + :a, :*, - IntegerNode(27...28)() + 0 ), - OperatorWriteNode(29...35)( - LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), + LocalVariableOperatorWriteNode(29...35)( + (29...30), (31...33), + IntegerNode(34...35)(), + :a, :/, - IntegerNode(34...35)() + 0 ), - AndWriteNode(36...43)( - LocalVariableWriteNode(36...37)(:a, 0, nil, (36...37), nil), + LocalVariableAndWriteNode(36...43)( + (36...37), + (38...41), CallNode(42...43)(nil, nil, (42...43), nil, nil, nil, nil, 2, "b"), - (38...41) + :a, + 0 ), - OrWriteNode(44...51)( - LocalVariableWriteNode(44...45)(:a, 0, nil, (44...45), nil), + LocalVariableOrWriteNode(44...51)( + (44...45), + (46...49), IntegerNode(50...51)(), - (46...49) + :a, + 0 ), CallNode(52...65)( ParenthesesNode(52...61)( StatementsNode(53...60)( - [OrWriteNode(53...60)( - LocalVariableWriteNode(53...54)(:a, 0, nil, (53...54), nil), + [LocalVariableOrWriteNode(53...60)( + (53...54), + (55...58), IntegerNode(59...60)(), - (55...58) + :a, + 0 )] ), (52...53), @@ -65,10 +81,12 @@ ProgramNode(0...233)( CallNode(66...83)( ParenthesesNode(66...76)( StatementsNode(67...75)( - [OrWriteNode(67...75)( - LocalVariableWriteNode(67...68)(:h, 0, nil, (67...68), nil), + [LocalVariableOrWriteNode(67...75)( + (67...68), + (69...72), HashNode(73...75)((73...74), [], (74...75)), - (69...72) + :h, + 0 )] ), (66...67), diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index 083a0140d9b974..4ef108e71a345e 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -6,8 +6,9 @@ ProgramNode(0...999)( (0...6), ConstantReadNode(7...8)(), StatementsNode(11...31)( - [OrWriteNode(11...31)( - LocalVariableWriteNode(11...14)(:foo, 0, nil, (11...14), nil), + [LocalVariableOrWriteNode(11...31)( + (11...14), + (15...18), ParenthesesNode(19...31)( StatementsNode(21...30)( [MultiWriteNode(21...30)( @@ -44,7 +45,8 @@ ProgramNode(0...999)( (19...20), (30...31) ), - (15...18) + :foo, + 0 )] ), (32...35), diff --git a/test/yarp/snapshots/whitequark/const_op_asgn.txt b/test/yarp/snapshots/whitequark/const_op_asgn.txt index e212170a1d799d..8a7bb4eae86a92 100644 --- a/test/yarp/snapshots/whitequark/const_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/const_op_asgn.txt @@ -1,53 +1,41 @@ ProgramNode(0...77)( [], StatementsNode(0...77)( - [OperatorWriteNode(0...8)( - ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), + [ConstantPathOperatorWriteNode(0...8)( + ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), (4...6), - :+, - IntegerNode(7...8)() + IntegerNode(7...8)(), + :+ ), - OperatorWriteNode(10...16)( - ConstantWriteNode(10...11)((10...11), nil, nil), + ConstantOperatorWriteNode(10...16)( + (10...11), (12...14), - :+, - IntegerNode(15...16)() + IntegerNode(15...16)(), + :+ ), - OperatorWriteNode(18...27)( - ConstantPathWriteNode(18...22)( - ConstantPathNode(18...22)( - ConstantReadNode(18...19)(), - ConstantReadNode(21...22)(), - (19...21) - ), - nil, - nil + ConstantPathOperatorWriteNode(18...27)( + ConstantPathNode(18...22)( + ConstantReadNode(18...19)(), + ConstantReadNode(21...22)(), + (19...21) ), (23...25), - :+, - IntegerNode(26...27)() + IntegerNode(26...27)(), + :+ ), DefNode(29...50)( (33...34), nil, nil, StatementsNode(36...45)( - [OrWriteNode(36...45)( - ConstantPathWriteNode(36...39)( - ConstantPathNode(36...39)( - nil, - ConstantReadNode(38...39)(), - (36...38) - ), + [ConstantPathOrWriteNode(36...45)( + ConstantPathNode(36...39)( nil, - nil + ConstantReadNode(38...39)(), + (36...38) ), - IntegerNode(44...45)(), - (40...43) + (40...43), + IntegerNode(44...45)() )] ), [], @@ -63,18 +51,14 @@ ProgramNode(0...77)( nil, nil, StatementsNode(59...72)( - [OrWriteNode(59...72)( - ConstantPathWriteNode(59...66)( - ConstantPathNode(59...66)( - SelfNode(59...63)(), - ConstantReadNode(65...66)(), - (63...65) - ), - nil, - nil + [ConstantPathOrWriteNode(59...72)( + ConstantPathNode(59...66)( + SelfNode(59...63)(), + ConstantReadNode(65...66)(), + (63...65) ), - IntegerNode(71...72)(), - (67...70) + (67...70), + IntegerNode(71...72)() )] ), [], diff --git a/test/yarp/snapshots/whitequark/op_asgn_cmd.txt b/test/yarp/snapshots/whitequark/op_asgn_cmd.txt index ccc8730d85e7c9..409aa26ee25c84 100644 --- a/test/yarp/snapshots/whitequark/op_asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/op_asgn_cmd.txt @@ -77,28 +77,13 @@ ProgramNode(0...64)( ), :+ ), - OperatorWriteNode(32...47)( - ConstantPathWriteNode(32...38)( - ConstantPathNode(32...38)( - CallNode(32...35)( - nil, - nil, - (32...35), - nil, - nil, - nil, - nil, - 2, - "foo" - ), - ConstantReadNode(37...38)(), - (35...37) - ), - nil, - nil + ConstantPathOperatorWriteNode(32...47)( + ConstantPathNode(32...38)( + CallNode(32...35)(nil, nil, (32...35), nil, nil, nil, nil, 2, "foo"), + ConstantReadNode(37...38)(), + (35...37) ), (39...41), - :+, CallNode(42...47)( nil, nil, @@ -121,7 +106,8 @@ ProgramNode(0...64)( nil, 0, "m" - ) + ), + :+ ), CallOperatorWriteNode(49...64)( CallNode(49...55)( diff --git a/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt b/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt index 8ae81caa8f5b1b..f9bfc7bfeb3649 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_op_assign.txt @@ -1,15 +1,17 @@ ProgramNode(0...22)( [:foo], StatementsNode(0...22)( - [OperatorWriteNode(0...22)( - LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), + [LocalVariableOperatorWriteNode(0...22)( + (0...3), (4...6), - :+, RescueModifierNode(7...22)( CallNode(7...11)(nil, nil, (7...11), nil, nil, nil, nil, 2, "meth"), (12...18), CallNode(19...22)(nil, nil, (19...22), nil, nil, nil, nil, 2, "bar") - ) + ), + :foo, + :+, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt index 4ca03bab0109ad..f2980c1d427a41 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt @@ -1,10 +1,9 @@ ProgramNode(0...437)( [:foo], StatementsNode(0...437)( - [OperatorWriteNode(0...27)( - LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), + [LocalVariableOperatorWriteNode(0...27)( + (0...3), (4...6), - :+, CallNode(7...27)( nil, nil, @@ -31,12 +30,14 @@ ProgramNode(0...437)( nil, 0, "raise" - ) + ), + :foo, + :+, + 0 ), - OperatorWriteNode(29...57)( - LocalVariableWriteNode(29...32)(:foo, 0, nil, (29...32), nil), + LocalVariableOperatorWriteNode(29...57)( + (29...32), (33...35), - :+, RescueModifierNode(36...57)( CallNode(36...46)( nil, @@ -63,7 +64,10 @@ ProgramNode(0...437)( ), (47...53), NilNode(54...57)() - ) + ), + :foo, + :+, + 0 ), LocalVariableWriteNode(59...85)( :foo, @@ -299,16 +303,13 @@ ProgramNode(0...437)( ), :+ ), - OrWriteNode(242...273)( - ConstantPathWriteNode(242...248)( - ConstantPathNode(242...248)( - LocalVariableReadNode(242...245)(:foo, 0), - ConstantReadNode(247...248)(), - (245...247) - ), - nil, - nil + ConstantPathOrWriteNode(242...273)( + ConstantPathNode(242...248)( + LocalVariableReadNode(242...245)(:foo, 0), + ConstantReadNode(247...248)(), + (245...247) ), + (249...252), CallNode(253...273)( nil, nil, @@ -335,19 +336,15 @@ ProgramNode(0...437)( nil, 0, "raise" - ), - (249...252) + ) ), - OrWriteNode(275...307)( - ConstantPathWriteNode(275...281)( - ConstantPathNode(275...281)( - LocalVariableReadNode(275...278)(:foo, 0), - ConstantReadNode(280...281)(), - (278...280) - ), - nil, - nil + ConstantPathOrWriteNode(275...307)( + ConstantPathNode(275...281)( + LocalVariableReadNode(275...278)(:foo, 0), + ConstantReadNode(280...281)(), + (278...280) ), + (282...285), RescueModifierNode(286...307)( CallNode(286...296)( nil, @@ -374,8 +371,7 @@ ProgramNode(0...437)( ), (297...303), NilNode(304...307)() - ), - (282...285) + ) ), CallOperatorWriteNode(309...339)( CallNode(309...315)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt index 6eb9e9a0c81ba0..f5ec48ce95ebb0 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt @@ -1,14 +1,12 @@ ProgramNode(0...74)( [:a, :b], StatementsNode(0...74)( - [OperatorWriteNode(0...18)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOperatorWriteNode(0...18)( + (0...1), (2...4), - :+, - OperatorWriteNode(5...18)( - LocalVariableWriteNode(5...6)(:b, 0, nil, (5...6), nil), + LocalVariableOperatorWriteNode(5...18)( + (5...6), (7...9), - :+, CallNode(10...18)( nil, nil, @@ -21,13 +19,18 @@ ProgramNode(0...74)( nil, 0, "raise" - ) - ) + ), + :b, + :+, + 0 + ), + :a, + :+, + 0 ), - OperatorWriteNode(20...37)( - LocalVariableWriteNode(20...21)(:a, 0, nil, (20...21), nil), + LocalVariableOperatorWriteNode(20...37)( + (20...21), (22...24), - :+, LocalVariableWriteNode(25...37)( :b, 0, @@ -46,15 +49,17 @@ ProgramNode(0...74)( ), (25...26), (27...28) - ) + ), + :a, + :+, + 0 ), LocalVariableWriteNode(39...56)( :a, 0, - OperatorWriteNode(43...56)( - LocalVariableWriteNode(43...44)(:b, 0, nil, (43...44), nil), + LocalVariableOperatorWriteNode(43...56)( + (43...44), (45...47), - :+, CallNode(48...56)( nil, nil, @@ -67,7 +72,10 @@ ProgramNode(0...74)( nil, 0, "raise" - ) + ), + :b, + :+, + 0 ), (39...40), (41...42) diff --git a/test/yarp/snapshots/whitequark/var_and_asgn.txt b/test/yarp/snapshots/whitequark/var_and_asgn.txt index a082413a7d05a7..a613761d87e163 100644 --- a/test/yarp/snapshots/whitequark/var_and_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_and_asgn.txt @@ -1,10 +1,12 @@ ProgramNode(0...7)( [:a], StatementsNode(0...7)( - [AndWriteNode(0...7)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableAndWriteNode(0...7)( + (0...1), + (2...5), IntegerNode(6...7)(), - (2...5) + :a, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/var_op_asgn.txt b/test/yarp/snapshots/whitequark/var_op_asgn.txt index 08b724031b7743..4591e68c96fbc8 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn.txt @@ -1,34 +1,36 @@ ProgramNode(0...53)( [:a], StatementsNode(0...53)( - [OperatorWriteNode(0...11)( - ClassVariableWriteNode(0...5)((0...5), nil, nil), + [ClassVariableOperatorWriteNode(0...11)( + (0...5), (6...8), - :|, - IntegerNode(9...11)() + IntegerNode(9...11)(), + :| ), - OperatorWriteNode(13...20)( - InstanceVariableWriteNode(13...15)((13...15), nil, nil), + InstanceVariableOperatorWriteNode(13...20)( + (13...15), (16...18), - :|, - IntegerNode(19...20)() + IntegerNode(19...20)(), + :| ), - OperatorWriteNode(22...28)( - LocalVariableWriteNode(22...23)(:a, 0, nil, (22...23), nil), + LocalVariableOperatorWriteNode(22...28)( + (22...23), (24...26), + IntegerNode(27...28)(), + :a, :+, - IntegerNode(27...28)() + 0 ), DefNode(30...53)( (34...35), nil, nil, StatementsNode(37...48)( - [OperatorWriteNode(37...48)( - ClassVariableWriteNode(37...42)((37...42), nil, nil), + [ClassVariableOperatorWriteNode(37...48)( + (37...42), (43...45), - :|, - IntegerNode(46...48)() + IntegerNode(46...48)(), + :| )] ), [], diff --git a/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt b/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt index f78329a1eba157..21c1fb6e1056a6 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn_cmd.txt @@ -1,10 +1,9 @@ ProgramNode(0...12)( [:foo], StatementsNode(0...12)( - [OperatorWriteNode(0...12)( - LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), + [LocalVariableOperatorWriteNode(0...12)( + (0...3), (4...6), - :+, CallNode(7...12)( nil, nil, @@ -15,7 +14,10 @@ ProgramNode(0...12)( nil, 0, "m" - ) + ), + :foo, + :+, + 0 )] ) ) diff --git a/test/yarp/snapshots/whitequark/var_or_asgn.txt b/test/yarp/snapshots/whitequark/var_or_asgn.txt index bb2f8c6b39cfaa..3fb56b7f4a88d8 100644 --- a/test/yarp/snapshots/whitequark/var_or_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_or_asgn.txt @@ -1,10 +1,12 @@ ProgramNode(0...7)( [:a], StatementsNode(0...7)( - [OrWriteNode(0...7)( - LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableOrWriteNode(0...7)( + (0...1), + (2...5), IntegerNode(6...7)(), - (2...5) + :a, + 0 )] ) ) diff --git a/yarp/config.yml b/yarp/config.yml index 3662c176098a50..24e8d0e227284d 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -399,19 +399,6 @@ nodes: left and right ^^^^^^^^^^^^^^ - - name: AndWriteNode - child_nodes: - - name: target - type: node - - name: value - type: node - - name: operator_loc - type: location - comment: | - Represents the use of the `&&=` operator. - - target &&= value - ^^^^^^^^^^^^^^^^ - name: ArgumentsNode child_nodes: - name: arguments @@ -738,6 +725,47 @@ nodes: class Foo end ^^^^^^^^^^^^^ + - name: ClassVariableAndWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a class variable. + + @@target &&= value + ^^^^^^^^^^^^^^^^ + - name: ClassVariableOperatorWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a class variable using an operator that isn't `=`. + + @@target += value + ^^^^^^^^^^^^^^^^^ + - name: ClassVariableOrWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a class variable. + + @@target ||= value + ^^^^^^^^^^^^^^^^^^ - name: ClassVariableReadNode comment: | Represents referencing a class variable. @@ -757,6 +785,61 @@ nodes: @@foo = 1 ^^^^^^^^^ + - name: ConstantAndWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a constant. + + Target &&= value + ^^^^^^^^^^^^^^^^ + - name: ConstantOperatorWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a constant using an operator that isn't `=`. + + Target += value + ^^^^^^^^^^^^^^^ + - name: ConstantOrWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a constant. + + Target ||= value + ^^^^^^^^^^^^^^^^ + - name: ConstantPathAndWriteNode + child_nodes: + - name: target + type: node + kind: ConstantPathNode + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a constant path. + + Parent::Child &&= value + ^^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathNode child_nodes: - name: parent @@ -770,6 +853,36 @@ nodes: Foo::Bar ^^^^^^^^ + - name: ConstantPathOperatorWriteNode + child_nodes: + - name: target + type: node + kind: ConstantPathNode + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a constant path using an operator that isn't `=`. + + Parent::Child += value + ^^^^^^^^^^^^^^^^^^^^^^ + - name: ConstantPathOrWriteNode + child_nodes: + - name: target + type: node + kind: ConstantPathNode + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a constant path. + + Parent::Child ||= value + ^^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathWriteNode child_nodes: - name: target @@ -1012,6 +1125,47 @@ nodes: super ^^^^^ + - name: GlobalVariableAndWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to a global variable. + + $target &&= value + ^^^^^^^^^^^^^^^^^ + - name: GlobalVariableOperatorWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to a global variable using an operator that isn't `=`. + + $target += value + ^^^^^^^^^^^^^^^^ + - name: GlobalVariableOrWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to a global variable. + + $target ||= value + ^^^^^^^^^^^^^^^^^ - name: GlobalVariableReadNode comment: | Represents referencing a global variable. @@ -1111,6 +1265,47 @@ nodes: case a; in b then c end ^^^^^^^^^^^ + - name: InstanceVariableAndWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `&&=` operator for assignment to an instance variable. + + @target &&= value + ^^^^^^^^^^^^^^^^^ + - name: InstanceVariableOperatorWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: operator + type: constant + comment: | + Represents assigning to an instance variable using an operator that isn't `=`. + + @target += value + ^^^^^^^^^^^^^^^^ + - name: InstanceVariableOrWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + comment: | + Represents the use of the `||=` operator for assignment to an instance variable. + + @target ||= value + ^^^^^^^^^^^^^^^^^ - name: InstanceVariableReadNode comment: | Represents referencing an instance variable. @@ -1248,6 +1443,59 @@ nodes: ->(value) { value * 2 } ^^^^^^^^^^^^^^^^^^^^^^^ + - name: LocalVariableAndWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: constant_id + type: constant + - name: depth + type: uint32 + comment: | + Represents the use of the `&&=` operator for assignment to a local variable. + + target &&= value + ^^^^^^^^^^^^^^^^ + - name: LocalVariableOperatorWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: constant_id + type: constant + - name: operator_id + type: constant + - name: depth + type: uint32 + comment: | + Represents assigning to a local variable using an operator that isn't `=`. + + target += value + ^^^^^^^^^^^^^^^ + - name: LocalVariableOrWriteNode + child_nodes: + - name: name_loc + type: location + - name: operator_loc + type: location + - name: value + type: node + - name: constant_id + type: constant + - name: depth + type: uint32 + comment: | + Represents the use of the `||=` operator for assignment to a local variable. + + target ||= value + ^^^^^^^^^^^^^^^^ - name: LocalVariableReadNode child_nodes: - name: constant_id @@ -1380,21 +1628,6 @@ nodes: $1 ^^ - - name: OperatorWriteNode - child_nodes: - - name: target - type: node - - name: operator_loc - type: location - - name: operator - type: constant - - name: value - type: node - comment: | - Represents the use of an operator on a write. - - target += value - ^^^^^^^^^^^^^^^ - name: OptionalParameterNode child_nodes: - name: constant_id @@ -1424,19 +1657,6 @@ nodes: left or right ^^^^^^^^^^^^^ - - name: OrWriteNode - child_nodes: - - name: target - type: node - - name: value - type: node - - name: operator_loc - type: location - comment: | - Represents the use of the `||=` operator. - - target ||= value - ^^^^^^^^^^^^^^^^ - name: ParametersNode child_nodes: - name: requireds diff --git a/yarp/yarp.c b/yarp/yarp.c index 012f8136e56c05..20d5628819134e 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -725,27 +725,6 @@ yp_and_node_create(yp_parser_t *parser, yp_node_t *left, const yp_token_t *opera return node; } -// Allocate and initialize a new AndWriteNode. -static yp_and_write_node_t * -yp_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_and_write_node_t); - - *node = (yp_and_write_node_t) { - { - .type = YP_NODE_AND_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - }, - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate an initialize a new arguments node. static yp_arguments_node_t * yp_arguments_node_create(yp_parser_t *parser) { @@ -1575,6 +1554,74 @@ yp_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const y return node; } +// Allocate and initialize a new ClassVariableAndWriteNode node. +static yp_class_variable_and_write_node_t * +yp_class_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_CLASS_VARIABLE_READ_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_class_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_and_write_node_t); + + *node = (yp_class_variable_and_write_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new ClassVariableOperatorWriteNode node. +static yp_class_variable_operator_write_node_t * +yp_class_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_class_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_operator_write_node_t); + + *node = (yp_class_variable_operator_write_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new ClassVariableOrWriteNode node. +static yp_class_variable_or_write_node_t * +yp_class_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_CLASS_VARIABLE_READ_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_class_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_or_write_node_t); + + *node = (yp_class_variable_or_write_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ClassVariableReadNode node. static yp_class_variable_read_node_t * yp_class_variable_read_node_create(yp_parser_t *parser, const yp_token_t *token) { @@ -1605,6 +1652,72 @@ yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp return node; } +// Allocate and initialize a new ConstantPathAndWriteNode node. +static yp_constant_path_and_write_node_t * +yp_constant_path_and_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_constant_path_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_and_write_node_t); + + *node = (yp_constant_path_and_write_node_t) { + { + .type = YP_NODE_CONSTANT_PATH_AND_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new ConstantPathOperatorWriteNode node. +static yp_constant_path_operator_write_node_t * +yp_constant_path_operator_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_constant_path_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_operator_write_node_t); + + *node = (yp_constant_path_operator_write_node_t) { + { + .type = YP_NODE_CONSTANT_PATH_OPERATOR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new ConstantPathOrWriteNode node. +static yp_constant_path_or_write_node_t * +yp_constant_path_or_write_node_create(yp_parser_t *parser, yp_constant_path_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_constant_path_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_path_or_write_node_t); + + *node = (yp_constant_path_or_write_node_t) { + { + .type = YP_NODE_CONSTANT_PATH_OR_WRITE_NODE, + .location = { + .start = target->base.location.start, + .end = value->location.end + } + }, + .target = target, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ConstantPathNode node. static yp_constant_path_node_t * yp_constant_path_node_create(yp_parser_t *parser, yp_node_t *parent, const yp_token_t *delimiter, yp_node_t *child) { @@ -1647,6 +1760,74 @@ yp_constant_path_write_node_create(yp_parser_t *parser, yp_constant_path_node_t return node; } +// Allocate and initialize a new ConstantAndWriteNode node. +static yp_constant_and_write_node_t * +yp_constant_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_CONSTANT_READ_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_constant_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_and_write_node_t); + + *node = (yp_constant_and_write_node_t) { + { + .type = YP_NODE_CONSTANT_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new ConstantOperatorWriteNode node. +static yp_constant_operator_write_node_t * +yp_constant_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_constant_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_operator_write_node_t); + + *node = (yp_constant_operator_write_node_t) { + { + .type = YP_NODE_CONSTANT_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new ConstantOrWriteNode node. +static yp_constant_or_write_node_t * +yp_constant_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_CONSTANT_READ_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_constant_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_constant_or_write_node_t); + + *node = (yp_constant_or_write_node_t) { + { + .type = YP_NODE_CONSTANT_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new ConstantReadNode node. static yp_constant_read_node_t * yp_constant_read_node_create(yp_parser_t *parser, const yp_token_t *name) { @@ -2081,6 +2262,74 @@ yp_hash_pattern_node_node_list_create(yp_parser_t *parser, yp_node_list_t *assoc return node; } +// Allocate and initialize a new GlobalVariableAndWriteNode node. +static yp_global_variable_and_write_node_t * +yp_global_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_GLOBAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_BACK_REFERENCE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_NUMBERED_REFERENCE_READ_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_global_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_and_write_node_t); + + *node = (yp_global_variable_and_write_node_t) { + { + .type = YP_NODE_GLOBAL_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new GlobalVariableOperatorWriteNode node. +static yp_global_variable_operator_write_node_t * +yp_global_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_global_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_operator_write_node_t); + + *node = (yp_global_variable_operator_write_node_t) { + { + .type = YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new GlobalVariableOrWriteNode node. +static yp_global_variable_or_write_node_t * +yp_global_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_GLOBAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_BACK_REFERENCE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_NUMBERED_REFERENCE_READ_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_global_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_global_variable_or_write_node_t); + + *node = (yp_global_variable_or_write_node_t) { + { + .type = YP_NODE_GLOBAL_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate a new GlobalVariableReadNode node. static yp_global_variable_read_node_t * yp_global_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name) { @@ -2371,6 +2620,74 @@ yp_in_node_create(yp_parser_t *parser, yp_node_t *pattern, yp_statements_node_t return node; } +// Allocate and initialize a new InstanceVariableAndWriteNode node. +static yp_instance_variable_and_write_node_t * +yp_instance_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_INSTANCE_VARIABLE_READ_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_instance_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_and_write_node_t); + + *node = (yp_instance_variable_and_write_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + +// Allocate and initialize a new InstanceVariableOperatorWriteNode node. +static yp_instance_variable_operator_write_node_t * +yp_instance_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + yp_instance_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_operator_write_node_t); + + *node = (yp_instance_variable_operator_write_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + }; + + return node; +} + +// Allocate and initialize a new InstanceVariableOrWriteNode node. +static yp_instance_variable_or_write_node_t * +yp_instance_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { + assert(YP_NODE_TYPE_P(target, YP_NODE_INSTANCE_VARIABLE_READ_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_instance_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_or_write_node_t); + + *node = (yp_instance_variable_or_write_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value + }; + + return node; +} + // Allocate and initialize a new InstanceVariableReadNode node. static yp_instance_variable_read_node_t * yp_instance_variable_read_node_create(yp_parser_t *parser, const yp_token_t *token) { @@ -2644,6 +2961,80 @@ yp_lambda_node_create( return node; } +// Allocate and initialize a new LocalVariableAndWriteNode node. +static yp_local_variable_and_write_node_t * +yp_local_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id, uint32_t depth) { + assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); + assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); + yp_local_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_and_write_node_t); + + *node = (yp_local_variable_and_write_node_t) { + { + .type = YP_NODE_LOCAL_VARIABLE_AND_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .constant_id = constant_id, + .depth = depth + }; + + return node; +} + +// Allocate and initialize a new LocalVariableOperatorWriteNode node. +static yp_local_variable_operator_write_node_t * +yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id, uint32_t depth) { + yp_local_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_operator_write_node_t); + + *node = (yp_local_variable_operator_write_node_t) { + { + .type = YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .constant_id = constant_id, + .operator_id = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), + .depth = depth + }; + + return node; +} + +// Allocate and initialize a new LocalVariableOrWriteNode node. +static yp_local_variable_or_write_node_t * +yp_local_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id, uint32_t depth) { + assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); + assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); + yp_local_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_or_write_node_t); + + *node = (yp_local_variable_or_write_node_t) { + { + .type = YP_NODE_LOCAL_VARIABLE_OR_WRITE_NODE, + .location = { + .start = target->location.start, + .end = value->location.end + } + }, + .name_loc = target->location, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), + .value = value, + .constant_id = constant_id, + .depth = depth + }; + + return node; +} + // Allocate a new LocalVariableReadNode node. static yp_local_variable_read_node_t * yp_local_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name, uint32_t depth) { @@ -2879,28 +3270,6 @@ yp_numbered_reference_read_node_create(yp_parser_t *parser, const yp_token_t *na return node; } -// Allocate and initialize a new OperatorWriteNode. -static yp_operator_write_node_t * -yp_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_operator_write_node_t); - - *node = (yp_operator_write_node_t) { - { - .type = YP_NODE_OPERATOR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - }, - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), - .value = value - }; - - return node; -} - // Allocate a new OptionalParameterNode node. static yp_optional_parameter_node_t * yp_optional_parameter_node_create(yp_parser_t *parser, const yp_token_t *name, const yp_token_t *operator, yp_node_t *value) { @@ -2944,27 +3313,6 @@ yp_or_node_create(yp_parser_t *parser, yp_node_t *left, const yp_token_t *operat return node; } -// Allocate and initialize a new OrWriteNode. -static yp_or_write_node_t * -yp_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - yp_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_or_write_node_t); - - *node = (yp_or_write_node_t) { - { - .type = YP_NODE_OR_WRITE_NODE, - .location = { - .start = target->location.start, - .end = value->location.end - }, - }, - .target = target, - .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), - .value = value - }; - - return node; -} - // Allocate and initialize a new ParametersNode node. static yp_parameters_node_t * yp_parameters_node_create(yp_parser_t *parser) { @@ -4074,13 +4422,15 @@ yp_parser_local_depth(yp_parser_t *parser, yp_token_t *token) { } // Add a local variable from a location to the current scope. -static void +static yp_constant_id_t yp_parser_local_add_location(yp_parser_t *parser, const char *start, const char *end) { yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, start, end); if (!yp_constant_id_list_includes(&parser->current_scope->locals, constant_id)) { yp_constant_id_list_append(&parser->current_scope->locals, constant_id); } + + return constant_id; } // Add a local variable from a token to the current scope. @@ -12259,19 +12609,57 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_CLASS_VARIABLE_READ_NODE: - case YP_NODE_CONSTANT_PATH_NODE: - case YP_NODE_CONSTANT_READ_NODE: - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { parser_lex(parser); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_global_variable_and_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_class_variable_and_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CONSTANT_PATH_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + return (yp_node_t *) yp_constant_path_and_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); + } + case YP_NODE_CONSTANT_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_constant_and_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); + yp_node_t *result = (yp_node_t *) yp_instance_variable_and_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *cast = (yp_local_variable_read_node_t *) node; + parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_and_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_and_write_node_create(parser, node, &token, value, cast->constant_id, cast->depth); + + yp_node_destroy(parser, node); + return result; } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12281,19 +12669,18 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t // will transform it into a local variable write. if (yp_call_node_variable_call_p(call_node)) { yp_location_t message_loc = call_node->message_loc; - yp_parser_local_add_location(parser, message_loc.start, message_loc.end); + yp_constant_id_t constant_id = yp_parser_local_add_location(parser, message_loc.start, message_loc.end); if (token_is_numbered_parameter(message_loc.start, message_loc.end)) { yp_diagnostic_list_append(&parser->error_list, message_loc.start, message_loc.end, "reserved for numbered parameter"); } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_and_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_and_write_node_create(parser, node, &token, value, constant_id, 0); + + yp_node_destroy(parser, node); + return result; } parser_lex(parser); @@ -12325,19 +12712,57 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_CLASS_VARIABLE_READ_NODE: - case YP_NODE_CONSTANT_PATH_NODE: - case YP_NODE_CONSTANT_READ_NODE: - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { parser_lex(parser); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_global_variable_or_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_class_variable_or_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CONSTANT_PATH_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + return (yp_node_t *) yp_constant_path_or_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); + } + case YP_NODE_CONSTANT_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_constant_or_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); + yp_node_t *result = (yp_node_t *) yp_instance_variable_or_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *cast = (yp_local_variable_read_node_t *) node; + parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - return (yp_node_t *) yp_or_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_or_write_node_create(parser, node, &token, value, cast->constant_id, cast->depth); + + yp_node_destroy(parser, node); + return result; } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12347,19 +12772,18 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t // will transform it into a local variable write. if (yp_call_node_variable_call_p(call_node)) { yp_location_t message_loc = call_node->message_loc; - yp_parser_local_add_location(parser, message_loc.start, message_loc.end); + yp_constant_id_t constant_id = yp_parser_local_add_location(parser, message_loc.start, message_loc.end); if (token_is_numbered_parameter(message_loc.start, message_loc.end)) { yp_diagnostic_list_append(&parser->error_list, message_loc.start, message_loc.end, "reserved for numbered parameter"); } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - return (yp_node_t *) yp_or_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_local_variable_or_write_node_create(parser, node, &token, value, constant_id, 0); + + yp_node_destroy(parser, node); + return result; } parser_lex(parser); @@ -12401,19 +12825,57 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_NODE_NUMBERED_REFERENCE_READ_NODE: yp_diagnostic_list_append(&parser->error_list, node->location.start, node->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_CLASS_VARIABLE_READ_NODE: - case YP_NODE_CONSTANT_PATH_NODE: - case YP_NODE_CONSTANT_READ_NODE: - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_global_variable_operator_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_class_variable_operator_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_CONSTANT_PATH_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + return (yp_node_t *) yp_constant_path_operator_write_node_create(parser, (yp_constant_path_node_t *) node, &token, value); + } + case YP_NODE_CONSTANT_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_constant_operator_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + parser_lex(parser); + + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_write_node_create(parser, node, &token, value); + + yp_node_destroy(parser, node); + return result; + } case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *cast = (yp_local_variable_read_node_t *) node; parser_lex(parser); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, cast->constant_id, cast->depth); - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator"); - return (yp_node_t *) yp_operator_write_node_create(parser, node, &token, value); + yp_node_destroy(parser, node); + return result; } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -12423,19 +12885,18 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t // will transform it into a local variable write. if (yp_call_node_variable_call_p(call_node)) { yp_location_t message_loc = call_node->message_loc; - yp_parser_local_add_location(parser, message_loc.start, message_loc.end); + yp_constant_id_t constant_id = yp_parser_local_add_location(parser, message_loc.start, message_loc.end); if (token_is_numbered_parameter(message_loc.start, message_loc.end)) { yp_diagnostic_list_append(&parser->error_list, message_loc.start, message_loc.end, "reserved for numbered parameter"); } parser_lex(parser); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); + yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, constant_id, 0); - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - - yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - return (yp_node_t *) yp_operator_write_node_create(parser, node, &token, value); + yp_node_destroy(parser, node); + return result; } yp_token_t operator = not_provided(parser); From 2ebaf077f61a078881a2c6f72355c69ba04f57e3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 11:22:05 -0400 Subject: [PATCH 003/208] [ruby/yarp] Provide a desugar visitor https://github.com/ruby/yarp/commit/9fad513089 --- lib/yarp.rb | 11 + lib/yarp/desugar_visitor.rb | 267 ++++++++++++++++++ lib/yarp/yarp.gemspec | 1 + test/desugar_visitor_test.rb | 57 ++++ .../lib/yarp/mutation_visitor.rb.erb | 2 +- yarp/templates/lib/yarp/node.rb.erb | 2 +- 6 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 lib/yarp/desugar_visitor.rb create mode 100644 test/desugar_visitor_test.rb diff --git a/lib/yarp.rb b/lib/yarp.rb index a3b0c3b0742fa5..b2d4a6eee489ad 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -52,6 +52,16 @@ def initialize(source, start_offset, length) @length = length end + # Create a new location object with the given options. + def copy(**options) + Location.new( + options.fetch(:source) { source }, + options.fetch(:start_offset) { start_offset }, + options.fetch(:length) { length } + ) + end + + # Returns a string representation of this location. def inspect "#" end @@ -508,6 +518,7 @@ def self.parse_serialize_file(filepath) require_relative "yarp/lex_compat" require_relative "yarp/mutation_visitor" +require_relative "yarp/desugar_visitor" require_relative "yarp/node" require_relative "yarp/ripper_compat" require_relative "yarp/serialize" diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb new file mode 100644 index 00000000000000..6e1831e98e8350 --- /dev/null +++ b/lib/yarp/desugar_visitor.rb @@ -0,0 +1,267 @@ +# frozen_string_literal: true + +module YARP + class DesugarVisitor < MutationVisitor + # @@foo &&= bar + # + # becomes + # + # @@foo && @@foo = bar + def visit_class_variable_and_write_node(node) + AndNode.new( + ClassVariableReadNode.new(node.name_loc), + ClassVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # @@foo ||= bar + # + # becomes + # + # @@foo || @@foo = bar + def visit_class_variable_or_write_node(node) + OrNode.new( + ClassVariableReadNode.new(node.name_loc), + ClassVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # @@foo += bar + # + # becomes + # + # @@foo = @@foo + bar + def visit_class_variable_operator_write_node(node) + desugar_operator_write_node(node, ClassVariableWriteNode, ClassVariableReadNode) + end + + # Foo &&= bar + # + # becomes + # + # Foo && Foo = bar + def visit_constant_and_write_node(node) + AndNode.new( + ConstantReadNode.new(node.name_loc), + ConstantWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Foo ||= bar + # + # becomes + # + # Foo || Foo = bar + def visit_constant_or_write_node(node) + OrNode.new( + ConstantReadNode.new(node.name_loc), + ConstantWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Foo += bar + # + # becomes + # + # Foo = Foo + bar + def visit_constant_operator_write_node(node) + desugar_operator_write_node(node, ConstantWriteNode, ConstantReadNode) + end + + # Foo::Bar &&= baz + # + # becomes + # + # Foo::Bar && Foo::Bar = baz + def visit_constant_path_and_write_node(node) + AndNode.new( + node.target, + ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Foo::Bar ||= baz + # + # becomes + # + # Foo::Bar || Foo::Bar = baz + def visit_constant_path_or_write_node(node) + OrNode.new( + node.target, + ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Foo::Bar += baz + # + # becomes + # + # Foo::Bar = Foo::Bar + baz + def visit_constant_path_operator_write_node(node) + ConstantPathWriteNode.new( + node.target, + CallNode.new( + node.target, + nil, + node.operator_loc.copy(length: node.operator_loc.length - 1), + nil, + ArgumentsNode.new([node.value], node.value.location), + nil, + nil, + 0, + node.operator_loc.slice.chomp("="), + node.location + ), + node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), + node.location + ) + end + + # $foo &&= bar + # + # becomes + # + # $foo && $foo = bar + def visit_global_variable_and_write_node(node) + AndNode.new( + GlobalVariableReadNode.new(node.name_loc), + GlobalVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # $foo ||= bar + # + # becomes + # + # $foo || $foo = bar + def visit_global_variable_or_write_node(node) + OrNode.new( + GlobalVariableReadNode.new(node.name_loc), + GlobalVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # $foo += bar + # + # becomes + # + # $foo = $foo + bar + def visit_global_variable_operator_write_node(node) + desugar_operator_write_node(node, GlobalVariableWriteNode, GlobalVariableReadNode) + end + + # @foo &&= bar + # + # becomes + # + # @foo && @foo = bar + def visit_instance_variable_and_write_node(node) + AndNode.new( + InstanceVariableReadNode.new(node.name_loc), + InstanceVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # @foo ||= bar + # + # becomes + # + # @foo || @foo = bar + def visit_instance_variable_or_write_node(node) + OrNode.new( + InstanceVariableReadNode.new(node.name_loc), + InstanceVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # @foo += bar + # + # becomes + # + # @foo = @foo + bar + def visit_instance_variable_operator_write_node(node) + desugar_operator_write_node(node, InstanceVariableWriteNode, InstanceVariableReadNode) + end + + # foo &&= bar + # + # becomes + # + # foo && foo = bar + def visit_local_variable_and_write_node(node) + AndNode.new( + LocalVariableReadNode.new(node.constant_id, node.depth, node.name_loc), + LocalVariableWriteNode.new(node.constant_id, node.depth, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # foo ||= bar + # + # becomes + # + # foo || foo = bar + def visit_local_variable_or_write_node(node) + OrNode.new( + LocalVariableReadNode.new(node.constant_id, node.depth, node.name_loc), + LocalVariableWriteNode.new(node.constant_id, node.depth, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # foo += bar + # + # becomes + # + # foo = foo + bar + def visit_local_variable_operator_write_node(node) + desugar_operator_write_node(node, LocalVariableWriteNode, LocalVariableReadNode, arguments: [node.constant_id, node.depth]) + end + + private + + # Desugar `x += y` to `x = x + y` + def desugar_operator_write_node(node, write_class, read_class, arguments: []) + write_class.new( + *arguments, + node.name_loc, + CallNode.new( + read_class.new(*arguments, node.name_loc), + nil, + node.operator_loc.copy(length: node.operator_loc.length - 1), + nil, + ArgumentsNode.new([node.value], node.value.location), + nil, + nil, + 0, + node.operator_loc.slice.chomp("="), + node.location + ), + node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), + node.location + ) + end + end +end diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index 70b6c7ebaa374f..fceda5b1a0ac86 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -59,6 +59,7 @@ Gem::Specification.new do |spec| "include/yarp/util/yp_strpbrk.h", "include/yarp/version.h", "lib/yarp.rb", + "lib/yarp/desugar_visitor.rb", "lib/yarp/ffi.rb", "lib/yarp/lex_compat.rb", "lib/yarp/mutation_visitor.rb", diff --git a/test/desugar_visitor_test.rb b/test/desugar_visitor_test.rb new file mode 100644 index 00000000000000..3af3d9deb43a08 --- /dev/null +++ b/test/desugar_visitor_test.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require "yarp_test_helper" + +class DesugarVisitorTest < Test::Unit::TestCase + def test_and_write + assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") + assert_desugars("(AndNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar &&= baz") + assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") + assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") + assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") + end + + def test_or_write + assert_desugars("(OrNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo ||= bar") + assert_desugars("(OrNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar ||= baz") + assert_desugars("(OrNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo ||= bar") + assert_desugars("(OrNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo ||= bar") + assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") + end + + def test_operator_write + assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") + assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") + assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") + assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") + assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") + end + + private + + def ast_inspect(node) + parts = [node.class.name.split("::").last] + + node.deconstruct_keys(nil).each do |_, value| + case value + when YARP::Node + parts << ast_inspect(value) + when Array + parts.concat(value.map { |element| ast_inspect(element) }) + end + end + + "(#{parts.join(" ")})" + end + + def assert_desugars(expected, source) + ast = YARP.parse(source).value.accept(YARP::DesugarVisitor.new) + assert_equal expected, ast_inspect(ast.statements.body.last) + end +end diff --git a/yarp/templates/lib/yarp/mutation_visitor.rb.erb b/yarp/templates/lib/yarp/mutation_visitor.rb.erb index c88cabbdbb030c..82e6bc32c07f31 100644 --- a/yarp/templates/lib/yarp/mutation_visitor.rb.erb +++ b/yarp/templates/lib/yarp/mutation_visitor.rb.erb @@ -9,7 +9,7 @@ module YARP def visit_<%= node.human %>(node) <%- params = node.params.select { |param| [NodeParam, OptionalNodeParam, NodeListParam].include?(param.class) } -%> <%- if params.any? -%> - node.copy(<%= params.map { |param| "#{param.name}: visit(node.#{param.name})" }.join(", ") %>) + node.copy(<%= params.map { |param| "#{param.name}: #{param.is_a?(NodeListParam) ? "visit_all" : "visit"}(node.#{param.name})" }.join(", ") %>) <%- else -%> node.copy <%- end -%> diff --git a/yarp/templates/lib/yarp/node.rb.erb b/yarp/templates/lib/yarp/node.rb.erb index 9bd9eb54e1d6ac..1a86350da889ac 100644 --- a/yarp/templates/lib/yarp/node.rb.erb +++ b/yarp/templates/lib/yarp/node.rb.erb @@ -52,7 +52,7 @@ module YARP def copy(**params) <%= node.name %>.new( <%- (node.params.map(&:name) + ["location"]).map do |name| -%> - <%= name %>: params.fetch(:<%= name %>) { self.<%= name %> }, + params.fetch(:<%= name %>) { <%= name %> }, <%- end -%> ) end From aeef7091096ec8e6ad44e431610d966db7a2c33f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 11:53:22 -0400 Subject: [PATCH 004/208] [ruby/yarp] Split up parse_target and parse_write https://github.com/ruby/yarp/commit/75d8bb93ea --- yarp/yarp.c | 208 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 166 insertions(+), 42 deletions(-) diff --git a/yarp/yarp.c b/yarp/yarp.c index 20d5628819134e..2b077f03855c82 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -7809,7 +7809,148 @@ parse_starred_expression(yp_parser_t *parser, yp_binding_power_t binding_power, // Convert the given node into a valid target node. static yp_node_t * -parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_node_t *value) { +parse_target(yp_parser_t *parser, yp_node_t *target) { + yp_token_t operator = not_provided(parser); + + switch (YP_NODE_TYPE(target)) { + case YP_NODE_MISSING_NODE: + return target; + case YP_NODE_CLASS_VARIABLE_READ_NODE: { + yp_class_variable_write_node_t *write_node = yp_class_variable_read_node_to_class_variable_write_node(parser, (yp_class_variable_read_node_t *) target, &operator, NULL); + yp_node_destroy(parser, target); + return (yp_node_t *) write_node; + } + case YP_NODE_CONSTANT_PATH_NODE: + return (yp_node_t *) yp_constant_path_write_node_create(parser, (yp_constant_path_node_t *) target, &operator, NULL); + case YP_NODE_CONSTANT_READ_NODE: { + yp_constant_write_node_t *node = yp_constant_write_node_create(parser, &target->location, &operator, NULL); + yp_node_destroy(parser, target); + + return (yp_node_t *) node; + } + case YP_NODE_BACK_REFERENCE_READ_NODE: + case YP_NODE_NUMBERED_REFERENCE_READ_NODE: + yp_diagnostic_list_append(&parser->error_list, target->location.start, target->location.end, "Can't set variable"); + /* fallthrough */ + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { + yp_global_variable_write_node_t *result = yp_global_variable_write_node_create(parser, &target->location, &operator, NULL); + yp_node_destroy(parser, target); + + return (yp_node_t *) result; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *local_read = (yp_local_variable_read_node_t *) target; + + yp_constant_id_t constant_id = local_read->constant_id; + uint32_t depth = local_read->depth; + + yp_location_t name_loc = target->location; + yp_node_destroy(parser, target); + + return (yp_node_t *) yp_local_variable_write_node_create(parser, constant_id, depth, NULL, &name_loc, &operator); + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + yp_node_t *write_node = (yp_node_t *) yp_instance_variable_write_node_create(parser, (yp_instance_variable_read_node_t *) target, &operator, NULL); + yp_node_destroy(parser, target); + return write_node; + } + case YP_NODE_MULTI_WRITE_NODE: { + yp_multi_write_node_t *multi_write = (yp_multi_write_node_t *) target; + yp_multi_write_node_operator_loc_set(multi_write, &operator); + return (yp_node_t *) multi_write; + } + case YP_NODE_SPLAT_NODE: { + yp_splat_node_t *splat = (yp_splat_node_t *) target; + + if (splat->expression != NULL) { + splat->expression = parse_target(parser, splat->expression); + } + + yp_location_t location = { .start = NULL, .end = NULL }; + yp_multi_write_node_t *multi_write = yp_multi_write_node_create(parser, &operator, NULL, &location, &location); + yp_multi_write_node_targets_append(multi_write, (yp_node_t *) splat); + + return (yp_node_t *) multi_write; + } + case YP_NODE_CALL_NODE: { + yp_call_node_t *call = (yp_call_node_t *) target; + // If we have no arguments to the call node and we need this to be a + // target then this is either a method call or a local variable write. + if ( + (call->opening_loc.start == NULL) && + (call->arguments == NULL) && + (call->block == NULL) + ) { + if (call->receiver == NULL) { + // When we get here, we have a local variable write, because it + // was previously marked as a method call but now we have an =. + // This looks like: + // + // foo = 1 + // + // When it was parsed in the prefix position, foo was seen as a + // method call with no receiver and no arguments. Now we have an + // =, so we know it's a local variable write. + const yp_location_t message = call->message_loc; + + yp_parser_local_add_location(parser, message.start, message.end); + yp_node_destroy(parser, target); + + yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, message.start, message.end); + target = (yp_node_t *) yp_local_variable_write_node_create(parser, constant_id, 0, NULL, &message, &operator); + + if (token_is_numbered_parameter(message.start, message.end)) { + yp_diagnostic_list_append(&parser->error_list, message.start, message.end, "reserved for numbered parameter"); + } + + return target; + } + + // The method name needs to change. If we previously had foo, we now + // need foo=. In this case we'll allocate a new owned string, copy + // the previous method name in, and append an =. + size_t length = yp_string_length(&call->name); + + char *name = calloc(length + 2, sizeof(char)); + if (name == NULL) return NULL; + + snprintf(name, length + 2, "%.*s=", (int) length, yp_string_source(&call->name)); + + // Now switch the name to the new string. + yp_string_free(&call->name); + yp_string_owned_init(&call->name, name, length + 1); + + return target; + } + + // If there is no call operator and the message is "[]" then this is + // an aref expression, and we can transform it into an aset + // expression. + if ( + (call->operator_loc.start == NULL) && + (call->message_loc.start[0] == '[') && + (call->message_loc.end[-1] == ']') && + (call->block == NULL) + ) { + // Free the previous name and replace it with "[]=". + yp_string_free(&call->name); + yp_string_constant_init(&call->name, "[]=", 3); + return target; + } + } + /* fallthrough */ + default: + // In this case we have a node that we don't know how to convert into a + // target. We need to treat it as an error. For now, we'll mark it as an + // error and just skip right past it. + yp_diagnostic_list_append(&parser->error_list, operator.start, operator.end, "Unexpected `='."); + return target; + } +} + +// Convert the given node into a valid write node. +static yp_node_t * +parse_write(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_node_t *value) { switch (YP_NODE_TYPE(target)) { case YP_NODE_MISSING_NODE: return target; @@ -7856,18 +7997,15 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no yp_multi_write_node_t *multi_write = (yp_multi_write_node_t *) target; yp_multi_write_node_operator_loc_set(multi_write, operator); - if (value != NULL) { - multi_write->value = value; - multi_write->base.location.end = value->location.end; - } - + multi_write->value = value; + multi_write->base.location.end = value->location.end; return (yp_node_t *) multi_write; } case YP_NODE_SPLAT_NODE: { yp_splat_node_t *splat = (yp_splat_node_t *) target; if (splat->expression != NULL) { - splat->expression = parse_target(parser, splat->expression, operator, value); + splat->expression = parse_write(parser, splat->expression, operator, value); } yp_location_t location = { .start = NULL, .end = NULL }; @@ -7920,12 +8058,10 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no // method call with no arguments. Now we have an =, so we know it's // a method call with an argument. In this case we will create the // arguments node, parse the argument, and add it to the list. - if (value) { - yp_arguments_node_t *arguments = yp_arguments_node_create(parser); - call->arguments = arguments; - yp_arguments_node_arguments_append(arguments, value); - target->location.end = arguments->base.location.end; - } + yp_arguments_node_t *arguments = yp_arguments_node_create(parser); + call->arguments = arguments; + yp_arguments_node_arguments_append(arguments, value); + target->location.end = arguments->base.location.end; // The method name needs to change. If we previously had foo, we now // need foo=. In this case we'll allocate a new owned string, copy @@ -7953,15 +8089,13 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no (call->message_loc.end[-1] == ']') && (call->block == NULL) ) { - if (value != NULL) { - if (call->arguments == NULL) { - call->arguments = yp_arguments_node_create(parser); - } - - yp_arguments_node_arguments_append(call->arguments, value); - target->location.end = value->location.end; + if (call->arguments == NULL) { + call->arguments = yp_arguments_node_create(parser); } + yp_arguments_node_arguments_append(call->arguments, value); + target->location.end = value->location.end; + // Free the previous name and replace it with "[]=". yp_string_free(&call->name); yp_string_constant_init(&call->name, "[]=", 3); @@ -7973,9 +8107,7 @@ parse_target(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_no // syntax error. In this case we'll fall through to our default // handling. We need to free the value that we parsed because there // is no way for us to attach it to the tree at this point. - if (value != NULL) { - yp_node_destroy(parser, value); - } + yp_node_destroy(parser, value); } /* fallthrough */ default: @@ -8003,7 +8135,7 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b // location that we know requires a multi write, as in the case of a for loop. // In this case we will set up the parsing loop slightly differently. if (first_target != NULL) { - first_target = parse_target(parser, first_target, &operator, NULL); + first_target = parse_target(parser, first_target); if (!match_type_p(parser, YP_TOKEN_COMMA)) { return first_target; @@ -8034,9 +8166,8 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b yp_node_t *name = NULL; if (token_begins_expression_p(parser->current.type)) { - yp_token_t operator = not_provided(parser); name = parse_expression(parser, binding_power, "Expected an expression after '*'."); - name = parse_target(parser, name, &operator, NULL); + name = parse_target(parser, name); } yp_node_t *splat = (yp_node_t *) yp_splat_node_create(parser, &star_operator, name); @@ -8104,7 +8235,7 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b } yp_node_t *target = parse_expression(parser, binding_power, "Expected another expression after ','."); - target = parse_target(parser, target, &operator, NULL); + target = parse_target(parser, target); yp_multi_write_node_targets_append(result, target); } @@ -8855,8 +8986,7 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_rescue_node_operator_set(rescue, &parser->previous); yp_node_t *reference = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected an exception variable after `=>` in rescue statement."); - yp_token_t operator = not_provided(parser); - reference = parse_target(parser, reference, &operator, NULL); + reference = parse_target(parser, reference); yp_rescue_node_reference_set(rescue, reference); break; @@ -8886,8 +9016,7 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_rescue_node_operator_set(rescue, &parser->previous); yp_node_t *reference = parse_expression(parser, YP_BINDING_POWER_INDEX, "Expected an exception variable after `=>` in rescue statement."); - yp_token_t operator = not_provided(parser); - reference = parse_target(parser, reference, &operator, NULL); + reference = parse_target(parser, reference); yp_rescue_node_reference_set(rescue, reference); break; @@ -12578,7 +12707,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_CASE_WRITABLE: { parser_lex(parser); yp_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, "Expected a value after =."); - return parse_target(parser, node, &token, value); + return parse_write(parser, node, &token, value); } case YP_NODE_SPLAT_NODE: { yp_splat_node_t *splat_node = (yp_splat_node_t *) node; @@ -12587,7 +12716,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t case YP_CASE_WRITABLE: parser_lex(parser); yp_node_t *value = parse_assignment_value(parser, previous_binding_power, binding_power, "Expected a value after =."); - return parse_target(parser, (yp_node_t *) splat_node, &token, value); + return parse_write(parser, (yp_node_t *) splat_node, &token, value); default: break; } @@ -12684,9 +12813,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + node = parse_target(parser, node); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); return (yp_node_t *) yp_call_operator_and_write_node_create(parser, (yp_call_node_t *) node, &token, value); @@ -12787,9 +12914,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t } parser_lex(parser); - - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); + node = parse_target(parser, node); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); return (yp_node_t *) yp_call_operator_or_write_node_create(parser, (yp_call_node_t *) node, &token, value); @@ -12899,10 +13024,9 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t return result; } - yp_token_t operator = not_provided(parser); - node = parse_target(parser, node, &operator, NULL); - + node = parse_target(parser, node); parser_lex(parser); + yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); return (yp_node_t *) yp_call_operator_write_node_create(parser, (yp_call_node_t *) node, &token, value); } From 7898b8e1ea01f3cc670543416d4d3ca90b3917aa Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 11:57:37 -0400 Subject: [PATCH 005/208] [ruby/yarp] Provide target node versions https://github.com/ruby/yarp/commit/a026564d38 --- test/yarp/location_test.rb | 68 ++++++++++-- test/yarp/snapshots/begin_rescue.txt | 10 +- test/yarp/snapshots/for.txt | 18 ++-- test/yarp/snapshots/seattlerb/dasgn_icky2.txt | 2 +- .../seattlerb/masgn_anon_splat_arg.txt | 2 +- .../seattlerb/masgn_arg_colon_arg.txt | 2 +- .../snapshots/seattlerb/masgn_arg_ident.txt | 2 +- .../seattlerb/masgn_arg_splat_arg.txt | 9 +- .../yarp/snapshots/seattlerb/masgn_colon2.txt | 14 +-- .../yarp/snapshots/seattlerb/masgn_colon3.txt | 12 +-- .../seattlerb/masgn_command_call.txt | 3 +- .../seattlerb/masgn_double_paren.txt | 4 +- .../snapshots/seattlerb/masgn_lhs_splat.txt | 5 +- test/yarp/snapshots/seattlerb/masgn_paren.txt | 4 +- .../snapshots/seattlerb/masgn_splat_arg.txt | 7 +- .../seattlerb/masgn_splat_arg_arg.txt | 9 +- .../seattlerb/masgn_var_star_var.txt | 4 +- .../seattlerb/mlhs_back_anonsplat.txt | 6 +- .../snapshots/seattlerb/mlhs_back_splat.txt | 11 +- .../seattlerb/mlhs_front_anonsplat.txt | 6 +- .../snapshots/seattlerb/mlhs_front_splat.txt | 11 +- .../seattlerb/mlhs_mid_anonsplat.txt | 12 +-- .../snapshots/seattlerb/mlhs_mid_splat.txt | 17 ++- test/yarp/snapshots/seattlerb/mlhs_rescue.txt | 4 +- .../snapshots/seattlerb/parse_line_to_ary.txt | 4 +- .../unparser/corpus/literal/assignment.txt | 52 ++++----- .../unparser/corpus/literal/block.txt | 20 +--- .../unparser/corpus/literal/defined.txt | 4 +- .../snapshots/unparser/corpus/literal/for.txt | 12 +-- .../unparser/corpus/literal/kwbegin.txt | 14 +-- .../unparser/corpus/literal/send.txt | 16 +-- test/yarp/snapshots/variables.txt | 26 ++--- .../snapshots/whitequark/and_or_masgn.txt | 8 +- .../snapshots/whitequark/cond_begin_masgn.txt | 4 +- test/yarp/snapshots/whitequark/for.txt | 4 +- test/yarp/snapshots/whitequark/for_mlhs.txt | 4 +- .../snapshots/whitequark/if_masgn__24.txt | 4 +- test/yarp/snapshots/whitequark/masgn.txt | 14 +-- test/yarp/snapshots/whitequark/masgn_attr.txt | 4 +- test/yarp/snapshots/whitequark/masgn_cmd.txt | 4 +- .../yarp/snapshots/whitequark/masgn_const.txt | 22 ++-- .../snapshots/whitequark/masgn_nested.txt | 8 +- .../yarp/snapshots/whitequark/masgn_splat.txt | 34 +++--- .../snapshots/whitequark/not_masgn__24.txt | 4 +- .../snapshots/whitequark/resbody_list_var.txt | 2 +- .../yarp/snapshots/whitequark/resbody_var.txt | 4 +- .../snapshots/whitequark/rescue_mod_masgn.txt | 4 +- yarp/config.yml | 48 +++++++++ yarp/yarp.c | 101 ++++++++---------- 49 files changed, 340 insertions(+), 323 deletions(-) diff --git a/test/yarp/location_test.rb b/test/yarp/location_test.rb index c7f9c0091bc420..424c9778f8b86e 100644 --- a/test/yarp/location_test.rb +++ b/test/yarp/location_test.rb @@ -215,6 +215,12 @@ def test_ClassVariableReadNode assert_location(ClassVariableReadNode, "@@foo") end + def test_ClassVariableTargetNode + assert_location(ClassVariableTargetNode, "@@foo, @@bar = baz", 0...5) do |node| + node.targets.first + end + end + def test_ClassVariableWriteNode assert_location(ClassVariableWriteNode, "@@foo = bar") end @@ -232,11 +238,17 @@ def test_ConstantPathNode def test_ConstantPathOperatorWriteNode assert_location(ConstantPathOperatorWriteNode, "Parent::Child += bar") end - + def test_ConstantPathOrWriteNode assert_location(ConstantPathOrWriteNode, "Parent::Child ||= bar") end + def test_ConstantPathTargetNode + assert_location(ConstantPathTargetNode, "::Foo, ::Bar = baz", 0...5) do |node| + node.targets.first + end + end + def test_ConstantPathWriteNode assert_location(ConstantPathWriteNode, "Foo::Bar = baz") assert_location(ConstantPathWriteNode, "::Foo = bar") @@ -246,11 +258,11 @@ def test_ConstantPathWriteNode def test_ConstantAndWriteNode assert_location(ConstantAndWriteNode, "Foo &&= bar") end - + def test_ConstantOperatorWriteNode assert_location(ConstantOperatorWriteNode, "Foo += bar") end - + def test_ConstantOrWriteNode assert_location(ConstantOrWriteNode, "Foo ||= bar") end @@ -260,6 +272,16 @@ def test_ConstantReadNode assert_location(ConstantReadNode, "Foo::Bar", 5...8, &:child) end + def test_ConstantTargetNode + assert_location(ConstantTargetNode, "Foo, Bar = baz", 0...3) do |node| + node.targets.first + end + end + + def test_ConstantWriteNode + assert_location(ConstantWriteNode, "Foo = bar") + end + def test_DefNode assert_location(DefNode, "def foo; bar; end") assert_location(DefNode, "def foo = bar") @@ -299,6 +321,10 @@ def test_FindPatternNode end end + def test_FlipFlopNode + assert_location(FlipFlopNode, "if foo..bar; end", 3..11, &:predicate) + end + def test_FloatNode assert_location(FloatNode, "0.0") assert_location(FloatNode, "1.0") @@ -331,11 +357,11 @@ def test_ForwardingSuperNode def test_GlobalVariableAndWriteNode assert_location(GlobalVariableAndWriteNode, "$foo &&= bar") end - + def test_GlobalVariableOperatorWriteNode assert_location(GlobalVariableOperatorWriteNode, "$foo += bar") end - + def test_GlobalVariableOrWriteNode assert_location(GlobalVariableOrWriteNode, "$foo ||= bar") end @@ -344,6 +370,12 @@ def test_GlobalVariableReadNode assert_location(GlobalVariableReadNode, "$foo") end + def test_GlobalVariableTargetNode + assert_location(GlobalVariableTargetNode, "$foo, $bar = baz", 0...4) do |node| + node.targets.first + end + end + def test_GlobalVariableWriteNode assert_location(GlobalVariableWriteNode, "$foo = bar") end @@ -377,11 +409,11 @@ def test_InNode def test_InstanceVariableAndWriteNode assert_location(InstanceVariableAndWriteNode, "@foo &&= bar") end - + def test_InstanceVariableOperatorWriteNode assert_location(InstanceVariableOperatorWriteNode, "@foo += bar") end - + def test_InstanceVariableOrWriteNode assert_location(InstanceVariableOrWriteNode, "@foo ||= bar") end @@ -390,6 +422,12 @@ def test_InstanceVariableReadNode assert_location(InstanceVariableReadNode, "@foo") end + def test_InstanceVariableTargetNode + assert_location(InstanceVariableTargetNode, "@foo, @bar = baz", 0...4) do |node| + node.targets.first + end + end + def test_InstanceVariableWriteNode assert_location(InstanceVariableWriteNode, "@foo = bar") end @@ -456,12 +494,12 @@ def test_LocalVariableAndWriteNode assert_location(LocalVariableAndWriteNode, "foo &&= bar") assert_location(LocalVariableAndWriteNode, "foo = 1; foo &&= bar", 9...20) end - + def test_LocalVariableOperatorWriteNode assert_location(LocalVariableOperatorWriteNode, "foo += bar") assert_location(LocalVariableOperatorWriteNode, "foo = 1; foo += bar", 9...19) end - + def test_LocalVariableOrWriteNode assert_location(LocalVariableOrWriteNode, "foo ||= bar") assert_location(LocalVariableOrWriteNode, "foo = 1; foo ||= bar", 9...20) @@ -471,6 +509,12 @@ def test_LocalVariableReadNode assert_location(LocalVariableReadNode, "foo = 1; foo", 9...12) end + def test_LocalVariableTargetNode + assert_location(LocalVariableTargetNode, "foo, bar = baz", 0...3) do |node| + node.targets.first + end + end + def test_LocalVariableWriteNode assert_location(LocalVariableWriteNode, "foo = bar") end @@ -751,6 +795,12 @@ def test_YieldNode assert_location(YieldNode, "yield(foo)") end + def test_all_tested + expected = YARP.constants.grep(/.Node$/).sort - %i[MissingNode ProgramNode] + actual = LocationTest.instance_methods(false).grep(/.Node$/).map { |name| name[5..].to_sym }.sort + assert_equal expected, actual + end + private def assert_location(kind, source, expected = 0...source.length) diff --git a/test/yarp/snapshots/begin_rescue.txt b/test/yarp/snapshots/begin_rescue.txt index f7195f80a08d54..4c902e53509898 100644 --- a/test/yarp/snapshots/begin_rescue.txt +++ b/test/yarp/snapshots/begin_rescue.txt @@ -223,7 +223,7 @@ ProgramNode(0...578)( (187...193), [ConstantReadNode(194...203)()], (204...206), - LocalVariableWriteNode(207...209)(:ex, 0, nil, (207...209), nil), + LocalVariableTargetNode(207...209)(:ex, 0), StatementsNode(212...213)( [CallNode(212...213)( nil, @@ -241,7 +241,7 @@ ProgramNode(0...578)( (214...220), [ConstantReadNode(221...237)(), ConstantReadNode(239...255)()], (256...258), - LocalVariableWriteNode(259...261)(:ex, 0, nil, (259...261), nil), + LocalVariableTargetNode(259...261)(:ex, 0), StatementsNode(264...265)( [CallNode(264...265)( nil, @@ -281,7 +281,7 @@ ProgramNode(0...578)( (281...287), [ConstantReadNode(288...297)()], (298...300), - LocalVariableWriteNode(301...303)(:ex, 0, nil, (301...303), nil), + LocalVariableTargetNode(301...303)(:ex, 0), StatementsNode(306...307)( [CallNode(306...307)( nil, @@ -532,7 +532,7 @@ ProgramNode(0...578)( (489...495), [ConstantReadNode(496...505)(), ConstantReadNode(507...522)()], (523...525), - LocalVariableWriteNode(526...528)(:ex, 0, nil, (526...528), nil), + LocalVariableTargetNode(526...528)(:ex, 0), StatementsNode(531...532)( [CallNode(531...532)( nil, @@ -571,7 +571,7 @@ ProgramNode(0...578)( (548...554), [ConstantReadNode(555...564)()], (565...567), - LocalVariableWriteNode(568...570)(:ex, 0, nil, (568...570), nil), + LocalVariableTargetNode(568...570)(:ex, 0), StatementsNode(573...574)( [CallNode(573...574)( nil, diff --git a/test/yarp/snapshots/for.txt b/test/yarp/snapshots/for.txt index 0bd8319aa1af36..53eb1caf581484 100644 --- a/test/yarp/snapshots/for.txt +++ b/test/yarp/snapshots/for.txt @@ -3,7 +3,7 @@ ProgramNode(0...143)( StatementsNode(0...143)( [ForNode(0...20)( MultiWriteNode(4...5)( - [LocalVariableWriteNode(4...5)(:i, 0, nil, (4...5), nil)], + [LocalVariableTargetNode(4...5)(:i, 0)], nil, nil, nil, @@ -23,7 +23,7 @@ ProgramNode(0...143)( ), ForNode(22...44)( MultiWriteNode(26...27)( - [LocalVariableWriteNode(26...27)(:i, 0, nil, (26...27), nil)], + [LocalVariableTargetNode(26...27)(:i, 0)], nil, nil, nil, @@ -43,8 +43,8 @@ ProgramNode(0...143)( ), ForNode(46...68)( MultiWriteNode(50...53)( - [LocalVariableWriteNode(50...51)(:i, 0, nil, (50...51), nil), - LocalVariableWriteNode(52...53)(:j, 0, nil, (52...53), nil)], + [LocalVariableTargetNode(50...51)(:i, 0), + LocalVariableTargetNode(52...53)(:j, 0)], nil, nil, nil, @@ -64,9 +64,9 @@ ProgramNode(0...143)( ), ForNode(70...94)( MultiWriteNode(74...79)( - [LocalVariableWriteNode(74...75)(:i, 0, nil, (74...75), nil), - LocalVariableWriteNode(76...77)(:j, 0, nil, (76...77), nil), - LocalVariableWriteNode(78...79)(:k, 0, nil, (78...79), nil)], + [LocalVariableTargetNode(74...75)(:i, 0), + LocalVariableTargetNode(76...77)(:j, 0), + LocalVariableTargetNode(78...79)(:k, 0)], nil, nil, nil, @@ -86,7 +86,7 @@ ProgramNode(0...143)( ), ForNode(96...119)( MultiWriteNode(100...101)( - [LocalVariableWriteNode(100...101)(:i, 0, nil, (100...101), nil)], + [LocalVariableTargetNode(100...101)(:i, 0)], nil, nil, nil, @@ -106,7 +106,7 @@ ProgramNode(0...143)( ), ForNode(121...143)( MultiWriteNode(125...126)( - [LocalVariableWriteNode(125...126)(:i, 0, nil, (125...126), nil)], + [LocalVariableTargetNode(125...126)(:i, 0)], nil, nil, nil, diff --git a/test/yarp/snapshots/seattlerb/dasgn_icky2.txt b/test/yarp/snapshots/seattlerb/dasgn_icky2.txt index 26d5ca491c6b6e..2aba42cc791ac3 100644 --- a/test/yarp/snapshots/seattlerb/dasgn_icky2.txt +++ b/test/yarp/snapshots/seattlerb/dasgn_icky2.txt @@ -28,7 +28,7 @@ ProgramNode(0...76)( (35...41), [ConstantReadNode(42...51)()], (52...54), - LocalVariableWriteNode(55...56)(:v, 0, nil, (55...56), nil), + LocalVariableTargetNode(55...56)(:v, 0), StatementsNode(61...66)([BreakNode(61...66)(nil, (61...66))]), nil ), diff --git a/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt b/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt index 738ee25165893a..c57ea8778fa388 100644 --- a/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_anon_splat_arg.txt @@ -9,7 +9,7 @@ ProgramNode(0...8)( nil, nil ), - LocalVariableWriteNode(3...4)(:a, 0, nil, (3...4), nil)], + LocalVariableTargetNode(3...4)(:a, 0)], (5...6), CallNode(7...8)(nil, nil, (7...8), nil, nil, nil, nil, 2, "b"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt b/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt index 1b1c0f43badc08..1a8275332e534b 100644 --- a/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_arg_colon_arg.txt @@ -2,7 +2,7 @@ ProgramNode(0...11)( [:a], StatementsNode(0...11)( [MultiWriteNode(0...11)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableTargetNode(0...1)(:a, 0), CallNode(3...7)( CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), (4...6), diff --git a/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt b/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt index 53ecd8fd183fc6..da590680c034a3 100644 --- a/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt +++ b/test/yarp/snapshots/seattlerb/masgn_arg_ident.txt @@ -2,7 +2,7 @@ ProgramNode(0...10)( [:a], StatementsNode(0...10)( [MultiWriteNode(0...10)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableTargetNode(0...1)(:a, 0), CallNode(3...6)( CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), (4...5), diff --git a/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt b/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt index 285c22f660ba87..c22781a5d01f29 100644 --- a/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_arg_splat_arg.txt @@ -2,12 +2,9 @@ ProgramNode(0...12)( [:a, :b, :c], StatementsNode(0...12)( [MultiWriteNode(0...12)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - SplatNode(3...5)( - (3...4), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil) - ), - LocalVariableWriteNode(7...8)(:c, 0, nil, (7...8), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + SplatNode(3...5)((3...4), LocalVariableTargetNode(4...5)(:b, 0)), + LocalVariableTargetNode(7...8)(:c, 0)], (9...10), CallNode(11...12)(nil, nil, (11...12), nil, nil, nil, nil, 2, "d"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_colon2.txt b/test/yarp/snapshots/seattlerb/masgn_colon2.txt index 490d1657b03762..99288bf229f285 100644 --- a/test/yarp/snapshots/seattlerb/masgn_colon2.txt +++ b/test/yarp/snapshots/seattlerb/masgn_colon2.txt @@ -2,15 +2,11 @@ ProgramNode(0...14)( [:a], StatementsNode(0...14)( [MultiWriteNode(0...14)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - ConstantPathWriteNode(3...7)( - ConstantPathNode(3...7)( - CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), - ConstantReadNode(6...7)(), - (4...6) - ), - nil, - nil + [LocalVariableTargetNode(0...1)(:a, 0), + ConstantPathTargetNode(3...7)( + CallNode(3...4)(nil, nil, (3...4), nil, nil, nil, nil, 2, "b"), + ConstantReadNode(6...7)(), + (4...6) )], (8...9), ArrayNode(10...14)( diff --git a/test/yarp/snapshots/seattlerb/masgn_colon3.txt b/test/yarp/snapshots/seattlerb/masgn_colon3.txt index f02dcc1f50756e..abb7125505e734 100644 --- a/test/yarp/snapshots/seattlerb/masgn_colon3.txt +++ b/test/yarp/snapshots/seattlerb/masgn_colon3.txt @@ -2,15 +2,11 @@ ProgramNode(0...15)( [], StatementsNode(0...15)( [MultiWriteNode(0...15)( - [ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + [ConstantPathTargetNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + ConstantPathTargetNode(5...8)( nil, - nil - ), - ConstantPathWriteNode(5...8)( - ConstantPathNode(5...8)(nil, ConstantReadNode(7...8)(), (5...7)), - nil, - nil + ConstantReadNode(7...8)(), + (5...7) )], (9...10), ArrayNode(11...15)( diff --git a/test/yarp/snapshots/seattlerb/masgn_command_call.txt b/test/yarp/snapshots/seattlerb/masgn_command_call.txt index ef2eab9d34ed0a..edb11e8dc07c49 100644 --- a/test/yarp/snapshots/seattlerb/masgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/masgn_command_call.txt @@ -2,8 +2,7 @@ ProgramNode(0...10)( [:a], StatementsNode(0...10)( [MultiWriteNode(0...10)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - SplatNode(1...2)((1...2), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), SplatNode(1...2)((1...2), nil)], (3...4), CallNode(5...10)( CallNode(5...6)(nil, nil, (5...6), nil, nil, nil, nil, 2, "b"), diff --git a/test/yarp/snapshots/seattlerb/masgn_double_paren.txt b/test/yarp/snapshots/seattlerb/masgn_double_paren.txt index 4f19128a219d20..72f0237aba9465 100644 --- a/test/yarp/snapshots/seattlerb/masgn_double_paren.txt +++ b/test/yarp/snapshots/seattlerb/masgn_double_paren.txt @@ -3,8 +3,8 @@ ProgramNode(2...9)( StatementsNode(2...9)( [MultiWriteNode(2...9)( [MultiWriteNode(2...5)( - [LocalVariableWriteNode(2...3)(:a, 0, nil, (2...3), nil), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil)], + [LocalVariableTargetNode(2...3)(:a, 0), + LocalVariableTargetNode(4...5)(:b, 0)], nil, nil, (1...2), diff --git a/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt b/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt index e8606e7152857b..0fe5b400fae43b 100644 --- a/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt +++ b/test/yarp/snapshots/seattlerb/masgn_lhs_splat.txt @@ -2,10 +2,7 @@ ProgramNode(0...12)( [:a], StatementsNode(0...12)( [MultiWriteNode(0...12)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:a, 0))], (3...4), ArrayNode(5...12)( [IntegerNode(5...6)(), IntegerNode(8...9)(), IntegerNode(11...12)()], diff --git a/test/yarp/snapshots/seattlerb/masgn_paren.txt b/test/yarp/snapshots/seattlerb/masgn_paren.txt index b47b00c417f004..20301dff96d676 100644 --- a/test/yarp/snapshots/seattlerb/masgn_paren.txt +++ b/test/yarp/snapshots/seattlerb/masgn_paren.txt @@ -2,8 +2,8 @@ ProgramNode(1...12)( [:a, :b], StatementsNode(1...12)( [MultiWriteNode(1...12)( - [LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil)], + [LocalVariableTargetNode(1...2)(:a, 0), + LocalVariableTargetNode(4...5)(:b, 0)], (7...8), CallNode(9...12)( CallNode(9...10)(nil, nil, (9...10), nil, nil, nil, nil, 2, "c"), diff --git a/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt b/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt index 21844a7c542b26..2832f367da1d7f 100644 --- a/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_splat_arg.txt @@ -3,16 +3,13 @@ ProgramNode(0...9)( StatementsNode(0...9)( [MultiWriteNode(0...9)( [MultiWriteNode(0...2)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:a, 0))], nil, nil, nil, nil ), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil)], + LocalVariableTargetNode(4...5)(:b, 0)], (6...7), CallNode(8...9)(nil, nil, (8...9), nil, nil, nil, nil, 2, "c"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt b/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt index af7f0b1b35721f..9046d08d2bb091 100644 --- a/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt +++ b/test/yarp/snapshots/seattlerb/masgn_splat_arg_arg.txt @@ -3,17 +3,14 @@ ProgramNode(0...12)( StatementsNode(0...12)( [MultiWriteNode(0...12)( [MultiWriteNode(0...2)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:a, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:a, 0))], nil, nil, nil, nil ), - LocalVariableWriteNode(4...5)(:b, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:c, 0, nil, (7...8), nil)], + LocalVariableTargetNode(4...5)(:b, 0), + LocalVariableTargetNode(7...8)(:c, 0)], (9...10), CallNode(11...12)(nil, nil, (11...12), nil, nil, nil, nil, 2, "d"), nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt b/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt index b2ce2af1738528..64d21d637316be 100644 --- a/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt +++ b/test/yarp/snapshots/seattlerb/masgn_var_star_var.txt @@ -2,9 +2,9 @@ ProgramNode(0...11)( [:a, :b], StatementsNode(0...11)( [MultiWriteNode(0...11)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), + [LocalVariableTargetNode(0...1)(:a, 0), SplatNode(3...4)((3...4), nil), - LocalVariableWriteNode(6...7)(:b, 0, nil, (6...7), nil)], + LocalVariableTargetNode(6...7)(:b, 0)], (8...9), CallNode(10...11)(nil, nil, (10...11), nil, nil, nil, nil, 2, "c"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt b/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt index c046218e1ee69e..ba484f89b0374a 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_back_anonsplat.txt @@ -2,9 +2,9 @@ ProgramNode(0...14)( [:a, :b, :c], StatementsNode(0...14)( [MultiWriteNode(0...14)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), SplatNode(9...10)((9...10), nil)], (11...12), CallNode(13...14)(nil, nil, (13...14), nil, nil, nil, nil, 2, "f"), diff --git a/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt b/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt index 1ba49176842da2..9578258257e340 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_back_splat.txt @@ -2,13 +2,10 @@ ProgramNode(0...15)( [:a, :b, :c, :s], StatementsNode(0...15)( [MultiWriteNode(0...15)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), - SplatNode(9...11)( - (9...10), - LocalVariableWriteNode(10...11)(:s, 0, nil, (10...11), nil) - )], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), + SplatNode(9...11)((9...10), LocalVariableTargetNode(10...11)(:s, 0))], (12...13), CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt b/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt index b035fbb8b7d883..8ed6e38ebdac89 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_front_anonsplat.txt @@ -9,9 +9,9 @@ ProgramNode(0...14)( nil, nil ), - LocalVariableWriteNode(3...4)(:x, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:y, 0, nil, (6...7), nil), - LocalVariableWriteNode(9...10)(:z, 0, nil, (9...10), nil)], + LocalVariableTargetNode(3...4)(:x, 0), + LocalVariableTargetNode(6...7)(:y, 0), + LocalVariableTargetNode(9...10)(:z, 0)], (11...12), CallNode(13...14)(nil, nil, (13...14), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt b/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt index 653b0e167ff5e8..07866df9edc139 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_front_splat.txt @@ -3,18 +3,15 @@ ProgramNode(0...15)( StatementsNode(0...15)( [MultiWriteNode(0...15)( [MultiWriteNode(0...2)( - [SplatNode(0...2)( - (0...1), - LocalVariableWriteNode(1...2)(:s, 0, nil, (1...2), nil) - )], + [SplatNode(0...2)((0...1), LocalVariableTargetNode(1...2)(:s, 0))], nil, nil, nil, nil ), - LocalVariableWriteNode(4...5)(:x, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:y, 0, nil, (7...8), nil), - LocalVariableWriteNode(10...11)(:z, 0, nil, (10...11), nil)], + LocalVariableTargetNode(4...5)(:x, 0), + LocalVariableTargetNode(7...8)(:y, 0), + LocalVariableTargetNode(10...11)(:z, 0)], (12...13), CallNode(14...15)(nil, nil, (14...15), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt b/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt index 313a4590f7ff5d..ce114d05287340 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_mid_anonsplat.txt @@ -2,13 +2,13 @@ ProgramNode(0...23)( [:a, :b, :c, :x, :y, :z], StatementsNode(0...23)( [MultiWriteNode(0...23)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), SplatNode(9...10)((9...10), nil), - LocalVariableWriteNode(12...13)(:x, 0, nil, (12...13), nil), - LocalVariableWriteNode(15...16)(:y, 0, nil, (15...16), nil), - LocalVariableWriteNode(18...19)(:z, 0, nil, (18...19), nil)], + LocalVariableTargetNode(12...13)(:x, 0), + LocalVariableTargetNode(15...16)(:y, 0), + LocalVariableTargetNode(18...19)(:z, 0)], (20...21), CallNode(22...23)(nil, nil, (22...23), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt b/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt index 5e307e81944223..672de03089e4be 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_mid_splat.txt @@ -2,16 +2,13 @@ ProgramNode(0...24)( [:a, :b, :c, :s, :x, :y, :z], StatementsNode(0...24)( [MultiWriteNode(0...24)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil), - LocalVariableWriteNode(6...7)(:c, 0, nil, (6...7), nil), - SplatNode(9...11)( - (9...10), - LocalVariableWriteNode(10...11)(:s, 0, nil, (10...11), nil) - ), - LocalVariableWriteNode(13...14)(:x, 0, nil, (13...14), nil), - LocalVariableWriteNode(16...17)(:y, 0, nil, (16...17), nil), - LocalVariableWriteNode(19...20)(:z, 0, nil, (19...20), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0), + LocalVariableTargetNode(6...7)(:c, 0), + SplatNode(9...11)((9...10), LocalVariableTargetNode(10...11)(:s, 0)), + LocalVariableTargetNode(13...14)(:x, 0), + LocalVariableTargetNode(16...17)(:y, 0), + LocalVariableTargetNode(19...20)(:z, 0)], (21...22), CallNode(23...24)(nil, nil, (23...24), nil, nil, nil, nil, 2, "f"), nil, diff --git a/test/yarp/snapshots/seattlerb/mlhs_rescue.txt b/test/yarp/snapshots/seattlerb/mlhs_rescue.txt index 13d418461bccb0..3e1cd57291d87b 100644 --- a/test/yarp/snapshots/seattlerb/mlhs_rescue.txt +++ b/test/yarp/snapshots/seattlerb/mlhs_rescue.txt @@ -2,8 +2,8 @@ ProgramNode(0...18)( [:a, :b], StatementsNode(0...18)( [MultiWriteNode(0...18)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0)], (5...6), RescueModifierNode(7...18)( CallNode(7...8)(nil, nil, (7...8), nil, nil, nil, nil, 2, "f"), diff --git a/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt b/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt index ed1ab308658bce..7f0a75913c6395 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_to_ary.txt @@ -2,8 +2,8 @@ ProgramNode(0...10)( [:a, :b], StatementsNode(0...10)( [MultiWriteNode(0...8)( - [LocalVariableWriteNode(0...1)(:a, 0, nil, (0...1), nil), - LocalVariableWriteNode(3...4)(:b, 0, nil, (3...4), nil)], + [LocalVariableTargetNode(0...1)(:a, 0), + LocalVariableTargetNode(3...4)(:b, 0)], (5...6), CallNode(7...8)(nil, nil, (7...8), nil, nil, nil, nil, 2, "c"), nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 71f7e8974bc187..6ee980d7c06511 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -3,8 +3,8 @@ ProgramNode(0...704)( StatementsNode(0...704)( [GlobalVariableWriteNode(0...6)((0...2), (3...4), IntegerNode(5...6)()), MultiWriteNode(8...24)( - [GlobalVariableWriteNode(8...10)((8...10), nil, nil), - GlobalVariableWriteNode(12...14)((12...14), nil, nil)], + [GlobalVariableTargetNode(8...10)(), + GlobalVariableTargetNode(12...14)()], (16...17), ArrayNode(18...24)( [IntegerNode(19...20)(), IntegerNode(22...23)()], @@ -16,14 +16,14 @@ ProgramNode(0...704)( ), MultiWriteNode(27...38)( [MultiWriteNode(27...29)( - [LocalVariableWriteNode(27...28)(:a, 0, nil, (27...28), nil), + [LocalVariableTargetNode(27...28)(:a, 0), SplatNode(28...29)((28...29), nil)], nil, nil, (26...27), (29...30) ), - LocalVariableWriteNode(32...33)(:b, 0, nil, (32...33), nil)], + LocalVariableTargetNode(32...33)(:b, 0)], (35...36), IntegerNode(37...38)(), (25...26), @@ -32,7 +32,7 @@ ProgramNode(0...704)( MultiWriteNode(40...48)( [SplatNode(40...42)( (40...41), - LocalVariableWriteNode(41...42)(:a, 0, nil, (41...42), nil) + LocalVariableTargetNode(41...42)(:a, 0) )], (44...45), ArrayNode(46...48)([], (46...47), (47...48)), @@ -42,7 +42,7 @@ ProgramNode(0...704)( MultiWriteNode(50...64)( [SplatNode(50...54)( (50...51), - LocalVariableWriteNode(51...54)(:foo, 0, nil, (51...54), nil) + LocalVariableTargetNode(51...54)(:foo, 0) )], (56...57), ArrayNode(58...64)( @@ -54,8 +54,8 @@ ProgramNode(0...704)( (54...55) ), MultiWriteNode(66...84)( - [ClassVariableWriteNode(66...69)((66...69), nil, nil), - ClassVariableWriteNode(71...74)((71...74), nil, nil)], + [ClassVariableTargetNode(66...69)(), + ClassVariableTargetNode(71...74)()], (76...77), ArrayNode(78...84)( [IntegerNode(79...80)(), IntegerNode(82...83)()], @@ -66,8 +66,8 @@ ProgramNode(0...704)( (74...75) ), MultiWriteNode(86...102)( - [InstanceVariableWriteNode(86...88)((86...88), nil, nil), - InstanceVariableWriteNode(90...92)((90...92), nil, nil)], + [InstanceVariableTargetNode(86...88)(), + InstanceVariableTargetNode(90...92)()], (94...95), ArrayNode(96...102)( [IntegerNode(97...98)(), IntegerNode(100...101)()], @@ -78,10 +78,10 @@ ProgramNode(0...704)( (92...93) ), MultiWriteNode(104...128)( - [LocalVariableWriteNode(104...105)(:a, 0, nil, (104...105), nil), + [LocalVariableTargetNode(104...105)(:a, 0), MultiWriteNode(108...113)( - [LocalVariableWriteNode(108...109)(:b, 0, nil, (108...109), nil), - LocalVariableWriteNode(111...112)(:c, 0, nil, (111...112), nil)], + [LocalVariableTargetNode(108...109)(:b, 0), + LocalVariableTargetNode(111...112)(:c, 0)], nil, nil, (107...108), @@ -102,7 +102,7 @@ ProgramNode(0...704)( (113...114) ), MultiWriteNode(130...144)( - [LocalVariableWriteNode(130...131)(:a, 0, nil, (130...131), nil), + [LocalVariableTargetNode(130...131)(:a, 0), SplatNode(133...134)((133...134), nil)], (136...137), ArrayNode(138...144)( @@ -114,10 +114,10 @@ ProgramNode(0...704)( (134...135) ), MultiWriteNode(146...163)( - [LocalVariableWriteNode(146...147)(:a, 0, nil, (146...147), nil), + [LocalVariableTargetNode(146...147)(:a, 0), SplatNode(149...153)( (149...150), - LocalVariableWriteNode(150...153)(:foo, 0, nil, (150...153), nil) + LocalVariableTargetNode(150...153)(:foo, 0) )], (155...156), ArrayNode(157...163)( @@ -129,8 +129,8 @@ ProgramNode(0...704)( (153...154) ), MultiWriteNode(165...179)( - [LocalVariableWriteNode(165...166)(:a, 0, nil, (165...166), nil), - LocalVariableWriteNode(168...169)(:b, 0, nil, (168...169), nil)], + [LocalVariableTargetNode(165...166)(:a, 0), + LocalVariableTargetNode(168...169)(:b, 0)], (171...172), ArrayNode(173...179)( [IntegerNode(174...175)(), IntegerNode(177...178)()], @@ -141,15 +141,15 @@ ProgramNode(0...704)( (169...170) ), MultiWriteNode(181...192)( - [LocalVariableWriteNode(181...182)(:a, 0, nil, (181...182), nil), - LocalVariableWriteNode(184...185)(:b, 0, nil, (184...185), nil)], + [LocalVariableTargetNode(181...182)(:a, 0), + LocalVariableTargetNode(184...185)(:b, 0)], (187...188), LocalVariableReadNode(189...192)(:foo, 0), (180...181), (185...186) ), MultiWriteNode(194...203)( - [LocalVariableWriteNode(194...195)(:a, 0, nil, (194...195), nil), + [LocalVariableTargetNode(194...195)(:a, 0), SplatNode(195...196)((195...196), nil)], (198...199), LocalVariableReadNode(200...203)(:foo, 0), @@ -324,14 +324,8 @@ ProgramNode(0...704)( ParenthesesNode(355...367)( StatementsNode(357...366)( [MultiWriteNode(357...366)( - [LocalVariableWriteNode(357...358)(:b, 0, nil, (357...358), nil), - LocalVariableWriteNode(360...361)( - :c, - 0, - nil, - (360...361), - nil - )], + [LocalVariableTargetNode(357...358)(:b, 0), + LocalVariableTargetNode(360...361)(:c, 0)], (363...364), IntegerNode(365...366)(), (356...357), diff --git a/test/yarp/snapshots/unparser/corpus/literal/block.txt b/test/yarp/snapshots/unparser/corpus/literal/block.txt index 787c8cb5929594..a115e84135b5f9 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/block.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/block.txt @@ -743,7 +743,7 @@ ProgramNode(0...737)( (365...371), [ConstantReadNode(372...381)()], (382...384), - LocalVariableWriteNode(385...386)(:e, 0, nil, (385...386), nil), + LocalVariableTargetNode(385...386)(:e, 0), nil, nil ), @@ -786,7 +786,7 @@ ProgramNode(0...737)( (402...408), [ConstantReadNode(409...418)()], (419...421), - LocalVariableWriteNode(422...425)(:bar, 0, nil, (422...425), nil), + LocalVariableTargetNode(422...425)(:bar, 0), StatementsNode(428...431)( [LocalVariableReadNode(428...431)(:bar, 0)] ), @@ -914,13 +914,7 @@ ProgramNode(0...737)( ) )], (514...516), - LocalVariableWriteNode(517...526)( - :exception, - 0, - nil, - (517...526), - nil - ), + LocalVariableTargetNode(517...526)(:exception, 0), StatementsNode(529...532)( [CallNode(529...532)( nil, @@ -1151,13 +1145,7 @@ ProgramNode(0...737)( ) )], (658...660), - LocalVariableWriteNode(661...670)( - :exception, - 0, - nil, - (661...670), - nil - ), + LocalVariableTargetNode(661...670)(:exception, 0), StatementsNode(673...676)( [CallNode(673...676)( nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/defined.txt b/test/yarp/snapshots/unparser/corpus/literal/defined.txt index 1ea9cb9d87a7b0..7ba02da908979e 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/defined.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/defined.txt @@ -18,8 +18,8 @@ ProgramNode(0...56)( ParenthesesNode(38...55)( StatementsNode(40...54)( [MultiWriteNode(40...54)( - [LocalVariableWriteNode(40...41)(:a, 0, nil, (40...41), nil), - LocalVariableWriteNode(43...44)(:b, 0, nil, (43...44), nil)], + [LocalVariableTargetNode(40...41)(:a, 0), + LocalVariableTargetNode(43...44)(:b, 0)], (46...47), ArrayNode(48...54)( [IntegerNode(49...50)(), IntegerNode(52...53)()], diff --git a/test/yarp/snapshots/unparser/corpus/literal/for.txt b/test/yarp/snapshots/unparser/corpus/literal/for.txt index 1bd6587bd7ad26..d7a916867674f3 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/for.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/for.txt @@ -9,7 +9,7 @@ ProgramNode(0...119)( ArgumentsNode(4...29)( [ForNode(4...29)( MultiWriteNode(8...9)( - [LocalVariableWriteNode(8...9)(:a, 0, nil, (8...9), nil)], + [LocalVariableTargetNode(8...9)(:a, 0)], nil, nil, nil, @@ -52,7 +52,7 @@ ProgramNode(0...119)( ), ForNode(31...56)( MultiWriteNode(35...36)( - [LocalVariableWriteNode(35...36)(:a, 0, nil, (35...36), nil)], + [LocalVariableTargetNode(35...36)(:a, 0)], nil, nil, nil, @@ -69,10 +69,10 @@ ProgramNode(0...119)( ), ForNode(57...88)( MultiWriteNode(61...68)( - [LocalVariableWriteNode(62...63)(:a, 0, nil, (62...63), nil), + [LocalVariableTargetNode(62...63)(:a, 0), SplatNode(65...67)( (65...66), - LocalVariableWriteNode(66...67)(:b, 0, nil, (66...67), nil) + LocalVariableTargetNode(66...67)(:b, 0) )], nil, nil, @@ -90,8 +90,8 @@ ProgramNode(0...119)( ), ForNode(89...119)( MultiWriteNode(93...99)( - [LocalVariableWriteNode(94...95)(:a, 0, nil, (94...95), nil), - LocalVariableWriteNode(97...98)(:b, 0, nil, (97...98), nil)], + [LocalVariableTargetNode(94...95)(:a, 0), + LocalVariableTargetNode(97...98)(:b, 0)], nil, nil, (93...94), diff --git a/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt b/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt index 0553be37eab3f6..7e9670dae50d94 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt @@ -98,7 +98,7 @@ ProgramNode(0...530)( (133...139), [ConstantReadNode(140...141)()], (142...144), - LocalVariableWriteNode(145...148)(:foo, 0, nil, (145...148), nil), + LocalVariableTargetNode(145...148)(:foo, 0), nil, nil ), @@ -294,7 +294,7 @@ ProgramNode(0...530)( (351...357), [], (358...360), - LocalVariableWriteNode(361...364)(:bar, 0, nil, (361...364), nil), + LocalVariableTargetNode(361...364)(:bar, 0), StatementsNode(367...370)( [LocalVariableReadNode(367...370)(:bar, 0)] ), @@ -311,7 +311,7 @@ ProgramNode(0...530)( (388...394), [ConstantReadNode(395...404)(), ConstantReadNode(406...411)()], (412...414), - LocalVariableWriteNode(415...418)(:bar, 0, nil, (415...418), nil), + LocalVariableTargetNode(415...418)(:bar, 0), StatementsNode(421...424)( [LocalVariableReadNode(421...424)(:bar, 0)] ), @@ -332,13 +332,7 @@ ProgramNode(0...530)( LocalVariableReadNode(461...464)(:bar, 0) )], (465...467), - LocalVariableWriteNode(468...477)( - :exception, - 0, - nil, - (468...477), - nil - ), + LocalVariableTargetNode(468...477)(:exception, 0), StatementsNode(480...483)( [CallNode(480...483)( nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index 4ef108e71a345e..b448ea3e143934 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -12,20 +12,8 @@ ProgramNode(0...999)( ParenthesesNode(19...31)( StatementsNode(21...30)( [MultiWriteNode(21...30)( - [LocalVariableWriteNode(21...22)( - :a, - 0, - nil, - (21...22), - nil - ), - LocalVariableWriteNode(24...25)( - :_, - 0, - nil, - (24...25), - nil - )], + [LocalVariableTargetNode(21...22)(:a, 0), + LocalVariableTargetNode(24...25)(:_, 0)], (27...28), CallNode(29...30)( nil, diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index a4373717f511d9..5d96122709915d 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -8,8 +8,8 @@ ProgramNode(0...293)( (13...14) ), MultiWriteNode(18...34)( - [ClassVariableWriteNode(18...23)((18...23), nil, nil), - ClassVariableWriteNode(25...30)((25...30), nil, nil)], + [ClassVariableTargetNode(18...23)(), + ClassVariableTargetNode(25...30)()], (31...32), IntegerNode(33...34)(), nil, @@ -45,8 +45,8 @@ ProgramNode(0...293)( (89...90) ), MultiWriteNode(94...108)( - [GlobalVariableWriteNode(94...98)((94...98), nil, nil), - GlobalVariableWriteNode(100...104)((100...104), nil, nil)], + [GlobalVariableTargetNode(94...98)(), + GlobalVariableTargetNode(100...104)()], (105...106), IntegerNode(107...108)(), nil, @@ -62,8 +62,8 @@ ProgramNode(0...293)( ) ), MultiWriteNode(123...137)( - [InstanceVariableWriteNode(123...127)((123...127), nil, nil), - InstanceVariableWriteNode(129...133)((129...133), nil, nil)], + [InstanceVariableTargetNode(123...127)(), + InstanceVariableTargetNode(129...133)()], (134...135), IntegerNode(136...137)(), nil, @@ -108,7 +108,7 @@ ProgramNode(0...293)( (177...178) ), MultiWriteNode(185...198)( - [LocalVariableWriteNode(185...188)(:foo, 0, nil, (185...188), nil), + [LocalVariableTargetNode(185...188)(:foo, 0), SplatNode(190...191)((190...191), nil)], (192...193), ArrayNode(194...198)( @@ -120,7 +120,7 @@ ProgramNode(0...293)( nil ), MultiWriteNode(200...211)( - [LocalVariableWriteNode(200...203)(:foo, 0, nil, (200...203), nil), + [LocalVariableTargetNode(200...203)(:foo, 0), SplatNode(203...204)((203...204), nil)], (205...206), ArrayNode(207...211)( @@ -132,10 +132,10 @@ ProgramNode(0...293)( nil ), MultiWriteNode(213...229)( - [LocalVariableWriteNode(213...216)(:foo, 0, nil, (213...216), nil), + [LocalVariableTargetNode(213...216)(:foo, 0), SplatNode(218...222)( (218...219), - LocalVariableWriteNode(219...222)(:bar, 0, nil, (219...222), nil) + LocalVariableTargetNode(219...222)(:bar, 0) )], (223...224), ArrayNode(225...229)( @@ -147,10 +147,10 @@ ProgramNode(0...293)( nil ), MultiWriteNode(231...258)( - [LocalVariableWriteNode(231...234)(:foo, 0, nil, (231...234), nil), + [LocalVariableTargetNode(231...234)(:foo, 0), MultiWriteNode(237...246)( - [LocalVariableWriteNode(237...240)(:bar, 0, nil, (237...240), nil), - LocalVariableWriteNode(242...245)(:baz, 0, nil, (242...245), nil)], + [LocalVariableTargetNode(237...240)(:bar, 0), + LocalVariableTargetNode(242...245)(:baz, 0)], nil, nil, (236...237), diff --git a/test/yarp/snapshots/whitequark/and_or_masgn.txt b/test/yarp/snapshots/whitequark/and_or_masgn.txt index eb768ac90a4731..bc64788ca43665 100644 --- a/test/yarp/snapshots/whitequark/and_or_masgn.txt +++ b/test/yarp/snapshots/whitequark/and_or_masgn.txt @@ -6,8 +6,8 @@ ProgramNode(0...40)( ParenthesesNode(7...19)( StatementsNode(8...18)( [MultiWriteNode(8...18)( - [LocalVariableWriteNode(8...9)(:a, 0, nil, (8...9), nil), - LocalVariableWriteNode(11...12)(:b, 0, nil, (11...12), nil)], + [LocalVariableTargetNode(8...9)(:a, 0), + LocalVariableTargetNode(11...12)(:b, 0)], (13...14), CallNode(15...18)( nil, @@ -34,8 +34,8 @@ ProgramNode(0...40)( ParenthesesNode(28...40)( StatementsNode(29...39)( [MultiWriteNode(29...39)( - [LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), - LocalVariableWriteNode(32...33)(:b, 0, nil, (32...33), nil)], + [LocalVariableTargetNode(29...30)(:a, 0), + LocalVariableTargetNode(32...33)(:b, 0)], (34...35), CallNode(36...39)( nil, diff --git a/test/yarp/snapshots/whitequark/cond_begin_masgn.txt b/test/yarp/snapshots/whitequark/cond_begin_masgn.txt index 10e2f5d8dc72dc..8114bd1cb4491d 100644 --- a/test/yarp/snapshots/whitequark/cond_begin_masgn.txt +++ b/test/yarp/snapshots/whitequark/cond_begin_masgn.txt @@ -7,8 +7,8 @@ ProgramNode(0...25)( StatementsNode(4...19)( [CallNode(4...7)(nil, nil, (4...7), nil, nil, nil, nil, 2, "bar"), MultiWriteNode(9...19)( - [LocalVariableWriteNode(9...10)(:a, 0, nil, (9...10), nil), - LocalVariableWriteNode(12...13)(:b, 0, nil, (12...13), nil)], + [LocalVariableTargetNode(9...10)(:a, 0), + LocalVariableTargetNode(12...13)(:b, 0)], (14...15), CallNode(16...19)( nil, diff --git a/test/yarp/snapshots/whitequark/for.txt b/test/yarp/snapshots/whitequark/for.txt index e62da36cd540fa..1250760a01f976 100644 --- a/test/yarp/snapshots/whitequark/for.txt +++ b/test/yarp/snapshots/whitequark/for.txt @@ -3,7 +3,7 @@ ProgramNode(0...48)( StatementsNode(0...48)( [ForNode(0...24)( MultiWriteNode(4...5)( - [LocalVariableWriteNode(4...5)(:a, 0, nil, (4...5), nil)], + [LocalVariableTargetNode(4...5)(:a, 0)], nil, nil, nil, @@ -30,7 +30,7 @@ ProgramNode(0...48)( ), ForNode(26...48)( MultiWriteNode(30...31)( - [LocalVariableWriteNode(30...31)(:a, 0, nil, (30...31), nil)], + [LocalVariableTargetNode(30...31)(:a, 0)], nil, nil, nil, diff --git a/test/yarp/snapshots/whitequark/for_mlhs.txt b/test/yarp/snapshots/whitequark/for_mlhs.txt index 07bdb0662a5b9f..fecaeb10caf949 100644 --- a/test/yarp/snapshots/whitequark/for_mlhs.txt +++ b/test/yarp/snapshots/whitequark/for_mlhs.txt @@ -3,8 +3,8 @@ ProgramNode(0...28)( StatementsNode(0...28)( [ForNode(0...28)( MultiWriteNode(4...8)( - [LocalVariableWriteNode(4...5)(:a, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:b, 0, nil, (7...8), nil)], + [LocalVariableTargetNode(4...5)(:a, 0), + LocalVariableTargetNode(7...8)(:b, 0)], nil, nil, nil, diff --git a/test/yarp/snapshots/whitequark/if_masgn__24.txt b/test/yarp/snapshots/whitequark/if_masgn__24.txt index 94e63ac4f7ff99..24c9ef784ddc58 100644 --- a/test/yarp/snapshots/whitequark/if_masgn__24.txt +++ b/test/yarp/snapshots/whitequark/if_masgn__24.txt @@ -6,8 +6,8 @@ ProgramNode(0...20)( ParenthesesNode(3...15)( StatementsNode(4...14)( [MultiWriteNode(4...14)( - [LocalVariableWriteNode(4...5)(:a, 0, nil, (4...5), nil), - LocalVariableWriteNode(7...8)(:b, 0, nil, (7...8), nil)], + [LocalVariableTargetNode(4...5)(:a, 0), + LocalVariableTargetNode(7...8)(:b, 0)], (9...10), CallNode(11...14)( nil, diff --git a/test/yarp/snapshots/whitequark/masgn.txt b/test/yarp/snapshots/whitequark/masgn.txt index 9945a348c11745..466758f3ab6ca2 100644 --- a/test/yarp/snapshots/whitequark/masgn.txt +++ b/test/yarp/snapshots/whitequark/masgn.txt @@ -2,8 +2,8 @@ ProgramNode(1...56)( [:foo, :bar, :baz], StatementsNode(1...56)( [MultiWriteNode(1...17)( - [LocalVariableWriteNode(1...4)(:foo, 0, nil, (1...4), nil), - LocalVariableWriteNode(6...9)(:bar, 0, nil, (6...9), nil)], + [LocalVariableTargetNode(1...4)(:foo, 0), + LocalVariableTargetNode(6...9)(:bar, 0)], (11...12), ArrayNode(13...17)( [IntegerNode(13...14)(), IntegerNode(16...17)()], @@ -14,8 +14,8 @@ ProgramNode(1...56)( (9...10) ), MultiWriteNode(19...34)( - [LocalVariableWriteNode(19...22)(:foo, 0, nil, (19...22), nil), - LocalVariableWriteNode(24...27)(:bar, 0, nil, (24...27), nil)], + [LocalVariableTargetNode(19...22)(:foo, 0), + LocalVariableTargetNode(24...27)(:bar, 0)], (28...29), ArrayNode(30...34)( [IntegerNode(30...31)(), IntegerNode(33...34)()], @@ -26,9 +26,9 @@ ProgramNode(1...56)( nil ), MultiWriteNode(36...56)( - [LocalVariableWriteNode(36...39)(:foo, 0, nil, (36...39), nil), - LocalVariableWriteNode(41...44)(:bar, 0, nil, (41...44), nil), - LocalVariableWriteNode(46...49)(:baz, 0, nil, (46...49), nil)], + [LocalVariableTargetNode(36...39)(:foo, 0), + LocalVariableTargetNode(41...44)(:bar, 0), + LocalVariableTargetNode(46...49)(:baz, 0)], (50...51), ArrayNode(52...56)( [IntegerNode(52...53)(), IntegerNode(55...56)()], diff --git a/test/yarp/snapshots/whitequark/masgn_attr.txt b/test/yarp/snapshots/whitequark/masgn_attr.txt index c5ab22eb0e980e..1c71499f78f448 100644 --- a/test/yarp/snapshots/whitequark/masgn_attr.txt +++ b/test/yarp/snapshots/whitequark/masgn_attr.txt @@ -13,7 +13,7 @@ ProgramNode(0...63)( 0, "A=" ), - LocalVariableWriteNode(8...11)(:foo, 0, nil, (8...11), nil)], + LocalVariableTargetNode(8...11)(:foo, 0)], (12...13), LocalVariableReadNode(14...17)(:foo, 0), nil, @@ -61,7 +61,7 @@ ProgramNode(0...63)( 0, "a=" ), - LocalVariableWriteNode(54...57)(:foo, 0, nil, (54...57), nil)], + LocalVariableTargetNode(54...57)(:foo, 0)], (58...59), LocalVariableReadNode(60...63)(:foo, 0), nil, diff --git a/test/yarp/snapshots/whitequark/masgn_cmd.txt b/test/yarp/snapshots/whitequark/masgn_cmd.txt index 1ca3b64a0a92e4..14d81d79bcf2b3 100644 --- a/test/yarp/snapshots/whitequark/masgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/masgn_cmd.txt @@ -2,8 +2,8 @@ ProgramNode(0...16)( [:foo, :bar], StatementsNode(0...16)( [MultiWriteNode(0...16)( - [LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), - LocalVariableWriteNode(5...8)(:bar, 0, nil, (5...8), nil)], + [LocalVariableTargetNode(0...3)(:foo, 0), + LocalVariableTargetNode(5...8)(:bar, 0)], (9...10), CallNode(11...16)( nil, diff --git a/test/yarp/snapshots/whitequark/masgn_const.txt b/test/yarp/snapshots/whitequark/masgn_const.txt index 4db22ceb7a3208..7aeb91557c4914 100644 --- a/test/yarp/snapshots/whitequark/masgn_const.txt +++ b/test/yarp/snapshots/whitequark/masgn_const.txt @@ -2,28 +2,20 @@ ProgramNode(0...34)( [:foo], StatementsNode(0...34)( [MultiWriteNode(0...14)( - [ConstantPathWriteNode(0...3)( - ConstantPathNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), - nil, - nil - ), - LocalVariableWriteNode(5...8)(:foo, 0, nil, (5...8), nil)], + [ConstantPathTargetNode(0...3)(nil, ConstantReadNode(2...3)(), (0...2)), + LocalVariableTargetNode(5...8)(:foo, 0)], (9...10), LocalVariableReadNode(11...14)(:foo, 0), nil, nil ), MultiWriteNode(16...34)( - [ConstantPathWriteNode(16...23)( - ConstantPathNode(16...23)( - SelfNode(16...20)(), - ConstantReadNode(22...23)(), - (20...22) - ), - nil, - nil + [ConstantPathTargetNode(16...23)( + SelfNode(16...20)(), + ConstantReadNode(22...23)(), + (20...22) ), - LocalVariableWriteNode(25...28)(:foo, 0, nil, (25...28), nil)], + LocalVariableTargetNode(25...28)(:foo, 0)], (29...30), LocalVariableReadNode(31...34)(:foo, 0), nil, diff --git a/test/yarp/snapshots/whitequark/masgn_nested.txt b/test/yarp/snapshots/whitequark/masgn_nested.txt index 98f1c094d00e70..3b087771354563 100644 --- a/test/yarp/snapshots/whitequark/masgn_nested.txt +++ b/test/yarp/snapshots/whitequark/masgn_nested.txt @@ -3,7 +3,7 @@ ProgramNode(2...30)( StatementsNode(2...30)( [MultiWriteNode(2...13)( [MultiWriteNode(2...4)( - [LocalVariableWriteNode(2...3)(:b, 0, nil, (2...3), nil), + [LocalVariableTargetNode(2...3)(:b, 0), SplatNode(3...4)((3...4), nil)], nil, nil, @@ -16,10 +16,10 @@ ProgramNode(2...30)( (6...7) ), MultiWriteNode(15...30)( - [LocalVariableWriteNode(15...16)(:a, 0, nil, (15...16), nil), + [LocalVariableTargetNode(15...16)(:a, 0), MultiWriteNode(19...24)( - [LocalVariableWriteNode(19...20)(:b, 0, nil, (19...20), nil), - LocalVariableWriteNode(22...23)(:c, 0, nil, (22...23), nil)], + [LocalVariableTargetNode(19...20)(:b, 0), + LocalVariableTargetNode(22...23)(:c, 0)], nil, nil, (18...19), diff --git a/test/yarp/snapshots/whitequark/masgn_splat.txt b/test/yarp/snapshots/whitequark/masgn_splat.txt index 635e1ae6fdd25b..e10e6d22d16dec 100644 --- a/test/yarp/snapshots/whitequark/masgn_splat.txt +++ b/test/yarp/snapshots/whitequark/masgn_splat.txt @@ -16,8 +16,8 @@ ProgramNode(0...139)( nil, nil ), - LocalVariableWriteNode(12...13)(:c, 0, nil, (12...13), nil), - LocalVariableWriteNode(15...16)(:d, 0, nil, (15...16), nil)], + LocalVariableTargetNode(12...13)(:c, 0), + LocalVariableTargetNode(15...16)(:d, 0)], (17...18), CallNode(19...22)(nil, nil, (19...22), nil, nil, nil, nil, 2, "bar"), nil, @@ -26,7 +26,7 @@ ProgramNode(0...139)( MultiWriteNode(24...32)( [SplatNode(24...26)( (24...25), - LocalVariableWriteNode(25...26)(:b, 0, nil, (25...26), nil) + LocalVariableTargetNode(25...26)(:b, 0) )], (27...28), CallNode(29...32)(nil, nil, (29...32), nil, nil, nil, nil, 2, "bar"), @@ -37,22 +37,22 @@ ProgramNode(0...139)( [MultiWriteNode(34...36)( [SplatNode(34...36)( (34...35), - LocalVariableWriteNode(35...36)(:b, 0, nil, (35...36), nil) + LocalVariableTargetNode(35...36)(:b, 0) )], nil, nil, nil, nil ), - LocalVariableWriteNode(38...39)(:c, 0, nil, (38...39), nil)], + LocalVariableTargetNode(38...39)(:c, 0)], (40...41), CallNode(42...45)(nil, nil, (42...45), nil, nil, nil, nil, 2, "bar"), nil, nil ), MultiWriteNode(47...65)( - [InstanceVariableWriteNode(47...51)((47...51), nil, nil), - ClassVariableWriteNode(53...58)((53...58), nil, nil)], + [InstanceVariableTargetNode(47...51)(), + ClassVariableTargetNode(53...58)()], (59...60), ArrayNode(61...65)( [SplatNode(61...65)( @@ -76,7 +76,7 @@ ProgramNode(0...139)( nil ), MultiWriteNode(67...77)( - [LocalVariableWriteNode(67...68)(:a, 0, nil, (67...68), nil), + [LocalVariableTargetNode(67...68)(:a, 0), SplatNode(70...71)((70...71), nil)], (72...73), CallNode(74...77)(nil, nil, (74...77), nil, nil, nil, nil, 2, "bar"), @@ -84,19 +84,19 @@ ProgramNode(0...139)( nil ), MultiWriteNode(79...92)( - [LocalVariableWriteNode(79...80)(:a, 0, nil, (79...80), nil), + [LocalVariableTargetNode(79...80)(:a, 0), SplatNode(82...83)((82...83), nil), - LocalVariableWriteNode(85...86)(:c, 0, nil, (85...86), nil)], + LocalVariableTargetNode(85...86)(:c, 0)], (87...88), CallNode(89...92)(nil, nil, (89...92), nil, nil, nil, nil, 2, "bar"), nil, nil ), MultiWriteNode(94...105)( - [LocalVariableWriteNode(94...95)(:a, 0, nil, (94...95), nil), + [LocalVariableTargetNode(94...95)(:a, 0), SplatNode(97...99)( (97...98), - LocalVariableWriteNode(98...99)(:b, 0, nil, (98...99), nil) + LocalVariableTargetNode(98...99)(:b, 0) )], (100...101), CallNode(102...105)( @@ -114,12 +114,12 @@ ProgramNode(0...139)( nil ), MultiWriteNode(107...121)( - [LocalVariableWriteNode(107...108)(:a, 0, nil, (107...108), nil), + [LocalVariableTargetNode(107...108)(:a, 0), SplatNode(110...112)( (110...111), - LocalVariableWriteNode(111...112)(:b, 0, nil, (111...112), nil) + LocalVariableTargetNode(111...112)(:b, 0) ), - LocalVariableWriteNode(114...115)(:c, 0, nil, (114...115), nil)], + LocalVariableTargetNode(114...115)(:c, 0)], (116...117), CallNode(118...121)( nil, @@ -136,8 +136,8 @@ ProgramNode(0...139)( nil ), MultiWriteNode(123...139)( - [LocalVariableWriteNode(123...124)(:a, 0, nil, (123...124), nil), - LocalVariableWriteNode(126...127)(:b, 0, nil, (126...127), nil)], + [LocalVariableTargetNode(123...124)(:a, 0), + LocalVariableTargetNode(126...127)(:b, 0)], (128...129), ArrayNode(130...139)( [SplatNode(130...134)( diff --git a/test/yarp/snapshots/whitequark/not_masgn__24.txt b/test/yarp/snapshots/whitequark/not_masgn__24.txt index badac67e44fb05..47895d8c462466 100644 --- a/test/yarp/snapshots/whitequark/not_masgn__24.txt +++ b/test/yarp/snapshots/whitequark/not_masgn__24.txt @@ -5,8 +5,8 @@ ProgramNode(0...13)( ParenthesesNode(1...13)( StatementsNode(2...12)( [MultiWriteNode(2...12)( - [LocalVariableWriteNode(2...3)(:a, 0, nil, (2...3), nil), - LocalVariableWriteNode(5...6)(:b, 0, nil, (5...6), nil)], + [LocalVariableTargetNode(2...3)(:a, 0), + LocalVariableTargetNode(5...6)(:b, 0)], (7...8), CallNode(9...12)( nil, diff --git a/test/yarp/snapshots/whitequark/resbody_list_var.txt b/test/yarp/snapshots/whitequark/resbody_list_var.txt index 7552914e005237..660abcc181dfff 100644 --- a/test/yarp/snapshots/whitequark/resbody_list_var.txt +++ b/test/yarp/snapshots/whitequark/resbody_list_var.txt @@ -20,7 +20,7 @@ ProgramNode(0...39)( "foo" )], (24...26), - LocalVariableWriteNode(27...29)(:ex, 0, nil, (27...29), nil), + LocalVariableTargetNode(27...29)(:ex, 0), StatementsNode(31...34)( [CallNode(31...34)( nil, diff --git a/test/yarp/snapshots/whitequark/resbody_var.txt b/test/yarp/snapshots/whitequark/resbody_var.txt index 8e2d566c19642f..861a91f79f04c1 100644 --- a/test/yarp/snapshots/whitequark/resbody_var.txt +++ b/test/yarp/snapshots/whitequark/resbody_var.txt @@ -10,7 +10,7 @@ ProgramNode(0...73)( (13...19), [], (20...22), - InstanceVariableWriteNode(23...26)((23...26), nil, nil), + InstanceVariableTargetNode(23...26)(), StatementsNode(28...31)( [CallNode(28...31)( nil, @@ -49,7 +49,7 @@ ProgramNode(0...73)( (51...57), [], (58...60), - LocalVariableWriteNode(61...63)(:ex, 0, nil, (61...63), nil), + LocalVariableTargetNode(61...63)(:ex, 0), StatementsNode(65...68)( [CallNode(65...68)( nil, diff --git a/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt b/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt index 60464244f58dbc..13023afc175e47 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_masgn.txt @@ -2,8 +2,8 @@ ProgramNode(0...29)( [:foo, :bar], StatementsNode(0...29)( [MultiWriteNode(0...29)( - [LocalVariableWriteNode(0...3)(:foo, 0, nil, (0...3), nil), - LocalVariableWriteNode(5...8)(:bar, 0, nil, (5...8), nil)], + [LocalVariableTargetNode(0...3)(:foo, 0), + LocalVariableTargetNode(5...8)(:bar, 0)], (9...10), RescueModifierNode(11...29)( CallNode(11...15)(nil, nil, (11...15), nil, nil, nil, nil, 2, "meth"), diff --git a/yarp/config.yml b/yarp/config.yml index 24e8d0e227284d..bb4d4174dac1c4 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -772,6 +772,12 @@ nodes: @@foo ^^^^^ + - name: ClassVariableTargetNode + comment: | + Represents writing to a class variable in a context that doesn't have an explicit value. + + @@foo, @@bar = baz + ^^^^^ ^^^^^ - name: ClassVariableWriteNode child_nodes: - name: name_loc @@ -883,6 +889,19 @@ nodes: Parent::Child ||= value ^^^^^^^^^^^^^^^^^^^^^^^ + - name: ConstantPathTargetNode + child_nodes: + - name: parent + type: node? + - name: child + type: node + - name: delimiter_loc + type: location + comment: | + Represents writing to a constant path in a context that doesn't have an explicit value. + + Foo::Foo, Bar::Bar = baz + ^^^^^^^^ ^^^^^^^^ - name: ConstantPathWriteNode child_nodes: - name: target @@ -909,6 +928,12 @@ nodes: Foo ^^^ + - name: ConstantTargetNode + comment: | + Represents writing to a constant in a context that doesn't have an explicit value. + + Foo, Bar = baz + ^^^ ^^^ - name: ConstantWriteNode child_nodes: - name: name_loc @@ -1172,6 +1197,12 @@ nodes: $foo ^^^^ + - name: GlobalVariableTargetNode + comment: | + Represents writing to a global variable in a context that doesn't have an explicit value. + + $foo, $bar = baz + ^^^^ ^^^^ - name: GlobalVariableWriteNode child_nodes: - name: name_loc @@ -1312,6 +1343,12 @@ nodes: @foo ^^^^ + - name: InstanceVariableTargetNode + comment: | + Represents writing to an instance variable in a context that doesn't have an explicit value. + + @foo, @bar = baz + ^^^^ ^^^^ - name: InstanceVariableWriteNode child_nodes: - name: name_loc @@ -1509,6 +1546,17 @@ nodes: foo ^^^ + - name: LocalVariableTargetNode + child_nodes: + - name: constant_id + type: constant + - name: depth + type: uint32 + comment: | + Represents writing to a local variable in a context that doesn't have an explicit value. + + foo, bar = baz + ^^^ ^^^ - name: LocalVariableWriteNode child_nodes: - name: constant_id diff --git a/yarp/yarp.c b/yarp/yarp.c index 2b077f03855c82..f8fd94997da119 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -1641,10 +1641,10 @@ yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp .type = YP_NODE_CLASS_VARIABLE_WRITE_NODE, .location = { .start = read_node->base.location.start, - .end = value != NULL ? value->location.end : read_node->base.location.end + .end = value->location.end }, }, - .name_loc = YP_LOCATION_NODE_VALUE((yp_node_t *)read_node), + .name_loc = YP_LOCATION_NODE_VALUE((yp_node_t *) read_node), .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -1749,7 +1749,7 @@ yp_constant_path_write_node_create(yp_parser_t *parser, yp_constant_path_node_t .type = YP_NODE_CONSTANT_PATH_WRITE_NODE, .location = { .start = target->base.location.start, - .end = (value == NULL ? target->base.location.end : value->location.end) + .end = value->location.end }, }, .target = target, @@ -1848,7 +1848,7 @@ yp_constant_write_node_create(yp_parser_t *parser, yp_location_t *name_loc, cons .type = YP_NODE_CONSTANT_WRITE_NODE, .location = { .start = name_loc->start, - .end = value != NULL ? value->location.end : name_loc->end + .end = value->location.end }, }, .name_loc = *name_loc, @@ -2355,7 +2355,7 @@ yp_global_variable_write_node_create(yp_parser_t *parser, const yp_location_t *n .type = YP_NODE_GLOBAL_VARIABLE_WRITE_NODE, .location = { .start = name_loc->start, - .end = (value == NULL ? name_loc->end : value->location.end) + .end = value->location.end }, }, .name_loc = *name_loc, @@ -2710,7 +2710,7 @@ yp_instance_variable_write_node_create(yp_parser_t *parser, yp_instance_variable .type = YP_NODE_INSTANCE_VARIABLE_WRITE_NODE, .location = { .start = read_node->base.location.start, - .end = value == NULL ? read_node->base.location.end : value->location.end + .end = value->location.end } }, .name_loc = YP_LOCATION_NODE_BASE_VALUE(read_node), @@ -3062,7 +3062,7 @@ yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t consta .type = YP_NODE_LOCAL_VARIABLE_WRITE_NODE, .location = { .start = name_loc->start, - .end = value == NULL ? name_loc->end : value->location.end + .end = value->location.end } }, .constant_id = constant_id, @@ -7810,55 +7810,42 @@ parse_starred_expression(yp_parser_t *parser, yp_binding_power_t binding_power, // Convert the given node into a valid target node. static yp_node_t * parse_target(yp_parser_t *parser, yp_node_t *target) { - yp_token_t operator = not_provided(parser); - switch (YP_NODE_TYPE(target)) { case YP_NODE_MISSING_NODE: return target; - case YP_NODE_CLASS_VARIABLE_READ_NODE: { - yp_class_variable_write_node_t *write_node = yp_class_variable_read_node_to_class_variable_write_node(parser, (yp_class_variable_read_node_t *) target, &operator, NULL); - yp_node_destroy(parser, target); - return (yp_node_t *) write_node; - } + case YP_NODE_CLASS_VARIABLE_READ_NODE: + assert(sizeof(yp_class_variable_target_node_t) == sizeof(yp_class_variable_read_node_t)); + target->type = YP_NODE_CLASS_VARIABLE_TARGET_NODE; + return target; case YP_NODE_CONSTANT_PATH_NODE: - return (yp_node_t *) yp_constant_path_write_node_create(parser, (yp_constant_path_node_t *) target, &operator, NULL); - case YP_NODE_CONSTANT_READ_NODE: { - yp_constant_write_node_t *node = yp_constant_write_node_create(parser, &target->location, &operator, NULL); - yp_node_destroy(parser, target); - - return (yp_node_t *) node; - } + assert(sizeof(yp_constant_path_target_node_t) == sizeof(yp_constant_path_node_t)); + target->type = YP_NODE_CONSTANT_PATH_TARGET_NODE; + return target; + case YP_NODE_CONSTANT_READ_NODE: + assert(sizeof(yp_constant_target_node_t) == sizeof(yp_constant_read_node_t)); + target->type = YP_NODE_CONSTANT_TARGET_NODE; + return target; case YP_NODE_BACK_REFERENCE_READ_NODE: + assert(sizeof(yp_global_variable_target_node_t) == sizeof(yp_back_reference_read_node_t)); + /* fallthrough */ case YP_NODE_NUMBERED_REFERENCE_READ_NODE: + assert(sizeof(yp_global_variable_target_node_t) == sizeof(yp_numbered_reference_read_node_t)); yp_diagnostic_list_append(&parser->error_list, target->location.start, target->location.end, "Can't set variable"); /* fallthrough */ - case YP_NODE_GLOBAL_VARIABLE_READ_NODE: { - yp_global_variable_write_node_t *result = yp_global_variable_write_node_create(parser, &target->location, &operator, NULL); - yp_node_destroy(parser, target); - - return (yp_node_t *) result; - } - case YP_NODE_LOCAL_VARIABLE_READ_NODE: { - yp_local_variable_read_node_t *local_read = (yp_local_variable_read_node_t *) target; - - yp_constant_id_t constant_id = local_read->constant_id; - uint32_t depth = local_read->depth; - - yp_location_t name_loc = target->location; - yp_node_destroy(parser, target); - - return (yp_node_t *) yp_local_variable_write_node_create(parser, constant_id, depth, NULL, &name_loc, &operator); - } - case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { - yp_node_t *write_node = (yp_node_t *) yp_instance_variable_write_node_create(parser, (yp_instance_variable_read_node_t *) target, &operator, NULL); - yp_node_destroy(parser, target); - return write_node; - } - case YP_NODE_MULTI_WRITE_NODE: { - yp_multi_write_node_t *multi_write = (yp_multi_write_node_t *) target; - yp_multi_write_node_operator_loc_set(multi_write, &operator); - return (yp_node_t *) multi_write; - } + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + assert(sizeof(yp_global_variable_target_node_t) == sizeof(yp_global_variable_read_node_t)); + target->type = YP_NODE_GLOBAL_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_LOCAL_VARIABLE_READ_NODE: + assert(sizeof(yp_local_variable_target_node_t) == sizeof(yp_local_variable_read_node_t)); + target->type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: + assert(sizeof(yp_instance_variable_target_node_t) == sizeof(yp_instance_variable_read_node_t)); + target->type = YP_NODE_INSTANCE_VARIABLE_TARGET_NODE; + return target; + case YP_NODE_MULTI_WRITE_NODE: + return target; case YP_NODE_SPLAT_NODE: { yp_splat_node_t *splat = (yp_splat_node_t *) target; @@ -7866,7 +7853,9 @@ parse_target(yp_parser_t *parser, yp_node_t *target) { splat->expression = parse_target(parser, splat->expression); } + yp_token_t operator = not_provided(parser); yp_location_t location = { .start = NULL, .end = NULL }; + yp_multi_write_node_t *multi_write = yp_multi_write_node_create(parser, &operator, NULL, &location, &location); yp_multi_write_node_targets_append(multi_write, (yp_node_t *) splat); @@ -7874,6 +7863,7 @@ parse_target(yp_parser_t *parser, yp_node_t *target) { } case YP_NODE_CALL_NODE: { yp_call_node_t *call = (yp_call_node_t *) target; + // If we have no arguments to the call node and we need this to be a // target then this is either a method call or a local variable write. if ( @@ -7896,8 +7886,11 @@ parse_target(yp_parser_t *parser, yp_node_t *target) { yp_parser_local_add_location(parser, message.start, message.end); yp_node_destroy(parser, target); - yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, message.start, message.end); - target = (yp_node_t *) yp_local_variable_write_node_create(parser, constant_id, 0, NULL, &message, &operator); + const yp_token_t name = { .type = YP_TOKEN_IDENTIFIER, .start = message.start, .end = message.end }; + target = (yp_node_t *) yp_local_variable_read_node_create(parser, &name, 0); + + assert(sizeof(yp_local_variable_target_node_t) == sizeof(yp_local_variable_read_node_t)); + target->type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE; if (token_is_numbered_parameter(message.start, message.end)) { yp_diagnostic_list_append(&parser->error_list, message.start, message.end, "reserved for numbered parameter"); @@ -7940,10 +7933,10 @@ parse_target(yp_parser_t *parser, yp_node_t *target) { } /* fallthrough */ default: - // In this case we have a node that we don't know how to convert into a - // target. We need to treat it as an error. For now, we'll mark it as an - // error and just skip right past it. - yp_diagnostic_list_append(&parser->error_list, operator.start, operator.end, "Unexpected `='."); + // In this case we have a node that we don't know how to convert + // into a target. We need to treat it as an error. For now, we'll + // mark it as an error and just skip right past it. + yp_diagnostic_list_append(&parser->error_list, target->location.start, target->location.end, "Unexpected write target."); return target; } } From 0df515c0959ec3b501132cf511a56f47d6f93682 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 13:03:16 -0400 Subject: [PATCH 006/208] [ruby/yarp] Mark local variable targets in pattern matching https://github.com/ruby/yarp/commit/6c6700a001 --- test/yarp/snapshots/patterns.txt | 366 +++--------------- test/yarp/snapshots/seattlerb/case_in.txt | 36 +- test/yarp/snapshots/seattlerb/case_in_31.txt | 2 +- test/yarp/snapshots/seattlerb/case_in_42.txt | 2 +- .../yarp/snapshots/seattlerb/case_in_42_2.txt | 2 +- .../seattlerb/case_in_array_pat_const.txt | 2 +- .../seattlerb/case_in_array_pat_const2.txt | 2 +- .../case_in_array_pat_paren_assign.txt | 2 +- .../yarp/snapshots/seattlerb/case_in_find.txt | 4 +- .../seattlerb/case_in_find_array.txt | 2 +- .../seattlerb/case_in_hash_pat_assign.txt | 2 +- .../seattlerb/case_in_hash_pat_rest.txt | 4 +- .../seattlerb/case_in_hash_pat_rest_solo.txt | 2 +- .../snapshots/seattlerb/parse_pattern_058.txt | 2 +- test/yarp/snapshots/seattlerb/rhs_asgn.txt | 2 +- .../unparser/corpus/literal/pattern.txt | 14 +- .../unparser/corpus/literal/since/30.txt | 6 +- .../pattern_matching_single_line.txt | 4 +- ...e_line_allowed_omission_of_parentheses.txt | 18 +- yarp/config.yml | 20 +- yarp/yarp.c | 15 +- 21 files changed, 109 insertions(+), 400 deletions(-) diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index d578a43f066cb2..a55f1ba2924a7e 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -3,7 +3,7 @@ ProgramNode(0...3743)( StatementsNode(0...3743)( [MatchRequiredNode(0...10)( CallNode(0...3)(nil, nil, (0...3), nil, nil, nil, nil, 2, "foo"), - LocalVariableWriteNode(7...10)(:bar, 0, nil, (7...10), nil), + LocalVariableTargetNode(7...10)(:bar, 0), (4...6) ), MatchRequiredNode(11...19)( @@ -1206,13 +1206,7 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1169...1177)( ConstantReadNode(1169...1172)(), - [LocalVariableWriteNode(1173...1176)( - :bar, - 0, - nil, - (1173...1176), - nil - )], + [LocalVariableTargetNode(1173...1176)(:bar, 0)], nil, [], (1172...1173), @@ -1237,21 +1231,9 @@ ProgramNode(0...3743)( [], SplatNode(1189...1193)( (1189...1190), - LocalVariableWriteNode(1190...1193)( - :bar, - 0, - nil, - (1190...1193), - nil - ) + LocalVariableTargetNode(1190...1193)(:bar, 0) ), - [LocalVariableWriteNode(1195...1198)( - :baz, - 0, - nil, - (1195...1198), - nil - )], + [LocalVariableTargetNode(1195...1198)(:baz, 0)], (1188...1189), (1198...1199) ), @@ -1271,22 +1253,10 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1207...1221)( ConstantReadNode(1207...1210)(), - [LocalVariableWriteNode(1211...1214)( - :bar, - 0, - nil, - (1211...1214), - nil - )], + [LocalVariableTargetNode(1211...1214)(:bar, 0)], SplatNode(1216...1220)( (1216...1217), - LocalVariableWriteNode(1217...1220)( - :baz, - 0, - nil, - (1217...1220), - nil - ) + LocalVariableTargetNode(1217...1220)(:baz, 0) ), [], (1210...1211), @@ -1310,30 +1280,12 @@ ProgramNode(0...3743)( ConstantReadNode(1229...1232)(), SplatNode(1233...1237)( (1233...1234), - LocalVariableWriteNode(1234...1237)( - :bar, - 0, - nil, - (1234...1237), - nil - ) + LocalVariableTargetNode(1234...1237)(:bar, 0) ), - [LocalVariableWriteNode(1239...1242)( - :baz, - 0, - nil, - (1239...1242), - nil - )], + [LocalVariableTargetNode(1239...1242)(:baz, 0)], SplatNode(1244...1248)( (1244...1245), - LocalVariableWriteNode(1245...1248)( - :qux, - 0, - nil, - (1245...1248), - nil - ) + LocalVariableTargetNode(1245...1248)(:qux, 0) ), (1232...1233), (1248...1249) @@ -1451,13 +1403,7 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1323...1331)( ConstantReadNode(1323...1326)(), - [LocalVariableWriteNode(1327...1330)( - :bar, - 0, - nil, - (1327...1330), - nil - )], + [LocalVariableTargetNode(1327...1330)(:bar, 0)], nil, [], (1326...1327), @@ -1482,21 +1428,9 @@ ProgramNode(0...3743)( [], SplatNode(1343...1347)( (1343...1344), - LocalVariableWriteNode(1344...1347)( - :bar, - 0, - nil, - (1344...1347), - nil - ) + LocalVariableTargetNode(1344...1347)(:bar, 0) ), - [LocalVariableWriteNode(1349...1352)( - :baz, - 0, - nil, - (1349...1352), - nil - )], + [LocalVariableTargetNode(1349...1352)(:baz, 0)], (1342...1343), (1352...1353) ), @@ -1516,22 +1450,10 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1361...1375)( ConstantReadNode(1361...1364)(), - [LocalVariableWriteNode(1365...1368)( - :bar, - 0, - nil, - (1365...1368), - nil - )], + [LocalVariableTargetNode(1365...1368)(:bar, 0)], SplatNode(1370...1374)( (1370...1371), - LocalVariableWriteNode(1371...1374)( - :baz, - 0, - nil, - (1371...1374), - nil - ) + LocalVariableTargetNode(1371...1374)(:baz, 0) ), [], (1364...1365), @@ -1555,30 +1477,12 @@ ProgramNode(0...3743)( ConstantReadNode(1383...1386)(), SplatNode(1387...1391)( (1387...1388), - LocalVariableWriteNode(1388...1391)( - :bar, - 0, - nil, - (1388...1391), - nil - ) + LocalVariableTargetNode(1388...1391)(:bar, 0) ), - [LocalVariableWriteNode(1393...1396)( - :baz, - 0, - nil, - (1393...1396), - nil - )], + [LocalVariableTargetNode(1393...1396)(:baz, 0)], SplatNode(1398...1402)( (1398...1399), - LocalVariableWriteNode(1399...1402)( - :qux, - 0, - nil, - (1399...1402), - nil - ) + LocalVariableTargetNode(1399...1402)(:qux, 0) ), (1386...1387), (1402...1403) @@ -1602,13 +1506,7 @@ ProgramNode(0...3743)( [], SplatNode(1412...1416)( (1412...1413), - LocalVariableWriteNode(1413...1416)( - :bar, - 0, - nil, - (1413...1416), - nil - ) + LocalVariableTargetNode(1413...1416)(:bar, 0) ), [], nil, @@ -1633,28 +1531,10 @@ ProgramNode(0...3743)( [], SplatNode(1424...1428)( (1424...1425), - LocalVariableWriteNode(1425...1428)( - :bar, - 0, - nil, - (1425...1428), - nil - ) + LocalVariableTargetNode(1425...1428)(:bar, 0) ), - [LocalVariableWriteNode(1430...1433)( - :baz, - 0, - nil, - (1430...1433), - nil - ), - LocalVariableWriteNode(1435...1438)( - :qux, - 0, - nil, - (1435...1438), - nil - )], + [LocalVariableTargetNode(1430...1433)(:baz, 0), + LocalVariableTargetNode(1435...1438)(:qux, 0)], nil, nil ), @@ -1674,30 +1554,12 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1446...1460)( nil, - [LocalVariableWriteNode(1446...1449)( - :bar, - 0, - nil, - (1446...1449), - nil - )], + [LocalVariableTargetNode(1446...1449)(:bar, 0)], SplatNode(1451...1455)( (1451...1452), - LocalVariableWriteNode(1452...1455)( - :baz, - 0, - nil, - (1452...1455), - nil - ) + LocalVariableTargetNode(1452...1455)(:baz, 0) ), - [LocalVariableWriteNode(1457...1460)( - :qux, - 0, - nil, - (1457...1460), - nil - )], + [LocalVariableTargetNode(1457...1460)(:qux, 0)], nil, nil ), @@ -1717,29 +1579,11 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1468...1482)( nil, - [LocalVariableWriteNode(1468...1471)( - :bar, - 0, - nil, - (1468...1471), - nil - ), - LocalVariableWriteNode(1473...1476)( - :baz, - 0, - nil, - (1473...1476), - nil - )], + [LocalVariableTargetNode(1468...1471)(:bar, 0), + LocalVariableTargetNode(1473...1476)(:baz, 0)], SplatNode(1478...1482)( (1478...1479), - LocalVariableWriteNode(1479...1482)( - :qux, - 0, - nil, - (1479...1482), - nil - ) + LocalVariableTargetNode(1479...1482)(:qux, 0) ), [], nil, @@ -1763,30 +1607,12 @@ ProgramNode(0...3743)( nil, SplatNode(1490...1494)( (1490...1491), - LocalVariableWriteNode(1491...1494)( - :bar, - 0, - nil, - (1491...1494), - nil - ) + LocalVariableTargetNode(1491...1494)(:bar, 0) ), - [LocalVariableWriteNode(1496...1499)( - :baz, - 0, - nil, - (1496...1499), - nil - )], + [LocalVariableTargetNode(1496...1499)(:baz, 0)], SplatNode(1501...1505)( (1501...1502), - LocalVariableWriteNode(1502...1505)( - :qux, - 0, - nil, - (1502...1505), - nil - ) + LocalVariableTargetNode(1502...1505)(:qux, 0) ), nil, nil @@ -1882,13 +1708,7 @@ ProgramNode(0...3743)( [], SplatNode(1544...1548)( (1544...1545), - LocalVariableWriteNode(1545...1548)( - :bar, - 0, - nil, - (1545...1548), - nil - ) + LocalVariableTargetNode(1545...1548)(:bar, 0) ), [], (1543...1544), @@ -1913,28 +1733,10 @@ ProgramNode(0...3743)( [], SplatNode(1558...1562)( (1558...1559), - LocalVariableWriteNode(1559...1562)( - :bar, - 0, - nil, - (1559...1562), - nil - ) + LocalVariableTargetNode(1559...1562)(:bar, 0) ), - [LocalVariableWriteNode(1564...1567)( - :baz, - 0, - nil, - (1564...1567), - nil - ), - LocalVariableWriteNode(1569...1572)( - :qux, - 0, - nil, - (1569...1572), - nil - )], + [LocalVariableTargetNode(1564...1567)(:baz, 0), + LocalVariableTargetNode(1569...1572)(:qux, 0)], (1557...1558), (1572...1573) ), @@ -1954,30 +1756,12 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1581...1597)( nil, - [LocalVariableWriteNode(1582...1585)( - :bar, - 0, - nil, - (1582...1585), - nil - )], + [LocalVariableTargetNode(1582...1585)(:bar, 0)], SplatNode(1587...1591)( (1587...1588), - LocalVariableWriteNode(1588...1591)( - :baz, - 0, - nil, - (1588...1591), - nil - ) + LocalVariableTargetNode(1588...1591)(:baz, 0) ), - [LocalVariableWriteNode(1593...1596)( - :qux, - 0, - nil, - (1593...1596), - nil - )], + [LocalVariableTargetNode(1593...1596)(:qux, 0)], (1581...1582), (1596...1597) ), @@ -1997,29 +1781,11 @@ ProgramNode(0...3743)( ), ArrayPatternNode(1605...1621)( nil, - [LocalVariableWriteNode(1606...1609)( - :bar, - 0, - nil, - (1606...1609), - nil - ), - LocalVariableWriteNode(1611...1614)( - :baz, - 0, - nil, - (1611...1614), - nil - )], + [LocalVariableTargetNode(1606...1609)(:bar, 0), + LocalVariableTargetNode(1611...1614)(:baz, 0)], SplatNode(1616...1620)( (1616...1617), - LocalVariableWriteNode(1617...1620)( - :qux, - 0, - nil, - (1617...1620), - nil - ) + LocalVariableTargetNode(1617...1620)(:qux, 0) ), [], (1605...1606), @@ -2043,30 +1809,12 @@ ProgramNode(0...3743)( nil, SplatNode(1630...1634)( (1630...1631), - LocalVariableWriteNode(1631...1634)( - :bar, - 0, - nil, - (1631...1634), - nil - ) + LocalVariableTargetNode(1631...1634)(:bar, 0) ), - [LocalVariableWriteNode(1636...1639)( - :baz, - 0, - nil, - (1636...1639), - nil - )], + [LocalVariableTargetNode(1636...1639)(:baz, 0)], SplatNode(1641...1645)( (1641...1642), - LocalVariableWriteNode(1642...1645)( - :qux, - 0, - nil, - (1642...1645), - nil - ) + LocalVariableTargetNode(1642...1645)(:qux, 0) ), (1629...1630), (1645...1646) @@ -2085,7 +1833,7 @@ ProgramNode(0...3743)( 2, "foo" ), - LocalVariableWriteNode(1655...1658)(:bar, 0, nil, (1655...1658), nil), + LocalVariableTargetNode(1655...1658)(:bar, 0), (1652...1654) ), MatchPredicateNode(1659...1667)( @@ -2535,13 +2283,7 @@ ProgramNode(0...3743)( "foo" ), [InNode(2013...2024)( - LocalVariableWriteNode(2016...2019)( - :bar, - 0, - nil, - (2016...2019), - nil - ), + LocalVariableTargetNode(2016...2019)(:bar, 0), nil, (2013...2015), (2020...2024) @@ -3176,13 +2918,7 @@ ProgramNode(0...3743)( (2765...2767), LocalVariableReadNode(2768...2771)(:baz, 0), StatementsNode(2761...2764)( - [LocalVariableWriteNode(2761...2764)( - :bar, - 0, - nil, - (2761...2764), - nil - )] + [LocalVariableTargetNode(2761...2764)(:bar, 0)] ), nil, nil @@ -4035,7 +3771,7 @@ ProgramNode(0...3743)( ), ArrayPatternNode(3696...3703)( nil, - [LocalVariableWriteNode(3700...3701)(:b, 0, nil, (3700...3701), nil)], + [LocalVariableTargetNode(3700...3701)(:b, 0)], nil, [], (3696...3697), @@ -4068,13 +3804,7 @@ ProgramNode(0...3743)( (3734...3735), "value" ), - LocalVariableWriteNode(3736...3737)( - :a, - 0, - nil, - (3736...3737), - nil - ), + LocalVariableTargetNode(3736...3737)(:a, 0), nil )], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index 8277b654c117ea..b83e196eceb2a8 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -195,7 +195,7 @@ ProgramNode(0...747)( [SymbolNode(292...294)((292...293), (293...294), nil, "b")], SplatNode(296...298)( (296...297), - LocalVariableWriteNode(297...298)(:_, 0, nil, (297...298), nil) + LocalVariableTargetNode(297...298)(:_, 0) ), [SymbolNode(300...302)((300...301), (301...302), nil, "c")], nil, @@ -262,12 +262,12 @@ ProgramNode(0...747)( ConstantReadNode(369...375)(), SplatNode(376...380)( (376...377), - LocalVariableWriteNode(377...380)(:lhs, 0, nil, (377...380), nil) + LocalVariableTargetNode(377...380)(:lhs, 0) ), - [LocalVariableWriteNode(382...383)(:x, 0, nil, (382...383), nil)], + [LocalVariableTargetNode(382...383)(:x, 0)], SplatNode(385...389)( (385...386), - LocalVariableWriteNode(386...389)(:rhs, 0, nil, (386...389), nil) + LocalVariableTargetNode(386...389)(:rhs, 0) ), (375...376), (389...390) @@ -287,12 +287,12 @@ ProgramNode(0...747)( ConstantReadNode(407...413)(), SplatNode(414...418)( (414...415), - LocalVariableWriteNode(415...418)(:lhs, 0, nil, (415...418), nil) + LocalVariableTargetNode(415...418)(:lhs, 0) ), - [LocalVariableWriteNode(420...421)(:x, 0, nil, (420...421), nil)], + [LocalVariableTargetNode(420...421)(:x, 0)], SplatNode(423...427)( (423...424), - LocalVariableWriteNode(424...427)(:rhs, 0, nil, (424...427), nil) + LocalVariableTargetNode(424...427)(:rhs, 0) ), (413...414), (427...428) @@ -329,7 +329,7 @@ ProgramNode(0...747)( ), StatementsNode(454...458)([TrueNode(454...458)()]) ), - LocalVariableWriteNode(462...463)(:c, 0, nil, (462...463), nil)], + LocalVariableTargetNode(462...463)(:c, 0)], nil, [], (445...446), @@ -349,20 +349,14 @@ ProgramNode(0...747)( ArrayPatternNode(481...506)( nil, [SymbolNode(482...484)((482...483), (483...484), nil, "a"), - LocalVariableWriteNode(486...487)(:b, 0, nil, (486...487), nil), - LocalVariableWriteNode(489...490)(:c, 0, nil, (489...490), nil), + LocalVariableTargetNode(486...487)(:b, 0), + LocalVariableTargetNode(489...490)(:c, 0), ArrayPatternNode(492...505)( nil, [SymbolNode(493...495)((493...494), (494...495), nil, "d")], SplatNode(497...499)( (497...498), - LocalVariableWriteNode(498...499)( - :e, - 0, - nil, - (498...499), - nil - ) + LocalVariableTargetNode(498...499)(:e, 0) ), [NilNode(501...504)()], (492...493), @@ -408,13 +402,7 @@ ProgramNode(0...747)( [ArrayPatternNode(550...557)( nil, [SymbolNode(551...553)((551...552), (552...553), nil, "b"), - LocalVariableWriteNode(555...556)( - :c, - 0, - nil, - (555...556), - nil - )], + LocalVariableTargetNode(555...556)(:c, 0)], nil, [], (550...551), diff --git a/test/yarp/snapshots/seattlerb/case_in_31.txt b/test/yarp/snapshots/seattlerb/case_in_31.txt index 26ff5c913fa23c..c34b271b848e76 100644 --- a/test/yarp/snapshots/seattlerb/case_in_31.txt +++ b/test/yarp/snapshots/seattlerb/case_in_31.txt @@ -9,7 +9,7 @@ ProgramNode(0...28)( [SymbolNode(12...14)((12...13), (13...14), nil, "b")], SplatNode(16...18)( (16...17), - LocalVariableWriteNode(17...18)(:c, 0, nil, (17...18), nil) + LocalVariableTargetNode(17...18)(:c, 0) ), [], (11...12), diff --git a/test/yarp/snapshots/seattlerb/case_in_42.txt b/test/yarp/snapshots/seattlerb/case_in_42.txt index 7cd17bb0e1ca50..b98fd9f4b78f60 100644 --- a/test/yarp/snapshots/seattlerb/case_in_42.txt +++ b/test/yarp/snapshots/seattlerb/case_in_42.txt @@ -9,7 +9,7 @@ ProgramNode(0...30)( [SymbolNode(11...13)((11...12), (12...13), nil, "b")], SplatNode(15...17)( (15...16), - LocalVariableWriteNode(16...17)(:_, 0, nil, (16...17), nil) + LocalVariableTargetNode(16...17)(:_, 0) ), [], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_42_2.txt b/test/yarp/snapshots/seattlerb/case_in_42_2.txt index 7c0c87e98c03a4..945be4ddcf574a 100644 --- a/test/yarp/snapshots/seattlerb/case_in_42_2.txt +++ b/test/yarp/snapshots/seattlerb/case_in_42_2.txt @@ -9,7 +9,7 @@ ProgramNode(0...32)( [], SplatNode(13...18)( (13...14), - LocalVariableWriteNode(14...18)(:list, 0, nil, (14...18), nil) + LocalVariableTargetNode(14...18)(:list, 0) ), [], (12...13), diff --git a/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt b/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt index b8eee482039b25..d503434f7e5ec7 100644 --- a/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt +++ b/test/yarp/snapshots/seattlerb/case_in_array_pat_const.txt @@ -6,7 +6,7 @@ ProgramNode(0...24)( [InNode(8...20)( ArrayPatternNode(11...15)( ConstantReadNode(11...12)(), - [LocalVariableWriteNode(13...14)(:c, 0, nil, (13...14), nil)], + [LocalVariableTargetNode(13...14)(:c, 0)], nil, [], (12...13), diff --git a/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt b/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt index 2e29ef9d16c9ec..828d83dd52dc92 100644 --- a/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt +++ b/test/yarp/snapshots/seattlerb/case_in_array_pat_const2.txt @@ -10,7 +10,7 @@ ProgramNode(0...27)( ConstantReadNode(14...15)(), (12...14) ), - [LocalVariableWriteNode(16...17)(:d, 0, nil, (16...17), nil)], + [LocalVariableTargetNode(16...17)(:d, 0)], nil, [], (15...16), diff --git a/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt b/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt index 85b069fb001cfb..c4e1380f7b1748 100644 --- a/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt +++ b/test/yarp/snapshots/seattlerb/case_in_array_pat_paren_assign.txt @@ -8,7 +8,7 @@ ProgramNode(0...29)( ConstantReadNode(11...12)(), [CapturePatternNode(13...19)( ConstantReadNode(13...14)(), - LocalVariableWriteNode(18...19)(:d, 0, nil, (18...19), nil), + LocalVariableTargetNode(18...19)(:d, 0), (15...17) )], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_find.txt b/test/yarp/snapshots/seattlerb/case_in_find.txt index 2d366423694d2a..3c8e6442448236 100644 --- a/test/yarp/snapshots/seattlerb/case_in_find.txt +++ b/test/yarp/snapshots/seattlerb/case_in_find.txt @@ -8,12 +8,12 @@ ProgramNode(0...27)( nil, SplatNode(13...15)( (13...14), - LocalVariableWriteNode(14...15)(:a, 0, nil, (14...15), nil) + LocalVariableTargetNode(14...15)(:a, 0) ), [SymbolNode(17...19)((17...18), (18...19), nil, "+")], SplatNode(21...23)( (21...22), - LocalVariableWriteNode(22...23)(:b, 0, nil, (22...23), nil) + LocalVariableTargetNode(22...23)(:b, 0) ), nil, nil diff --git a/test/yarp/snapshots/seattlerb/case_in_find_array.txt b/test/yarp/snapshots/seattlerb/case_in_find_array.txt index 410fda576aa5ca..616c8f864bc634 100644 --- a/test/yarp/snapshots/seattlerb/case_in_find_array.txt +++ b/test/yarp/snapshots/seattlerb/case_in_find_array.txt @@ -8,7 +8,7 @@ ProgramNode(0...28)( nil, SplatNode(12...13)((12...13), nil), [SymbolNode(15...17)((15...16), (16...17), nil, "b"), - LocalVariableWriteNode(19...20)(:c, 0, nil, (19...20), nil)], + LocalVariableTargetNode(19...20)(:c, 0)], SplatNode(22...23)((22...23), nil), (11...12), (23...24) diff --git a/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt b/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt index 92933ca0718055..6c6c9e8b675c2b 100644 --- a/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt +++ b/test/yarp/snapshots/seattlerb/case_in_hash_pat_assign.txt @@ -10,7 +10,7 @@ ProgramNode(0...56)( SymbolNode(13...15)(nil, (13...14), (14...15), "b"), CapturePatternNode(16...28)( ConstantReadNode(16...23)(), - LocalVariableWriteNode(27...28)(:x, 0, nil, (27...28), nil), + LocalVariableTargetNode(27...28)(:x, 0), (24...26) ), nil diff --git a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt index ed7b16685280e2..9998de3cfe6b15 100644 --- a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt +++ b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest.txt @@ -8,11 +8,11 @@ ProgramNode(0...35)( nil, [AssocNode(11...15)( SymbolNode(11...13)(nil, (11...12), (12...13), "b"), - LocalVariableWriteNode(14...15)(:c, 0, nil, (14...15), nil), + LocalVariableTargetNode(14...15)(:c, 0), nil ), AssocSplatNode(17...23)( - LocalVariableWriteNode(19...23)(:rest, 0, nil, (19...23), nil), + LocalVariableTargetNode(19...23)(:rest, 0), (17...19) )], nil, diff --git a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt index 7bdc02e6fb8af4..7d6634c7f8d3d6 100644 --- a/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt +++ b/test/yarp/snapshots/seattlerb/case_in_hash_pat_rest_solo.txt @@ -7,7 +7,7 @@ ProgramNode(0...29)( HashPatternNode(11...17)( nil, [AssocSplatNode(11...17)( - LocalVariableWriteNode(13...17)(:rest, 0, nil, (13...17), nil), + LocalVariableTargetNode(13...17)(:rest, 0), (11...13) )], nil, diff --git a/test/yarp/snapshots/seattlerb/parse_pattern_058.txt b/test/yarp/snapshots/seattlerb/parse_pattern_058.txt index 23b0ba5d102d0e..d9416d4a61c613 100644 --- a/test/yarp/snapshots/seattlerb/parse_pattern_058.txt +++ b/test/yarp/snapshots/seattlerb/parse_pattern_058.txt @@ -20,7 +20,7 @@ ProgramNode(0...43)( nil ), AssocSplatNode(20...26)( - LocalVariableWriteNode(22...26)(:rest, 0, nil, (22...26), nil), + LocalVariableTargetNode(22...26)(:rest, 0), (20...22) )], nil, diff --git a/test/yarp/snapshots/seattlerb/rhs_asgn.txt b/test/yarp/snapshots/seattlerb/rhs_asgn.txt index 7ba161b170a660..604a14667b1e6d 100644 --- a/test/yarp/snapshots/seattlerb/rhs_asgn.txt +++ b/test/yarp/snapshots/seattlerb/rhs_asgn.txt @@ -3,7 +3,7 @@ ProgramNode(0...7)( StatementsNode(0...7)( [MatchRequiredNode(0...7)( IntegerNode(0...2)(), - LocalVariableWriteNode(6...7)(:n, 0, nil, (6...7), nil), + LocalVariableTargetNode(6...7)(:n, 0), (3...5) )] ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/pattern.txt b/test/yarp/snapshots/unparser/corpus/literal/pattern.txt index 9488cc78502614..227938293bdc78 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/pattern.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/pattern.txt @@ -9,7 +9,7 @@ ProgramNode(0...408)( [IntegerNode(14...15)(), IntegerNode(17...18)()], SplatNode(20...22)( (20...21), - LocalVariableWriteNode(21...22)(:a, 0, nil, (21...22), nil) + LocalVariableTargetNode(21...22)(:a, 0) ), [IntegerNode(24...25)()], (13...14), @@ -65,7 +65,7 @@ ProgramNode(0...408)( nil, [AssocNode(85...88)( AssocSplatNode(85...88)( - LocalVariableWriteNode(87...88)(:a, 0, nil, (87...88), nil), + LocalVariableTargetNode(87...88)(:a, 0), (85...87) ), nil, @@ -102,8 +102,8 @@ ProgramNode(0...408)( InNode(128...152)( ArrayPatternNode(131...140)( nil, - [LocalVariableWriteNode(132...133)(:x, 0, nil, (132...133), nil), - LocalVariableWriteNode(135...136)(:y, 0, nil, (135...136), nil)], + [LocalVariableTargetNode(132...133)(:x, 0), + LocalVariableTargetNode(135...136)(:y, 0)], SplatNode(138...139)((138...139), nil), [], (131...132), @@ -190,7 +190,7 @@ ProgramNode(0...408)( InNode(268...289)( CapturePatternNode(271...277)( IntegerNode(271...272)(), - LocalVariableWriteNode(276...277)(:a, 0, nil, (276...277), nil), + LocalVariableTargetNode(276...277)(:a, 0), (273...275) ), StatementsNode(285...289)([TrueNode(285...289)()]), @@ -239,7 +239,7 @@ ProgramNode(0...408)( [IntegerNode(360...361)(), IntegerNode(363...364)()], SplatNode(366...368)( (366...367), - LocalVariableWriteNode(367...368)(:a, 0, nil, (367...368), nil) + LocalVariableTargetNode(367...368)(:a, 0) ), [IntegerNode(370...371)()], (359...360), @@ -279,7 +279,7 @@ ProgramNode(0...408)( IntegerNode(400...401)(), ArrayPatternNode(405...408)( nil, - [LocalVariableWriteNode(406...407)(:a, 0, nil, (406...407), nil)], + [LocalVariableTargetNode(406...407)(:a, 0)], nil, [], (405...406), diff --git a/test/yarp/snapshots/unparser/corpus/literal/since/30.txt b/test/yarp/snapshots/unparser/corpus/literal/since/30.txt index 15d5136d8308f9..4aba7677950132 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/since/30.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/since/30.txt @@ -5,7 +5,7 @@ ProgramNode(0...51)( IntegerNode(0...1)(), ArrayPatternNode(5...8)( nil, - [LocalVariableWriteNode(6...7)(:a, 0, nil, (6...7), nil)], + [LocalVariableTargetNode(6...7)(:a, 0)], nil, [], (5...6), @@ -42,10 +42,10 @@ ProgramNode(0...51)( FindPatternNode(39...51)( nil, SplatNode(40...41)((40...41), nil), - [LocalVariableWriteNode(43...44)(:a, 0, nil, (43...44), nil)], + [LocalVariableTargetNode(43...44)(:a, 0)], SplatNode(46...50)( (46...47), - LocalVariableWriteNode(47...50)(:foo, 0, nil, (47...50), nil) + LocalVariableTargetNode(47...50)(:foo, 0) ), (39...40), (50...51) diff --git a/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt b/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt index 7a0546c56951ec..c428d83cda5e3d 100644 --- a/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt +++ b/test/yarp/snapshots/whitequark/pattern_matching_single_line.txt @@ -5,7 +5,7 @@ ProgramNode(0...24)( IntegerNode(0...1)(), ArrayPatternNode(5...8)( nil, - [LocalVariableWriteNode(6...7)(:a, 0, nil, (6...7), nil)], + [LocalVariableTargetNode(6...7)(:a, 0)], nil, [], (5...6), @@ -18,7 +18,7 @@ ProgramNode(0...24)( IntegerNode(13...14)(), ArrayPatternNode(18...21)( nil, - [LocalVariableWriteNode(19...20)(:a, 0, nil, (19...20), nil)], + [LocalVariableTargetNode(19...20)(:a, 0)], nil, [], (18...19), diff --git a/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt b/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt index c6d53befee0d44..1e009aa3db8a71 100644 --- a/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt +++ b/test/yarp/snapshots/whitequark/pattern_matching_single_line_allowed_omission_of_parentheses.txt @@ -9,8 +9,8 @@ ProgramNode(0...142)( ), ArrayPatternNode(10...14)( nil, - [LocalVariableWriteNode(10...11)(:a, 0, nil, (10...11), nil), - LocalVariableWriteNode(13...14)(:b, 0, nil, (13...14), nil)], + [LocalVariableTargetNode(10...11)(:a, 0), + LocalVariableTargetNode(13...14)(:b, 0)], nil, [], nil, @@ -27,8 +27,8 @@ ProgramNode(0...142)( ), ArrayPatternNode(29...33)( nil, - [LocalVariableWriteNode(29...30)(:a, 0, nil, (29...30), nil), - LocalVariableWriteNode(32...33)(:b, 0, nil, (32...33), nil)], + [LocalVariableTargetNode(29...30)(:a, 0), + LocalVariableTargetNode(32...33)(:b, 0)], nil, [], nil, @@ -99,7 +99,7 @@ ProgramNode(0...142)( nil, [AssocNode(89...99)( SymbolNode(89...93)(nil, (89...92), (92...93), "key"), - LocalVariableWriteNode(94...99)(:value, 0, nil, (94...99), nil), + LocalVariableTargetNode(94...99)(:value, 0), nil )], nil, @@ -123,13 +123,7 @@ ProgramNode(0...142)( nil, [AssocNode(125...135)( SymbolNode(125...129)(nil, (125...128), (128...129), "key"), - LocalVariableWriteNode(130...135)( - :value, - 0, - nil, - (130...135), - nil - ), + LocalVariableTargetNode(130...135)(:value, 0), nil )], nil, diff --git a/yarp/config.yml b/yarp/config.yml index bb4d4174dac1c4..d7269f336d3069 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -908,9 +908,9 @@ nodes: type: node kind: ConstantPathNode - name: operator_loc - type: location? + type: location - name: value - type: node? + type: node comment: | Represents writing to a constant path. @@ -939,9 +939,9 @@ nodes: - name: name_loc type: location - name: value - type: node? + type: node - name: operator_loc - type: location? + type: location comment: | Represents writing to a constant. @@ -1208,9 +1208,9 @@ nodes: - name: name_loc type: location - name: operator_loc - type: location? + type: location - name: value - type: node? + type: node comment: | Represents writing to a global variable. @@ -1354,9 +1354,9 @@ nodes: - name: name_loc type: location - name: value - type: node? + type: node - name: operator_loc - type: location? + type: location comment: | Represents writing to an instance variable. @@ -1564,11 +1564,11 @@ nodes: - name: depth type: uint32 - name: value - type: node? + type: node - name: name_loc type: location - name: operator_loc - type: location? + type: location comment: | Represents writing to a local variable. diff --git a/yarp/yarp.c b/yarp/yarp.c index f8fd94997da119..ad762d89a32d59 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -3075,21 +3075,18 @@ yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t consta return node; } -// Allocate and initialize a new LocalVariableWriteNode node without an operator or target. -static yp_local_variable_write_node_t * +// Allocate and initialize a new LocalVariableTargetNode node. +static yp_local_variable_target_node_t * yp_local_variable_target_node_create(yp_parser_t *parser, const yp_token_t *name) { - yp_local_variable_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_write_node_t); + yp_local_variable_target_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_target_node_t); - *node = (yp_local_variable_write_node_t) { + *node = (yp_local_variable_target_node_t) { { - .type = YP_NODE_LOCAL_VARIABLE_WRITE_NODE, + .type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE, .location = YP_LOCATION_TOKEN_VALUE(name) }, .constant_id = yp_parser_constant_id_token(parser, name), - .depth = 0, - .value = NULL, - .name_loc = YP_LOCATION_TOKEN_VALUE(name), - .operator_loc = { .start = NULL, .end = NULL } + .depth = 0 }; return node; From dd07b70253e958e547aff607d6a95a0bad77d56e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:33:14 -0400 Subject: [PATCH 007/208] Fix mutation visitor build for YARP templating --- .gitignore | 1 + common.mk | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 9218a84c4ddacd..631a489eeff1f8 100644 --- a/.gitignore +++ b/.gitignore @@ -256,6 +256,7 @@ lcov*.info /wasm/tests/*.wasm # YARP +/lib/yarp/mutation_visitor.rb /lib/yarp/node.rb /lib/yarp/serialize.rb /yarp/api_node.c diff --git a/common.mk b/common.mk index b0f9369fbc8682..a05f0ddda95479 100644 --- a/common.mk +++ b/common.mk @@ -202,6 +202,11 @@ $(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time $(Q) $(MAKEDIRS) $(@D) @$(NULLCMD) > $@ +main: $(srcdir)/lib/yarp/mutation_visitor.rb +srcs: $(srcdir)/lib/yarp/mutation_visitor.rb +$(srcdir)/lib/yarp/mutation_visitor.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/mutation_visitor.rb.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/mutation_visitor.rb $(srcdir)/lib/yarp/mutation_visitor.rb + main: $(srcdir)/lib/yarp/node.rb srcs: $(srcdir)/lib/yarp/node.rb $(srcdir)/lib/yarp/node.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/node.rb.erb From f603497105bd41c1863490a557ff105bf7e17830 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 21 Aug 2023 16:27:06 +0200 Subject: [PATCH 008/208] [ruby/yarp] Use templating to avoid duplicating the YARP version in many places https://github.com/ruby/yarp/commit/9c359fd92e --- lib/yarp.rb | 1 + lib/yarp/ffi.rb | 6 ++++-- lib/yarp/version.rb | 5 +++++ lib/yarp/yarp.gemspec | 5 ++++- yarp/extension.c | 8 ++------ yarp/extension.h | 2 -- yarp/templates/include/yarp/version.h.erb | 9 +++++++++ yarp/templates/java/org/yarp/Loader.java.erb | 10 +++------- yarp/templates/lib/yarp/serialize.rb.erb | 8 +++----- yarp/templates/template.rb | 6 ++++++ yarp/yarp.h | 1 + 11 files changed, 38 insertions(+), 23 deletions(-) create mode 100644 lib/yarp/version.rb create mode 100644 yarp/templates/include/yarp/version.h.erb diff --git a/lib/yarp.rb b/lib/yarp.rb index b2d4a6eee489ad..5a176e9e62d7c4 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -523,6 +523,7 @@ def self.parse_serialize_file(filepath) require_relative "yarp/ripper_compat" require_relative "yarp/serialize" require_relative "yarp/pack" +require_relative "yarp/version" if RUBY_ENGINE == "ruby" and !ENV["YARP_FFI_BACKEND"] require "yarp/yarp" diff --git a/lib/yarp/ffi.rb b/lib/yarp/ffi.rb index a547d506063f42..31c1ce3cc718b1 100644 --- a/lib/yarp/ffi.rb +++ b/lib/yarp/ffi.rb @@ -171,8 +171,10 @@ def self.with(filepath, &block) # the YARP module. private_constant :LibRubyParser - # The version constant is set by reading the result of calling yp_version. - VERSION = LibRubyParser.yp_version.read_string + library_version = LibRubyParser.yp_version.read_string + if library_version != YARP::VERSION + raise "The YARP library version (#{library_version}) does not match the expected version (#{YARP::VERSION})" + end def self.dump_internal(source, source_size, filepath) LibRubyParser::YPBuffer.with do |buffer| diff --git a/lib/yarp/version.rb b/lib/yarp/version.rb new file mode 100644 index 00000000000000..e450bfb526f866 --- /dev/null +++ b/lib/yarp/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module YARP + VERSION = "0.8.0" +end diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index fceda5b1a0ac86..671eddc7a20e52 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -1,8 +1,10 @@ # frozen_string_literal: true +require_relative 'lib/yarp/version' + Gem::Specification.new do |spec| spec.name = "yarp" - spec.version = "0.8.0" + spec.version = YARP::VERSION spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -67,6 +69,7 @@ Gem::Specification.new do |spec| "lib/yarp/pack.rb", "lib/yarp/ripper_compat.rb", "lib/yarp/serialize.rb", + "lib/yarp/version.rb", "src/diagnostic.c", "src/enc/yp_big5.c", "src/enc/yp_euc_jp.c", diff --git a/yarp/extension.c b/yarp/extension.c index 8aef456c0010a9..3c678902001ca4 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -511,12 +511,12 @@ RUBY_FUNC_EXPORTED void Init_yarp(void) { // Make sure that the YARP library version matches the expected version. // Otherwise something was compiled incorrectly. - if (strcmp(yp_version(), EXPECTED_YARP_VERSION) != 0) { + if (strcmp(yp_version(), YP_VERSION) != 0) { rb_raise( rb_eRuntimeError, "The YARP library version (%s) does not match the expected version (%s)", yp_version(), - EXPECTED_YARP_VERSION + YP_VERSION ); } @@ -532,10 +532,6 @@ Init_yarp(void) { rb_cYARPParseWarning = rb_define_class_under(rb_cYARP, "ParseWarning", rb_cObject); rb_cYARPParseResult = rb_define_class_under(rb_cYARP, "ParseResult", rb_cObject); - // Define the version string here so that we can use the constants defined - // in yarp.h. - rb_define_const(rb_cYARP, "VERSION", rb_str_new2(EXPECTED_YARP_VERSION)); - rb_define_const(rb_cYARP, "BACKEND", ID2SYM(rb_intern("CExtension"))); // First, the functions that have to do with lexing and parsing. diff --git a/yarp/extension.h b/yarp/extension.h index 75d5cc84a7d77b..2796060fb1a085 100644 --- a/yarp/extension.h +++ b/yarp/extension.h @@ -5,8 +5,6 @@ #include #include "yarp.h" -#define EXPECTED_YARP_VERSION "0.8.0" - VALUE yp_source_new(yp_parser_t *parser); VALUE yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALUE source); VALUE yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding); diff --git a/yarp/templates/include/yarp/version.h.erb b/yarp/templates/include/yarp/version.h.erb new file mode 100644 index 00000000000000..4ebcd69b5fc243 --- /dev/null +++ b/yarp/templates/include/yarp/version.h.erb @@ -0,0 +1,9 @@ +#ifndef YARP_VERSION_H +#define YARP_VERSION_H + +#define YP_VERSION_MAJOR <%= YARP_VERSION_MAJOR %> +#define YP_VERSION_MINOR <%= YARP_VERSION_MINOR %> +#define YP_VERSION_PATCH <%= YARP_VERSION_PATCH %> +#define YP_VERSION "<%= YARP_VERSION %>" + +#endif // YARP_VERSION_H diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb index 312e232182592e..3e7dcebafb2a5c 100644 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ b/yarp/templates/java/org/yarp/Loader.java.erb @@ -48,10 +48,6 @@ public class Loader { private ConstantPool constantPool; private final Nodes.Source source; - private byte MAJOR_VERSION = (byte) 0; - private byte MINOR_VERSION = (byte) 8; - private byte PATCH_VERSION = (byte) 0; - private Loader(byte[] serialized, Nodes.Source source) { this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder()); this.source = source; @@ -63,9 +59,9 @@ public class Loader { expect((byte) 'R'); expect((byte) 'P'); - expect(MAJOR_VERSION); - expect(MINOR_VERSION); - expect(PATCH_VERSION); + expect((byte) <%= YARP_VERSION_MAJOR %>); + expect((byte) <%= YARP_VERSION_MINOR %>); + expect((byte) <%= YARP_VERSION_PATCH %>); // This loads the name of the encoding. We don't actually do anything // with it just yet. diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index 8ee072c0b18a19..524405d2edd15c 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -13,10 +13,6 @@ end module YARP module Serialize - MAJOR_VERSION = 0 - MINOR_VERSION = 8 - PATCH_VERSION = 0 - def self.load(input, serialized) Loader.new(Source.new(input), serialized).load end @@ -64,7 +60,9 @@ module YARP def load raise "Invalid serialization" if io.read(4) != "YARP" - raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] + if io.read(3).unpack("C3") != [<%= YARP_VERSION_MAJOR %>, <%= YARP_VERSION_MINOR %>, <%= YARP_VERSION_PATCH %>] + raise "Invalid serialization version" + end @encoding = Encoding.find(io.read(load_varint)) @input = input.force_encoding(@encoding).freeze diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index e34d5b1a7be6ea..4216c68b95af3c 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -4,6 +4,11 @@ require "fileutils" require "yaml" +require_relative "../lib/yarp/version" + +YARP_VERSION = YARP::VERSION +YARP_VERSION_MAJOR, YARP_VERSION_MINOR, YARP_VERSION_PATCH = YARP_VERSION.split(".") + COMMON_FLAGS = 1 class Param @@ -312,6 +317,7 @@ def locals TEMPLATES = [ "ext/yarp/api_node.c", "include/yarp/ast.h", + "include/yarp/version.h", "java/org/yarp/Loader.java", "java/org/yarp/Nodes.java", "java/org/yarp/AbstractNodeVisitor.java", diff --git a/yarp/yarp.h b/yarp/yarp.h index 078483193d58a5..b0879a44545428 100644 --- a/yarp/yarp.h +++ b/yarp/yarp.h @@ -13,6 +13,7 @@ #include "yarp/util/yp_char.h" #include "yarp/util/yp_memchr.h" #include "yarp/util/yp_strpbrk.h" +#include "yarp/version.h" #include #include From d2392e1ce2b9fb113ca19cdcae2a2fe7f8a3c8ec Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:43:04 -0400 Subject: [PATCH 009/208] [ruby/yarp] Fix relative require in template script https://github.com/ruby/yarp/commit/dea9eb5a12 --- yarp/templates/template.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index 4216c68b95af3c..59a779be1f0495 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -4,7 +4,13 @@ require "fileutils" require "yaml" -require_relative "../lib/yarp/version" +if File.exist?(File.expand_path("../lib/yarp/version", __dir__)) + # Within the gem/local repository + require_relative "../lib/yarp/version" +else + # Within CRuby + require_relative "../../lib/yarp/version" +end YARP_VERSION = YARP::VERSION YARP_VERSION_MAJOR, YARP_VERSION_MINOR, YARP_VERSION_PATCH = YARP_VERSION.split(".") From 2e29bd6ce08bd52b15d94bc35bb443874565edb3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:43:43 -0400 Subject: [PATCH 010/208] [ruby/yarp] Use .rb suffix to find version file https://github.com/ruby/yarp/commit/7c530c79c2 --- yarp/templates/template.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index 59a779be1f0495..944cb7edc547d4 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -4,7 +4,7 @@ require "fileutils" require "yaml" -if File.exist?(File.expand_path("../lib/yarp/version", __dir__)) +if File.exist?(File.expand_path("../lib/yarp/version.rb", __dir__)) # Within the gem/local repository require_relative "../lib/yarp/version" else From a7c7cd11bc5c33173f3eeab8600819f84fe2d662 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:47:25 -0400 Subject: [PATCH 011/208] Update sync script to handle yarp/version.h --- .gitignore | 1 + common.mk | 4 ++++ tool/update-deps | 1 + yarp/version.h | 4 ---- 4 files changed, 6 insertions(+), 4 deletions(-) delete mode 100644 yarp/version.h diff --git a/.gitignore b/.gitignore index 631a489eeff1f8..cb7c533bac1ac7 100644 --- a/.gitignore +++ b/.gitignore @@ -265,3 +265,4 @@ lcov*.info /yarp/prettyprint.c /yarp/serialize.c /yarp/token_type.c +/yarp/version.h diff --git a/common.mk b/common.mk index a05f0ddda95479..627a5b954ad859 100644 --- a/common.mk +++ b/common.mk @@ -225,6 +225,10 @@ srcs: yarp/ast.h yarp/ast.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/ast.h.erb $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/ast.h $@ +srcs: yarp/version.h +yarp/version.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/version.h.erb + $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/version.h $@ + srcs: yarp/node.c yarp/node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/node.c.erb $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/node.c $@ diff --git a/tool/update-deps b/tool/update-deps index 5662188deec402..90107e50c83758 100755 --- a/tool/update-deps +++ b/tool/update-deps @@ -156,6 +156,7 @@ FILES_NEED_VPATH = %w[ yarp/prettyprint.c yarp/serialize.c yarp/token_type.c + yarp/version.h ] # Multiple files with same filename. diff --git a/yarp/version.h b/yarp/version.h deleted file mode 100644 index 179543f54d751b..00000000000000 --- a/yarp/version.h +++ /dev/null @@ -1,4 +0,0 @@ -#define YP_VERSION_MAJOR 0 -#define YP_VERSION_MINOR 8 -#define YP_VERSION_PATCH 0 -#define YP_VERSION "0.8.0" From 934552618e8b8a71b3910fdb35ee65f239379ce3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:48:45 -0400 Subject: [PATCH 012/208] [ruby/yarp] Fix relative require for version in YARP gemspec https://github.com/ruby/yarp/commit/ca8e8cfa0d --- lib/yarp/yarp.gemspec | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index 671eddc7a20e52..b04f893d046414 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -1,6 +1,12 @@ # frozen_string_literal: true -require_relative 'lib/yarp/version' +if File.exist?(File.expand_path("version.rb", __dir__)) + # CRuby + require_relative "version" +else + # Within the gem/local repository + require_relative "lib/yarp/version" +end Gem::Specification.new do |spec| spec.name = "yarp" From 85c8e8b9bd1c50c6affff019b8749271a64a9950 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:53:01 -0400 Subject: [PATCH 013/208] Fix VPATH for yarp/version.h --- common.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common.mk b/common.mk index 627a5b954ad859..c193aaae493c1e 100644 --- a/common.mk +++ b/common.mk @@ -19771,7 +19771,7 @@ yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h -yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/version.h +yarp/yarp.$(OBJEXT): {$(VPATH)}/yarp/version.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.c yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/yarp.$(OBJEXT): {$(VPATH)}config.h From 74780c3e7f58e2c098f11f481c9ea0302a6a05de Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Aug 2023 17:56:08 -0400 Subject: [PATCH 014/208] [ruby/yarp] Call shorthand should not result in a message location https://github.com/ruby/yarp/commit/ad0f9d35e3 --- test/yarp/snapshots/method_calls.txt | 6 +++--- test/yarp/snapshots/seattlerb/call_colon_parens.txt | 2 +- test/yarp/snapshots/seattlerb/call_dot_parens.txt | 2 +- test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt | 2 +- test/yarp/snapshots/seattlerb/thingy.txt | 4 ++-- test/yarp/snapshots/whitequark/send_call.txt | 4 ++-- yarp/yarp.c | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/yarp/snapshots/method_calls.txt b/test/yarp/snapshots/method_calls.txt index b2c84d0d3573c2..ebda131febe278 100644 --- a/test/yarp/snapshots/method_calls.txt +++ b/test/yarp/snapshots/method_calls.txt @@ -64,7 +64,7 @@ ProgramNode(0...1237)( CallNode(58...62)( CallNode(58...59)(nil, nil, (58...59), nil, nil, nil, nil, 2, "a"), (59...60), - (0...0), + nil, (60...61), nil, (61...62), @@ -75,7 +75,7 @@ ProgramNode(0...1237)( CallNode(64...75)( CallNode(64...65)(nil, nil, (64...65), nil, nil, nil, nil, 2, "a"), (65...66), - (0...0), + nil, (66...67), ArgumentsNode(67...74)( [IntegerNode(67...68)(), @@ -413,7 +413,7 @@ ProgramNode(0...1237)( CallNode(212...217)( CallNode(212...213)(nil, nil, (212...213), nil, nil, nil, nil, 2, "a"), (213...215), - (0...0), + nil, (215...216), nil, (216...217), diff --git a/test/yarp/snapshots/seattlerb/call_colon_parens.txt b/test/yarp/snapshots/seattlerb/call_colon_parens.txt index 2e707afcfb6717..14c7247bcd47cb 100644 --- a/test/yarp/snapshots/seattlerb/call_colon_parens.txt +++ b/test/yarp/snapshots/seattlerb/call_colon_parens.txt @@ -4,7 +4,7 @@ ProgramNode(0...5)( [CallNode(0...5)( IntegerNode(0...1)(), (1...3), - (0...0), + nil, (3...4), nil, (4...5), diff --git a/test/yarp/snapshots/seattlerb/call_dot_parens.txt b/test/yarp/snapshots/seattlerb/call_dot_parens.txt index 5a32c3b9cf23a4..073336ed48525b 100644 --- a/test/yarp/snapshots/seattlerb/call_dot_parens.txt +++ b/test/yarp/snapshots/seattlerb/call_dot_parens.txt @@ -4,7 +4,7 @@ ProgramNode(0...4)( [CallNode(0...4)( IntegerNode(0...1)(), (1...2), - (0...0), + nil, (2...3), nil, (3...4), diff --git a/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt b/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt index 63a6f3822afae9..5800090fbbb9d5 100644 --- a/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt +++ b/test/yarp/snapshots/seattlerb/safe_call_dot_parens.txt @@ -4,7 +4,7 @@ ProgramNode(0...5)( [CallNode(0...5)( CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), (1...3), - (0...0), + nil, (3...4), nil, (4...5), diff --git a/test/yarp/snapshots/seattlerb/thingy.txt b/test/yarp/snapshots/seattlerb/thingy.txt index f12a16ed4fc808..929dd9fd96899e 100644 --- a/test/yarp/snapshots/seattlerb/thingy.txt +++ b/test/yarp/snapshots/seattlerb/thingy.txt @@ -4,7 +4,7 @@ ProgramNode(0...15)( [CallNode(0...6)( CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "f"), (1...2), - (0...0), + nil, (2...3), ArgumentsNode(3...5)([IntegerNode(3...5)()]), (5...6), @@ -15,7 +15,7 @@ ProgramNode(0...15)( CallNode(8...15)( CallNode(8...9)(nil, nil, (8...9), nil, nil, nil, nil, 2, "f"), (9...11), - (0...0), + nil, (11...12), ArgumentsNode(12...14)([IntegerNode(12...14)()]), (14...15), diff --git a/test/yarp/snapshots/whitequark/send_call.txt b/test/yarp/snapshots/whitequark/send_call.txt index 4280323078034f..f96195145baf54 100644 --- a/test/yarp/snapshots/whitequark/send_call.txt +++ b/test/yarp/snapshots/whitequark/send_call.txt @@ -4,7 +4,7 @@ ProgramNode(0...17)( [CallNode(0...7)( CallNode(0...3)(nil, nil, (0...3), nil, nil, nil, nil, 2, "foo"), (3...4), - (0...0), + nil, (4...5), ArgumentsNode(5...6)([IntegerNode(5...6)()]), (6...7), @@ -15,7 +15,7 @@ ProgramNode(0...17)( CallNode(9...17)( CallNode(9...12)(nil, nil, (9...12), nil, nil, nil, nil, 2, "foo"), (12...14), - (0...0), + nil, (14...15), ArgumentsNode(15...16)([IntegerNode(15...16)()]), (16...17), diff --git a/yarp/yarp.c b/yarp/yarp.c index ad762d89a32d59..29846998a44332 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -1197,7 +1197,7 @@ yp_call_node_create(yp_parser_t *parser) { }, .receiver = NULL, .operator_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, - .message_loc = YP_LOCATION_NULL_VALUE(parser), + .message_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .opening_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .arguments = NULL, .closing_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, From df11a08d933bc0efa754ccb944d1c8997c56ee7d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Aug 2023 18:07:32 -0400 Subject: [PATCH 015/208] [ruby/yarp] Add closing_loc to UntilNode https://github.com/ruby/yarp/commit/4362cecc2c --- test/yarp/snapshots/method_calls.txt | 1 + .../seattlerb/parse_until_not_canonical.txt | 1 + .../parse_until_not_noncanonical.txt | 1 + .../unparser/corpus/literal/send.txt | 1 + .../unparser/corpus/literal/while.txt | 12 ++++- .../unparser/corpus/semantic/while.txt | 4 ++ test/yarp/snapshots/until.txt | 6 +++ test/yarp/snapshots/whitequark/until.txt | 2 + test/yarp/snapshots/whitequark/until_mod.txt | 1 + test/yarp/snapshots/whitequark/until_post.txt | 1 + yarp/config.yml | 2 + yarp/yarp.c | 50 ++++++++++--------- 12 files changed, 58 insertions(+), 24 deletions(-) diff --git a/test/yarp/snapshots/method_calls.txt b/test/yarp/snapshots/method_calls.txt index ebda131febe278..a6e6709045dd45 100644 --- a/test/yarp/snapshots/method_calls.txt +++ b/test/yarp/snapshots/method_calls.txt @@ -1418,6 +1418,7 @@ ProgramNode(0...1237)( ), UntilNode(1121...1153)( (1121...1126), + (1150...1153), CallNode(1127...1128)( nil, nil, diff --git a/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt b/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt index acac8dd857298b..e84c4bf8a92476 100644 --- a/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_until_not_canonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [UntilNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt b/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt index acac8dd857298b..e84c4bf8a92476 100644 --- a/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_until_not_noncanonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [UntilNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index b448ea3e143934..54262865df905e 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -281,6 +281,7 @@ ProgramNode(0...999)( CallNode(255...272)( UntilNode(255...268)( (255...260), + (265...268), CallNode(261...264)( nil, nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/while.txt b/test/yarp/snapshots/unparser/corpus/literal/while.txt index dae7e0f0f31215..6807dec9e4c44c 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/while.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/while.txt @@ -165,6 +165,7 @@ ProgramNode(0...620)( StatementsNode(159...178)( [UntilNode(159...178)( (169...174), + nil, LocalVariableReadNode(175...178)(:foo, 0), StatementsNode(159...168)( [LocalVariableWriteNode(159...168)( @@ -460,6 +461,7 @@ ProgramNode(0...620)( ), UntilNode(429...460)( (451...456), + nil, CallNode(457...460)( nil, nil, @@ -585,15 +587,23 @@ ProgramNode(0...620)( ), 0 ), - UntilNode(557...572)((557...562), FalseNode(563...568)(), nil, 0), + UntilNode(557...572)( + (557...562), + (569...572), + FalseNode(563...568)(), + nil, + 0 + ), UntilNode(573...592)( (573...578), + (589...592), FalseNode(579...584)(), StatementsNode(587...588)([IntegerNode(587...588)()]), 0 ), UntilNode(593...620)( (593...598), + (617...620), ParenthesesNode(599...608)( StatementsNode(600...607)( [CallNode(600...607)( diff --git a/test/yarp/snapshots/unparser/corpus/semantic/while.txt b/test/yarp/snapshots/unparser/corpus/semantic/while.txt index 148db33da6523f..171447dc678f35 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/while.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/while.txt @@ -3,6 +3,7 @@ ProgramNode(0...188)( StatementsNode(0...188)( [UntilNode(0...13)( (2...7), + nil, CallNode(8...13)( nil, nil, @@ -21,6 +22,7 @@ ProgramNode(0...188)( ), UntilNode(15...34)( (15...20), + (31...34), CallNode(21...26)( nil, nil, @@ -63,6 +65,7 @@ ProgramNode(0...188)( ), UntilNode(57...75)( (59...64), + nil, AndNode(65...75)( CallNode(65...66)(nil, nil, (65...66), nil, nil, nil, nil, 2, "b"), CallNode(70...75)( @@ -97,6 +100,7 @@ ProgramNode(0...188)( ), UntilNode(98...130)( (100...105), + nil, CallNode(106...130)( nil, nil, diff --git a/test/yarp/snapshots/until.txt b/test/yarp/snapshots/until.txt index 3218ce2d423b1b..6a93708e5b6da2 100644 --- a/test/yarp/snapshots/until.txt +++ b/test/yarp/snapshots/until.txt @@ -3,36 +3,42 @@ ProgramNode(0...109)( StatementsNode(0...109)( [UntilNode(0...18)( (0...5), + (15...18), TrueNode(6...10)(), StatementsNode(12...13)([IntegerNode(12...13)()]), 0 ), UntilNode(20...32)( (22...27), + nil, TrueNode(28...32)(), StatementsNode(20...21)([IntegerNode(20...21)()]), 0 ), UntilNode(34...50)( (40...45), + nil, TrueNode(46...50)(), StatementsNode(34...39)([BreakNode(34...39)(nil, (34...39))]), 0 ), UntilNode(52...67)( (57...62), + nil, TrueNode(63...67)(), StatementsNode(52...56)([NextNode(52...56)(nil, (52...56))]), 0 ), UntilNode(69...86)( (76...81), + nil, TrueNode(82...86)(), StatementsNode(69...75)([ReturnNode(69...75)((69...75), nil)]), 0 ), UntilNode(88...109)( (99...104), + nil, CallNode(105...109)( nil, nil, diff --git a/test/yarp/snapshots/whitequark/until.txt b/test/yarp/snapshots/whitequark/until.txt index 2eae2d5307f9b7..5a40a615bd68ef 100644 --- a/test/yarp/snapshots/whitequark/until.txt +++ b/test/yarp/snapshots/whitequark/until.txt @@ -3,6 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [UntilNode(0...21)( (0...5), + (18...21), CallNode(6...9)(nil, nil, (6...9), nil, nil, nil, nil, 2, "foo"), StatementsNode(13...17)( [CallNode(13...17)( @@ -21,6 +22,7 @@ ProgramNode(0...42)( ), UntilNode(23...42)( (23...28), + (39...42), CallNode(29...32)(nil, nil, (29...32), nil, nil, nil, nil, 2, "foo"), StatementsNode(34...38)( [CallNode(34...38)( diff --git a/test/yarp/snapshots/whitequark/until_mod.txt b/test/yarp/snapshots/whitequark/until_mod.txt index 62f9d4992e8a93..ab5acaf4b37eb4 100644 --- a/test/yarp/snapshots/whitequark/until_mod.txt +++ b/test/yarp/snapshots/whitequark/until_mod.txt @@ -3,6 +3,7 @@ ProgramNode(0...14)( StatementsNode(0...14)( [UntilNode(0...14)( (5...10), + nil, CallNode(11...14)(nil, nil, (11...14), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...4)( [CallNode(0...4)(nil, nil, (0...4), nil, nil, nil, nil, 2, "meth")] diff --git a/test/yarp/snapshots/whitequark/until_post.txt b/test/yarp/snapshots/whitequark/until_post.txt index 27e0a60e407e5f..87a0fa3a12e0f0 100644 --- a/test/yarp/snapshots/whitequark/until_post.txt +++ b/test/yarp/snapshots/whitequark/until_post.txt @@ -3,6 +3,7 @@ ProgramNode(0...24)( StatementsNode(0...24)( [UntilNode(0...24)( (15...20), + nil, CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...14)( [BeginNode(0...14)( diff --git a/yarp/config.yml b/yarp/config.yml index d7269f336d3069..d9c0891c7ac214 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -2141,6 +2141,8 @@ nodes: child_nodes: - name: keyword_loc type: location + - name: closing_loc + type: location? - name: predicate type: node - name: statements diff --git a/yarp/yarp.c b/yarp/yarp.c index 29846998a44332..a5981c3d6ee112 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -4209,34 +4209,43 @@ yp_unless_node_end_keyword_loc_set(yp_unless_node_t *node, const yp_token_t *end // Allocate a new UntilNode node. static yp_until_node_t * -yp_until_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { +yp_until_node_create(yp_parser_t *parser, const yp_token_t *keyword, const yp_token_t *closing, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { yp_until_node_t *node = YP_ALLOC_NODE(parser, yp_until_node_t); - bool has_statements = (statements != NULL) && (statements->body.size != 0); - const char *start = NULL; - if (has_statements && (keyword->start > statements->base.location.start)) { - start = statements->base.location.start; - } else { - start = keyword->start; - } + *node = (yp_until_node_t) { + { + .type = YP_NODE_UNTIL_NODE, + .flags = flags, + .location = { + .start = keyword->start, + .end = closing->end, + }, + }, + .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .predicate = predicate, + .statements = statements + }; - const char *end = NULL; - if (has_statements && (predicate->location.end < statements->base.location.end)) { - end = statements->base.location.end; - } else { - end = predicate->location.end; - } + return node; +} + +// Allocate a new UntilNode node. +static yp_until_node_t * +yp_until_node_modifier_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { + yp_until_node_t *node = YP_ALLOC_NODE(parser, yp_until_node_t); *node = (yp_until_node_t) { { .type = YP_NODE_UNTIL_NODE, .flags = flags, .location = { - .start = start, - .end = end, + .start = statements->base.location.start, + .end = predicate->location.end, }, }, .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .predicate = predicate, .statements = statements }; @@ -11882,12 +11891,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { expect(parser, YP_TOKEN_KEYWORD_END, "Expected `end` to close `until` statement."); } - yp_until_node_t *until_node = yp_until_node_create(parser, &keyword, predicate, statements, 0); - if (parser->previous.type == YP_TOKEN_KEYWORD_END) { - until_node->base.location.end = parser->previous.end; - } - - return (yp_node_t *) until_node; + return (yp_node_t *) yp_until_node_create(parser, &keyword, &parser->previous, predicate, statements, 0); } case YP_TOKEN_KEYWORD_WHILE: { yp_do_loop_stack_push(parser, true); @@ -13185,7 +13189,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_statements_node_body_append(statements, node); yp_node_t *predicate = parse_expression(parser, binding_power, "Expected a predicate after 'until'"); - return (yp_node_t *) yp_until_node_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); + return (yp_node_t *) yp_until_node_modifier_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); } case YP_TOKEN_KEYWORD_WHILE_MODIFIER: { parser_lex(parser); From b112e89bb1de4019595647e62405e6b88902383e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Aug 2023 18:10:47 -0400 Subject: [PATCH 016/208] [ruby/yarp] Add closing_loc to WhileNode https://github.com/ruby/yarp/commit/b4132b876d --- test/yarp/snapshots/method_calls.txt | 1 + .../seattlerb/parse_while_not_canonical.txt | 1 + .../parse_while_not_noncanonical.txt | 1 + .../unparser/corpus/literal/send.txt | 1 + .../unparser/corpus/literal/while.txt | 19 ++++++- .../unparser/corpus/semantic/while.txt | 3 ++ test/yarp/snapshots/while.txt | 10 ++++ .../whitequark/bug_while_not_parens_do.txt | 1 + .../class_definition_in_while_cond.txt | 4 ++ .../if_while_after_class__since_32.txt | 2 + .../method_definition_in_while_cond.txt | 4 ++ test/yarp/snapshots/whitequark/while.txt | 2 + test/yarp/snapshots/whitequark/while_mod.txt | 1 + test/yarp/snapshots/whitequark/while_post.txt | 1 + yarp/config.yml | 2 + yarp/yarp.c | 49 ++++++++++--------- 16 files changed, 79 insertions(+), 23 deletions(-) diff --git a/test/yarp/snapshots/method_calls.txt b/test/yarp/snapshots/method_calls.txt index a6e6709045dd45..cfbb0784711b44 100644 --- a/test/yarp/snapshots/method_calls.txt +++ b/test/yarp/snapshots/method_calls.txt @@ -1369,6 +1369,7 @@ ProgramNode(0...1237)( [SymbolNode(1067...1069)((1067...1068), (1068...1069), nil, "a"), WhileNode(1073...1117)( (1073...1078), + (1114...1117), CallNode(1079...1080)( nil, nil, diff --git a/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt b/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt index b113e0ad537a68..8d152ba76ec6f6 100644 --- a/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_while_not_canonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [WhileNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt b/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt index b113e0ad537a68..8d152ba76ec6f6 100644 --- a/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt +++ b/test/yarp/snapshots/seattlerb/parse_while_not_noncanonical.txt @@ -3,6 +3,7 @@ ProgramNode(0...30)( StatementsNode(0...30)( [WhileNode(0...30)( (0...5), + (27...30), CallNode(6...18)( CallNode(10...18)( CallNode(10...13)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index 54262865df905e..ca65b7d58d4b12 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -308,6 +308,7 @@ ProgramNode(0...999)( CallNode(273...290)( WhileNode(273...286)( (273...278), + (283...286), CallNode(279...282)( nil, nil, diff --git a/test/yarp/snapshots/unparser/corpus/literal/while.txt b/test/yarp/snapshots/unparser/corpus/literal/while.txt index 6807dec9e4c44c..003c03d6147727 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/while.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/while.txt @@ -32,6 +32,7 @@ ProgramNode(0...620)( StatementsNode(27...60)( [WhileNode(27...60)( (27...32), + (57...60), CallNode(33...36)( nil, nil, @@ -72,6 +73,7 @@ ProgramNode(0...620)( StatementsNode(80...106)( [WhileNode(80...106)( (90...95), + nil, CallNode(96...106)( LocalVariableReadNode(96...99)(:foo, 0), nil, @@ -132,6 +134,7 @@ ProgramNode(0...620)( StatementsNode(123...142)( [WhileNode(123...142)( (133...138), + nil, LocalVariableReadNode(139...142)(:foo, 0), StatementsNode(123...132)( [LocalVariableWriteNode(123...132)( @@ -199,6 +202,7 @@ ProgramNode(0...620)( StatementsNode(195...224)( [WhileNode(195...224)( (195...200), + (221...224), CallNode(201...204)( nil, nil, @@ -266,6 +270,7 @@ ProgramNode(0...620)( StatementsNode(258...291)( [WhileNode(258...291)( (258...263), + (288...291), CallNode(264...267)( nil, nil, @@ -340,6 +345,7 @@ ProgramNode(0...620)( StatementsNode(329...362)( [WhileNode(329...362)( (329...334), + (359...362), LocalVariableReadNode(335...338)(:foo, 0), StatementsNode(345...354)( [LocalVariableWriteNode(345...354)( @@ -380,6 +386,7 @@ ProgramNode(0...620)( StatementsNode(376...401)( [WhileNode(376...401)( (392...397), + nil, CallNode(398...401)( nil, nil, @@ -424,6 +431,7 @@ ProgramNode(0...620)( ), WhileNode(403...428)( (419...424), + nil, CallNode(425...428)( nil, nil, @@ -510,6 +518,7 @@ ProgramNode(0...620)( ), WhileNode(461...492)( (483...488), + nil, CallNode(489...492)( nil, nil, @@ -556,15 +565,23 @@ ProgramNode(0...620)( ), 1 ), - WhileNode(493...508)((493...498), FalseNode(499...504)(), nil, 0), + WhileNode(493...508)( + (493...498), + (505...508), + FalseNode(499...504)(), + nil, + 0 + ), WhileNode(509...528)( (509...514), + (525...528), FalseNode(515...520)(), StatementsNode(523...524)([IntegerNode(523...524)()]), 0 ), WhileNode(529...556)( (529...534), + (553...556), ParenthesesNode(535...544)( StatementsNode(536...543)( [CallNode(536...543)( diff --git a/test/yarp/snapshots/unparser/corpus/semantic/while.txt b/test/yarp/snapshots/unparser/corpus/semantic/while.txt index 171447dc678f35..c381369c414e36 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/while.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/while.txt @@ -41,6 +41,7 @@ ProgramNode(0...188)( ), WhileNode(36...55)( (46...51), + nil, LocalVariableReadNode(52...55)(:foo, 0), StatementsNode(36...45)( [LocalVariableWriteNode(36...45)( @@ -88,6 +89,7 @@ ProgramNode(0...188)( ), WhileNode(77...96)( (77...82), + (93...96), LocalVariableWriteNode(83...88)( :a, 0, @@ -159,6 +161,7 @@ ProgramNode(0...188)( ), WhileNode(155...184)( (155...160), + (181...184), LocalVariableReadNode(161...164)(:foo, 0), StatementsNode(169...178)( [LocalVariableWriteNode(169...178)( diff --git a/test/yarp/snapshots/while.txt b/test/yarp/snapshots/while.txt index 484eb68b2c9c02..667dd30359bd3d 100644 --- a/test/yarp/snapshots/while.txt +++ b/test/yarp/snapshots/while.txt @@ -3,36 +3,42 @@ ProgramNode(0...314)( StatementsNode(0...314)( [WhileNode(0...18)( (0...5), + (15...18), TrueNode(6...10)(), StatementsNode(12...13)([IntegerNode(12...13)()]), 0 ), WhileNode(20...32)( (22...27), + nil, TrueNode(28...32)(), StatementsNode(20...21)([IntegerNode(20...21)()]), 0 ), WhileNode(34...50)( (40...45), + nil, TrueNode(46...50)(), StatementsNode(34...39)([BreakNode(34...39)(nil, (34...39))]), 0 ), WhileNode(52...67)( (57...62), + nil, TrueNode(63...67)(), StatementsNode(52...56)([NextNode(52...56)(nil, (52...56))]), 0 ), WhileNode(69...86)( (76...81), + nil, TrueNode(82...86)(), StatementsNode(69...75)([ReturnNode(69...75)((69...75), nil)]), 0 ), WhileNode(88...109)( (99...104), + nil, CallNode(105...109)( nil, nil, @@ -64,6 +70,7 @@ ProgramNode(0...314)( ), WhileNode(111...161)( (111...116), + (158...161), DefNode(117...149)( (126...129), SelfNode(121...125)(), @@ -105,6 +112,7 @@ ProgramNode(0...314)( ), WhileNode(163...210)( (163...168), + (207...210), ClassNode(169...198)( [:a], (169...174), @@ -138,6 +146,7 @@ ProgramNode(0...314)( ), WhileNode(212...260)( (212...217), + (257...260), SingletonClassNode(218...248)( [], (218...223), @@ -163,6 +172,7 @@ ProgramNode(0...314)( ), WhileNode(262...314)( (262...267), + (311...314), SingletonClassNode(268...302)( [:a], (268...273), diff --git a/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt b/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt index 0f517f935c115f..f5e086059a86c3 100644 --- a/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt +++ b/test/yarp/snapshots/whitequark/bug_while_not_parens_do.txt @@ -3,6 +3,7 @@ ProgramNode(0...23)( StatementsNode(0...23)( [WhileNode(0...23)( (0...5), + (20...23), CallNode(6...16)( ParenthesesNode(10...16)( StatementsNode(11...15)([TrueNode(11...15)()]), diff --git a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt index 0af9f40e8a7ec7..a3108d7060ca79 100644 --- a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt +++ b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt @@ -3,6 +3,7 @@ ProgramNode(0...197)( StatementsNode(0...197)( [WhileNode(0...52)( (0...5), + (49...52), SingletonClassNode(6...40)( [:a], (6...11), @@ -34,6 +35,7 @@ ProgramNode(0...197)( ), WhileNode(54...102)( (54...59), + (99...102), SingletonClassNode(60...90)( [], (60...65), @@ -59,6 +61,7 @@ ProgramNode(0...197)( ), WhileNode(104...151)( (104...109), + (148...151), ClassNode(110...139)( [:a], (110...115), @@ -92,6 +95,7 @@ ProgramNode(0...197)( ), WhileNode(153...197)( (153...158), + (194...197), ClassNode(159...185)( [], (159...164), diff --git a/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt b/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt index 2096097f5c6ee9..1ebb67a457477a 100644 --- a/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt +++ b/test/yarp/snapshots/whitequark/if_while_after_class__since_32.txt @@ -27,6 +27,7 @@ ProgramNode(0...178)( ConstantPathNode(46...82)( WhileNode(46...74)( (46...51), + (71...74), TrueNode(52...56)(), StatementsNode(58...70)( [BreakNode(58...70)( @@ -69,6 +70,7 @@ ProgramNode(0...178)( ConstantPathNode(137...173)( WhileNode(137...165)( (137...142), + (162...165), TrueNode(143...147)(), StatementsNode(149...161)( [BreakNode(149...161)( diff --git a/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt b/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt index 3b67815c7b8647..3179d39b959edc 100644 --- a/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt +++ b/test/yarp/snapshots/whitequark/method_definition_in_while_cond.txt @@ -3,6 +3,7 @@ ProgramNode(0...190)( StatementsNode(0...190)( [WhileNode(0...45)( (0...5), + (42...45), DefNode(6...33)( (10...13), nil, @@ -44,6 +45,7 @@ ProgramNode(0...190)( ), WhileNode(47...89)( (47...52), + (86...89), DefNode(53...77)( (57...60), nil, @@ -74,6 +76,7 @@ ProgramNode(0...190)( ), WhileNode(91...141)( (91...96), + (138...141), DefNode(97...129)( (106...109), SelfNode(101...105)(), @@ -115,6 +118,7 @@ ProgramNode(0...190)( ), WhileNode(143...190)( (143...148), + (187...190), DefNode(149...178)( (158...161), SelfNode(153...157)(), diff --git a/test/yarp/snapshots/whitequark/while.txt b/test/yarp/snapshots/whitequark/while.txt index 7d95f466c8c0d5..7529b6e1bc1ec6 100644 --- a/test/yarp/snapshots/whitequark/while.txt +++ b/test/yarp/snapshots/whitequark/while.txt @@ -3,6 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [WhileNode(0...21)( (0...5), + (18...21), CallNode(6...9)(nil, nil, (6...9), nil, nil, nil, nil, 2, "foo"), StatementsNode(13...17)( [CallNode(13...17)( @@ -21,6 +22,7 @@ ProgramNode(0...42)( ), WhileNode(23...42)( (23...28), + (39...42), CallNode(29...32)(nil, nil, (29...32), nil, nil, nil, nil, 2, "foo"), StatementsNode(34...38)( [CallNode(34...38)( diff --git a/test/yarp/snapshots/whitequark/while_mod.txt b/test/yarp/snapshots/whitequark/while_mod.txt index b1a0203da3d19f..2f0ade7006b5fc 100644 --- a/test/yarp/snapshots/whitequark/while_mod.txt +++ b/test/yarp/snapshots/whitequark/while_mod.txt @@ -3,6 +3,7 @@ ProgramNode(0...14)( StatementsNode(0...14)( [WhileNode(0...14)( (5...10), + nil, CallNode(11...14)(nil, nil, (11...14), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...4)( [CallNode(0...4)(nil, nil, (0...4), nil, nil, nil, nil, 2, "meth")] diff --git a/test/yarp/snapshots/whitequark/while_post.txt b/test/yarp/snapshots/whitequark/while_post.txt index 28a423e1562b6e..1b2169107a160f 100644 --- a/test/yarp/snapshots/whitequark/while_post.txt +++ b/test/yarp/snapshots/whitequark/while_post.txt @@ -3,6 +3,7 @@ ProgramNode(0...24)( StatementsNode(0...24)( [WhileNode(0...24)( (15...20), + nil, CallNode(21...24)(nil, nil, (21...24), nil, nil, nil, nil, 2, "foo"), StatementsNode(0...14)( [BeginNode(0...14)( diff --git a/yarp/config.yml b/yarp/config.yml index d9c0891c7ac214..4cb060d80e080b 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -2180,6 +2180,8 @@ nodes: child_nodes: - name: keyword_loc type: location + - name: closing_loc + type: location? - name: predicate type: node - name: statements diff --git a/yarp/yarp.c b/yarp/yarp.c index a5981c3d6ee112..5299ae940107a4 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -4293,34 +4293,43 @@ yp_when_node_statements_set(yp_when_node_t *node, yp_statements_node_t *statemen // Allocate a new WhileNode node. static yp_while_node_t * -yp_while_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { +yp_while_node_create(yp_parser_t *parser, const yp_token_t *keyword, const yp_token_t *closing, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { yp_while_node_t *node = YP_ALLOC_NODE(parser, yp_while_node_t); - const char *start = NULL; - bool has_statements = (statements != NULL) && (statements->body.size != 0); - if (has_statements && (keyword->start > statements->base.location.start)) { - start = statements->base.location.start; - } else { - start = keyword->start; - } + *node = (yp_while_node_t) { + { + .type = YP_NODE_WHILE_NODE, + .flags = flags, + .location = { + .start = keyword->start, + .end = closing->end + }, + }, + .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), + .predicate = predicate, + .statements = statements + }; - const char *end = NULL; - if (has_statements && (predicate->location.end < statements->base.location.end)) { - end = statements->base.location.end; - } else { - end = predicate->location.end; - } + return node; +} + +// Allocate a new WhileNode node. +static yp_while_node_t * +yp_while_node_modifier_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t *predicate, yp_statements_node_t *statements, yp_node_flags_t flags) { + yp_while_node_t *node = YP_ALLOC_NODE(parser, yp_while_node_t); *node = (yp_while_node_t) { { .type = YP_NODE_WHILE_NODE, .flags = flags, .location = { - .start = start, - .end = end, + .start = statements->base.location.start, + .end = predicate->location.end }, }, .keyword_loc = YP_LOCATION_TOKEN_VALUE(keyword), + .closing_loc = YP_OPTIONAL_LOCATION_NOT_PROVIDED_VALUE, .predicate = predicate, .statements = statements }; @@ -11912,11 +11921,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { expect(parser, YP_TOKEN_KEYWORD_END, "Expected `end` to close `while` statement."); } - yp_while_node_t *while_node = yp_while_node_create(parser, &keyword, predicate, statements, 0); - if (parser->previous.type == YP_TOKEN_KEYWORD_END) { - while_node->base.location.end = parser->previous.end; - } - return (yp_node_t *) while_node; + return (yp_node_t *) yp_while_node_create(parser, &keyword, &parser->previous, predicate, statements, 0); } case YP_TOKEN_PERCENT_LOWER_I: { parser_lex(parser); @@ -13197,7 +13202,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_statements_node_body_append(statements, node); yp_node_t *predicate = parse_expression(parser, binding_power, "Expected a predicate after 'while'"); - return (yp_node_t *) yp_while_node_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); + return (yp_node_t *) yp_while_node_modifier_create(parser, &token, predicate, statements, YP_NODE_TYPE_P(node, YP_NODE_BEGIN_NODE) ? YP_LOOP_FLAGS_BEGIN_MODIFIER : 0); } case YP_TOKEN_QUESTION_MARK: { parser_lex(parser); From b9a2c96747cfac2bcc2883335b40f2a2d61d34cb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Aug 2023 21:00:00 -0400 Subject: [PATCH 017/208] [ruby/yarp] Ensure interpolated symbols converted to regular symbols get opening and closing https://github.com/ruby/yarp/commit/386655d54f --- test/yarp/snapshots/patterns.txt | 27 +++++-- .../snapshots/seattlerb/dsym_esc_to_sym.txt | 4 +- test/yarp/snapshots/seattlerb/dsym_to_sym.txt | 4 +- .../unparser/corpus/literal/literal.txt | 17 +++-- .../snapshots/whitequark/interp_digit_var.txt | 4 +- ...ser_slash_slash_n_escaping_in_literals.txt | 2 +- yarp/config.yml | 2 +- yarp/yarp.c | 75 +++++++++---------- 8 files changed, 77 insertions(+), 58 deletions(-) diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index a55f1ba2924a7e..d5d37f8665202b 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -38,7 +38,7 @@ ProgramNode(0...3743)( ), MatchRequiredNode(78...91)( CallNode(78...81)(nil, nil, (78...81), nil, nil, nil, nil, 2, "foo"), - SymbolNode(85...91)(nil, (87...90), nil, "foo"), + SymbolNode(85...91)((85...87), (87...90), (90...91), "foo"), (82...84) ), MatchRequiredNode(92...104)( @@ -461,8 +461,8 @@ ProgramNode(0...3743)( "foo" ), RangeNode(472...488)( - SymbolNode(472...478)(nil, (474...477), nil, "foo"), - SymbolNode(482...488)(nil, (484...487), nil, "foo"), + SymbolNode(472...478)((472...474), (474...477), (477...478), "foo"), + SymbolNode(482...488)((482...484), (484...487), (487...488), "foo"), (479...481), 0 ), @@ -1943,7 +1943,12 @@ ProgramNode(0...3743)( 2, "foo" ), - SymbolNode(1733...1739)(nil, (1735...1738), nil, "foo"), + SymbolNode(1733...1739)( + (1733...1735), + (1735...1738), + (1738...1739), + "foo" + ), (1730...1732) ), MatchPredicateNode(1740...1752)( @@ -2442,7 +2447,12 @@ ProgramNode(0...3743)( "foo" ), [InNode(2196...2210)( - SymbolNode(2199...2205)(nil, (2201...2204), nil, "foo"), + SymbolNode(2199...2205)( + (2199...2201), + (2201...2204), + (2204...2205), + "foo" + ), nil, (2196...2198), (2206...2210) @@ -3134,7 +3144,12 @@ ProgramNode(0...3743)( (3000...3002), LocalVariableReadNode(3003...3006)(:baz, 0), StatementsNode(2993...2999)( - [SymbolNode(2993...2999)(nil, (2995...2998), nil, "foo")] + [SymbolNode(2993...2999)( + (2993...2995), + (2995...2998), + (2998...2999), + "foo" + )] ), nil, nil diff --git a/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt b/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt index 16cd806d018f85..4f6697237b1a70 100644 --- a/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt +++ b/test/yarp/snapshots/seattlerb/dsym_esc_to_sym.txt @@ -1,4 +1,6 @@ ProgramNode(0...17)( [], - StatementsNode(0...17)([SymbolNode(0...17)(nil, (2...16), nil, "Varietà")]) + StatementsNode(0...17)( + [SymbolNode(0...17)((0...2), (2...16), (16...17), "Varietà")] + ) ) diff --git a/test/yarp/snapshots/seattlerb/dsym_to_sym.txt b/test/yarp/snapshots/seattlerb/dsym_to_sym.txt index f2e3452dc12e9b..d383b67184c1dc 100644 --- a/test/yarp/snapshots/seattlerb/dsym_to_sym.txt +++ b/test/yarp/snapshots/seattlerb/dsym_to_sym.txt @@ -2,8 +2,8 @@ ProgramNode(0...32)( [], StatementsNode(0...32)( [AliasNode(0...17)( - SymbolNode(6...11)(nil, (8...10), nil, "<<"), - SymbolNode(12...17)(nil, (14...16), nil, ">>"), + SymbolNode(6...11)((6...8), (8...10), (10...11), "<<"), + SymbolNode(12...17)((12...14), (14...16), (16...17), ">>"), (0...5) ), AliasNode(19...32)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/literal.txt b/test/yarp/snapshots/unparser/corpus/literal/literal.txt index 9b0fad50720ceb..bc9f86842dce37 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/literal.txt @@ -276,11 +276,11 @@ ProgramNode(0...916)( XStringNode(435...439)((435...436), (436...438), (438...439), "`"), XStringNode(440...443)((440...441), (441...442), (442...443), "\""), SymbolNode(444...448)((444...445), (445...448), nil, "foo"), - SymbolNode(449...455)(nil, (451...454), nil, "A B"), + SymbolNode(449...455)((449...451), (451...454), (454...455), "A B"), SymbolNode(456...460)((456...457), (457...460), nil, "foo"), - SymbolNode(461...467)(nil, (463...466), nil, "A B"), - SymbolNode(468...475)(nil, (470...474), nil, "A\"B"), - InterpolatedSymbolNode(476...479)((476...478), [], (478...479)), + SymbolNode(461...467)((461...463), (463...466), (466...467), "A B"), + SymbolNode(468...475)((468...470), (470...474), (474...475), "A\"B"), + SymbolNode(476...479)((476...478), (0...0), (478...479), ""), RegularExpressionNode(480...485)( (480...481), (481...484), @@ -620,7 +620,7 @@ ProgramNode(0...916)( HashNode(828...843)( (828...829), [AssocNode(830...841)( - SymbolNode(830...836)(nil, (832...835), nil, "a b"), + SymbolNode(830...836)((830...832), (832...835), (835...836), "a b"), IntegerNode(840...841)(), (837...839) )], @@ -677,7 +677,12 @@ ProgramNode(0...916)( 0, "foo" ), - SymbolNode(893...901)(nil, (895...900), nil, "a\\\n" + "b"), + SymbolNode(893...901)( + (893...895), + (895...900), + (900...901), + "a\\\n" + "b" + ), InterpolatedXStringNode(902...916)( (902...903), [StringNode(903...907)(nil, (903...907), nil, " x\n"), diff --git a/test/yarp/snapshots/whitequark/interp_digit_var.txt b/test/yarp/snapshots/whitequark/interp_digit_var.txt index 4af55a97a8bb00..48d7ac96716907 100644 --- a/test/yarp/snapshots/whitequark/interp_digit_var.txt +++ b/test/yarp/snapshots/whitequark/interp_digit_var.txt @@ -83,8 +83,8 @@ ProgramNode(1...465)( "\#@@1", 0 ), - SymbolNode(294...300)(nil, (296...299), nil, "\#@1"), - SymbolNode(304...311)(nil, (306...310), nil, "\#@@1"), + SymbolNode(294...300)((294...296), (296...299), (299...300), "\#@1"), + SymbolNode(304...311)((304...306), (306...310), (310...311), "\#@@1"), SymbolNode(315...321)((315...317), (317...320), (320...321), "\#@1"), SymbolNode(325...332)((325...327), (327...331), (331...332), "\#@@1"), XStringNode(336...341)((336...337), (337...340), (340...341), "\#@1"), diff --git a/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt b/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt index 16db486beb243e..3a3377f96d6aa2 100644 --- a/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt +++ b/test/yarp/snapshots/whitequark/parser_slash_slash_n_escaping_in_literals.txt @@ -41,7 +41,7 @@ ProgramNode(0...210)( "ab", 0 ), - SymbolNode(123...130)(nil, (125...129), nil, "ab"), + SymbolNode(123...130)((123...125), (125...129), (129...130), "ab"), SymbolNode(132...139)((132...134), (134...138), (138...139), "ab"), InterpolatedStringNode(141...150)( (141...150), diff --git a/yarp/config.yml b/yarp/config.yml index 4cb060d80e080b..62a3c2249c2e98 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -2084,7 +2084,7 @@ nodes: - name: opening_loc type: location? - name: value_loc - type: location + type: location? - name: closing_loc type: location? - name: unescaped diff --git a/yarp/yarp.c b/yarp/yarp.c index 5299ae940107a4..46a57fe4e25abf 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2824,12 +2824,6 @@ yp_interpolated_symbol_node_append(yp_interpolated_symbol_node_t *node, yp_node_ node->base.location.end = part->location.end; } -static inline void -yp_interpolated_symbol_node_closing_set(yp_interpolated_symbol_node_t *node, const yp_token_t *closing) { - node->closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing); - node->base.location.end = closing->end; -} - // Allocate a new InterpolatedXStringNode node. static yp_interpolated_x_string_node_t * yp_interpolated_xstring_node_create(yp_parser_t *parser, const yp_token_t *opening, const yp_token_t *closing) { @@ -4057,20 +4051,20 @@ yp_symbol_node_label_p(yp_node_t *node) { // Convert the given StringNode node to a SymbolNode node. static yp_symbol_node_t * -yp_string_node_to_symbol_node(yp_parser_t *parser, yp_string_node_t *node) { +yp_string_node_to_symbol_node(yp_parser_t *parser, yp_string_node_t *node, const yp_token_t *opening, const yp_token_t *closing) { yp_symbol_node_t *new_node = YP_ALLOC_NODE(parser, yp_symbol_node_t); *new_node = (yp_symbol_node_t) { { .type = YP_NODE_SYMBOL_NODE, .location = { - .start = node->base.location.start - 2, - .end = node->base.location.end + 1 + .start = opening->start, + .end = closing->end } }, - .opening_loc = node->opening_loc, + .opening_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(opening), .value_loc = node->content_loc, - .closing_loc = node->closing_loc, + .closing_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(closing), .unescaped = node->unescaped }; @@ -9576,14 +9570,10 @@ parse_string_part(yp_parser_t *parser) { static yp_node_t * parse_symbol(yp_parser_t *parser, yp_lex_mode_t *lex_mode, yp_lex_state_t next_state) { - bool lex_string = lex_mode->mode == YP_LEX_STRING; - bool can_be_interpolated = lex_string && lex_mode->as.string.interpolation; yp_token_t opening = parser->previous; - if (!lex_string) { - if (next_state != YP_LEX_STATE_NONE) { - lex_state_set(parser, next_state); - } + if (lex_mode->mode != YP_LEX_STRING) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); yp_token_t symbol; switch (parser->current.type) { @@ -9613,37 +9603,44 @@ parse_symbol(yp_parser_t *parser, yp_lex_mode_t *lex_mode, yp_lex_state_t next_s return (yp_node_t *) yp_symbol_node_create_and_unescape(parser, &opening, &symbol, &closing, YP_UNESCAPE_ALL); } - if (can_be_interpolated) { - // Create a node_list first. We'll use this to check if it should be an InterpolatedSymbolNode - // or a SymbolNode + if (lex_mode->as.string.interpolation) { + // If we have the end of the symbol, then we can return an empty symbol. + if (match_type_p(parser, YP_TOKEN_STRING_END)) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); + parser_lex(parser); + + yp_token_t content = not_provided(parser); + yp_token_t closing = parser->previous; + return (yp_node_t *) yp_symbol_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_NONE); + } + + // Now we can parse the first part of the symbol. + yp_node_t *part = parse_string_part(parser); + + // If we got a string part, then it's possible that we could transform + // what looks like an interpolated symbol into a regular symbol. + if (part && YP_NODE_TYPE_P(part, YP_NODE_STRING_NODE) && match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); + parser_lex(parser); + + return (yp_node_t *) yp_string_node_to_symbol_node(parser, (yp_string_node_t *) part, &opening, &parser->previous); + } + + // Create a node_list first. We'll use this to check if it should be an + // InterpolatedSymbolNode or a SymbolNode. yp_node_list_t node_list = YP_EMPTY_NODE_LIST; + if (part) yp_node_list_append(&node_list, part); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - yp_node_t *part = parse_string_part(parser); - if (part != NULL) { + if ((part = parse_string_part(parser)) != NULL) { yp_node_list_append(&node_list, part); } } - yp_node_t *res; - // If the only element on the node_list is a StringNode, we know this is a SymbolNode - // and not an InterpolatedSymbolNode - if (node_list.size == 1 && YP_NODE_TYPE_P(node_list.nodes[0], YP_NODE_STRING_NODE)) { - res = (yp_node_t *)yp_string_node_to_symbol_node(parser, (yp_string_node_t *)node_list.nodes[0]); - free(node_list.nodes); - } - else { - yp_interpolated_symbol_node_t *interpolated = yp_interpolated_symbol_node_create(parser, &opening, &node_list, &opening); - yp_interpolated_symbol_node_closing_set(interpolated, &parser->current); - res = (yp_node_t *) interpolated; - } - - if (next_state != YP_LEX_STATE_NONE) { - lex_state_set(parser, next_state); - } + if (next_state != YP_LEX_STATE_NONE) lex_state_set(parser, next_state); expect(parser, YP_TOKEN_STRING_END, "Expected a closing delimiter for an interpolated symbol."); - return res; + return (yp_node_t *) yp_interpolated_symbol_node_create(parser, &opening, &node_list, &parser->previous); } yp_token_t content; From a31b069a8a0194ef589f0c81ff5da1b11374d7fd Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 21 Aug 2023 21:12:52 -0400 Subject: [PATCH 018/208] [ruby/yarp] Track block opening and closing locations https://github.com/ruby/yarp/commit/7984e4ddc7 --- test/yarp/errors_test.rb | 4 +++ test/yarp/snapshots/lambda.txt | 6 +++++ test/yarp/snapshots/patterns.txt | 12 +++++++++ test/yarp/snapshots/procs.txt | 22 ++++++++++++++++ .../call_array_lambda_block_call.txt | 2 ++ .../call_stabby_do_end_with_block.txt | 2 ++ .../call_stabby_with_braces_block.txt | 2 ++ test/yarp/snapshots/seattlerb/case_in.txt | 2 ++ .../yarp/snapshots/seattlerb/difficult3_5.txt | 2 ++ test/yarp/snapshots/seattlerb/difficult6_.txt | 2 ++ test/yarp/snapshots/seattlerb/do_lambda.txt | 2 ++ .../seattlerb/lambda_do_vs_brace.txt | 12 +++++++-- .../seattlerb/stabby_arg_no_paren.txt | 2 ++ .../stabby_arg_opt_splat_arg_block_omfg.txt | 2 ++ .../seattlerb/stabby_block_iter_call.txt | 2 ++ ...bby_block_iter_call_no_target_with_arg.txt | 2 ++ .../snapshots/seattlerb/stabby_block_kw.txt | 2 ++ .../seattlerb/stabby_block_kw__required.txt | 2 ++ .../snapshots/seattlerb/stabby_proc_scope.txt | 2 ++ .../unparser/corpus/literal/lambda.txt | 8 ++++++ .../unparser/corpus/literal/since/27.txt | 2 ++ test/yarp/snapshots/whitequark/bug_435.txt | 2 ++ test/yarp/snapshots/whitequark/bug_cmdarg.txt | 2 ++ .../whitequark/bug_lambda_leakage.txt | 2 ++ test/yarp/snapshots/whitequark/kwnilarg.txt | 2 ++ .../whitequark/numbered_args_after_27.txt | 4 +++ .../snapshots/whitequark/parser_bug_507.txt | 2 ++ .../snapshots/whitequark/parser_bug_645.txt | 2 ++ .../whitequark/rescue_in_lambda_block.txt | 2 ++ .../snapshots/whitequark/ruby_bug_11107.txt | 2 ++ .../snapshots/whitequark/ruby_bug_11380.txt | 2 ++ .../snapshots/whitequark/ruby_bug_15789.txt | 8 ++++++ .../yarp/snapshots/whitequark/send_lambda.txt | 6 +++-- .../snapshots/whitequark/send_lambda_args.txt | 4 +++ .../whitequark/send_lambda_args_noparen.txt | 4 +++ .../whitequark/send_lambda_args_shadow.txt | 2 ++ .../whitequark/send_lambda_legacy.txt | 4 ++- yarp/config.yml | 4 +++ yarp/yarp.c | 25 ++++++++++++------- 39 files changed, 159 insertions(+), 14 deletions(-) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index d58fd27448a1ad..7b20ceadabe0af 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -581,6 +581,8 @@ def test_do_not_allow_trailing_commas_in_lambda_parameters expected = LambdaNode( [:a, :b], Location(), + Location(), + Location(), BlockParametersNode( ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), [], @@ -924,6 +926,8 @@ def test_do_not_allow_forward_arguments_in_lambda_literals expected = LambdaNode( [:"..."], Location(), + Location(), + Location(), BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), nil ) diff --git a/test/yarp/snapshots/lambda.txt b/test/yarp/snapshots/lambda.txt index 23c4ebc072e31b..0832abf4e8077d 100644 --- a/test/yarp/snapshots/lambda.txt +++ b/test/yarp/snapshots/lambda.txt @@ -4,6 +4,8 @@ ProgramNode(0...92)( [LambdaNode(0...14)( [:foo], (0...2), + (12...13), + (13...14), BlockParametersNode(2...11)( ParametersNode(6...9)( [RequiredParameterNode(6...9)(:foo)], @@ -23,6 +25,8 @@ ProgramNode(0...92)( LambdaNode(16...34)( [:x], (16...18), + (31...32), + (33...34), BlockParametersNode(18...30)( ParametersNode(19...29)( [], @@ -66,6 +70,8 @@ ProgramNode(0...92)( LambdaNode(36...51)( [:a], (36...38), + (49...50), + (50...51), BlockParametersNode(38...48)( ParametersNode(39...47)( [], diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index d5d37f8665202b..de5946a7cf0da2 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -323,6 +323,8 @@ ProgramNode(0...3743)( LambdaNode(343...353)( [], (343...345), + (346...347), + (352...353), nil, StatementsNode(348...351)([LocalVariableReadNode(348...351)(:bar, 1)]) ), @@ -868,6 +870,8 @@ ProgramNode(0...3743)( LambdaNode(916...926)( [], (916...918), + (919...920), + (925...926), nil, StatementsNode(921...924)( [LocalVariableReadNode(921...924)(:bar, 1)] @@ -876,6 +880,8 @@ ProgramNode(0...3743)( LambdaNode(930...940)( [], (930...932), + (933...934), + (939...940), nil, StatementsNode(935...938)( [LocalVariableReadNode(935...938)(:bar, 1)] @@ -2268,6 +2274,8 @@ ProgramNode(0...3743)( LambdaNode(1991...2001)( [], (1991...1993), + (1994...1995), + (2000...2001), nil, StatementsNode(1996...1999)( [LocalVariableReadNode(1996...1999)(:bar, 1)] @@ -2898,6 +2906,8 @@ ProgramNode(0...3743)( LambdaNode(2727...2737)( [], (2727...2729), + (2730...2731), + (2736...2737), nil, StatementsNode(2732...2735)( [LocalVariableReadNode(2732...2735)(:bar, 1)] @@ -3727,6 +3737,8 @@ ProgramNode(0...3743)( [LambdaNode(3647...3657)( [], (3647...3649), + (3650...3651), + (3656...3657), nil, StatementsNode(3652...3655)( [LocalVariableReadNode(3652...3655)(:bar, 1)] diff --git a/test/yarp/snapshots/procs.txt b/test/yarp/snapshots/procs.txt index 3db8ec5b1378d9..adfb0637079532 100644 --- a/test/yarp/snapshots/procs.txt +++ b/test/yarp/snapshots/procs.txt @@ -4,6 +4,8 @@ ProgramNode(0...266)( [LambdaNode(0...21)( [:a, :b, :c, :d], (0...2), + (16...17), + (20...21), BlockParametersNode(3...15)( ParametersNode(4...5)( [RequiredParameterNode(4...5)(:a)], @@ -23,6 +25,8 @@ ProgramNode(0...266)( LambdaNode(23...39)( [], (23...25), + (26...28), + (36...39), nil, BeginNode(29...39)( nil, @@ -36,6 +40,8 @@ ProgramNode(0...266)( LambdaNode(41...69)( [], (41...43), + (44...46), + (66...69), nil, BeginNode(47...69)( nil, @@ -49,6 +55,8 @@ ProgramNode(0...266)( LambdaNode(71...81)( [], (71...73), + (74...75), + (80...81), nil, StatementsNode(76...79)( [CallNode(76...79)(nil, nil, (76...79), nil, nil, nil, nil, 2, "foo")] @@ -57,6 +65,8 @@ ProgramNode(0...266)( LambdaNode(83...98)( [], (83...85), + (86...88), + (95...98), nil, StatementsNode(90...93)( [CallNode(90...93)(nil, nil, (90...93), nil, nil, nil, nil, 2, "foo")] @@ -65,6 +75,8 @@ ProgramNode(0...266)( LambdaNode(100...129)( [:a, :b, :c, :d, :e], (100...102), + (124...125), + (128...129), BlockParametersNode(103...123)( ParametersNode(103...123)( [RequiredParameterNode(103...104)(:a)], @@ -90,6 +102,8 @@ ProgramNode(0...266)( LambdaNode(131...171)( [:a, :b, :c, :d, :e, :f, :g], (131...133), + (166...167), + (170...171), BlockParametersNode(134...165)( ParametersNode(135...164)( [RequiredParameterNode(135...136)(:a)], @@ -115,6 +129,8 @@ ProgramNode(0...266)( LambdaNode(173...218)( [:a, :b, :c, :d, :e, :f, :g], (173...175), + (208...210), + (215...218), BlockParametersNode(176...207)( ParametersNode(177...206)( [RequiredParameterNode(177...178)(:a)], @@ -140,6 +156,8 @@ ProgramNode(0...266)( LambdaNode(220...245)( [:a], (220...222), + (227...228), + (244...245), BlockParametersNode(223...226)( ParametersNode(224...225)( [RequiredParameterNode(224...225)(:a)], @@ -158,6 +176,8 @@ ProgramNode(0...266)( [LambdaNode(229...243)( [:b], (229...231), + (234...235), + (242...243), BlockParametersNode(232...233)( ParametersNode(232...233)( [RequiredParameterNode(232...233)(:b)], @@ -193,6 +213,8 @@ ProgramNode(0...266)( LambdaNode(247...266)( [:a, :b, :c], (247...249), + (263...264), + (265...266), BlockParametersNode(250...262)( ParametersNode(251...261)( [RequiredDestructuredParameterNode(251...257)( diff --git a/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt b/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt index 1d8c2f1472c4f1..c83adb235f72be 100644 --- a/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt +++ b/test/yarp/snapshots/seattlerb/call_array_lambda_block_call.txt @@ -11,6 +11,8 @@ ProgramNode(0...18)( [LambdaNode(3...10)( [], (3...5), + (8...9), + (9...10), BlockParametersNode(5...7)(nil, [], (5...6), (6...7)), nil )], diff --git a/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt b/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt index e48578da2aad53..f541c529851ce2 100644 --- a/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt +++ b/test/yarp/snapshots/seattlerb/call_stabby_do_end_with_block.txt @@ -10,6 +10,8 @@ ProgramNode(0...22)( [LambdaNode(2...13)( [], (2...4), + (5...7), + (10...13), nil, StatementsNode(8...9)([IntegerNode(8...9)()]) )] diff --git a/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt b/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt index 36fbd7fb1a8778..83c62571fd7c44 100644 --- a/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt +++ b/test/yarp/snapshots/seattlerb/call_stabby_with_braces_block.txt @@ -10,6 +10,8 @@ ProgramNode(0...19)( [LambdaNode(2...10)( [], (2...4), + (5...6), + (9...10), nil, StatementsNode(7...8)([IntegerNode(7...8)()]) )] diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index b83e196eceb2a8..8a5fafdf74b6a8 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -313,6 +313,8 @@ ProgramNode(0...747)( [LambdaNode(446...460)( [:b], (446...448), + (452...453), + (459...460), BlockParametersNode(448...451)( ParametersNode(449...450)( [RequiredParameterNode(449...450)(:b)], diff --git a/test/yarp/snapshots/seattlerb/difficult3_5.txt b/test/yarp/snapshots/seattlerb/difficult3_5.txt index e7aaafc87a1261..746786b8a5fb10 100644 --- a/test/yarp/snapshots/seattlerb/difficult3_5.txt +++ b/test/yarp/snapshots/seattlerb/difficult3_5.txt @@ -10,6 +10,8 @@ ProgramNode(0...19)( [LambdaNode(2...19)( [], (2...4), + (7...8), + (18...19), BlockParametersNode(4...6)(nil, [], (4...5), (5...6)), StatementsNode(9...17)( [CallNode(9...17)( diff --git a/test/yarp/snapshots/seattlerb/difficult6_.txt b/test/yarp/snapshots/seattlerb/difficult6_.txt index 20e419bbc45455..2d1677e108f7e9 100644 --- a/test/yarp/snapshots/seattlerb/difficult6_.txt +++ b/test/yarp/snapshots/seattlerb/difficult6_.txt @@ -4,6 +4,8 @@ ProgramNode(0...25)( [LambdaNode(0...25)( [:a, :b], (0...2), + (13...14), + (24...25), BlockParametersNode(2...12)( ParametersNode(3...11)( [RequiredParameterNode(3...4)(:a)], diff --git a/test/yarp/snapshots/seattlerb/do_lambda.txt b/test/yarp/snapshots/seattlerb/do_lambda.txt index 86b3548eb4b278..a7eb211623e8fe 100644 --- a/test/yarp/snapshots/seattlerb/do_lambda.txt +++ b/test/yarp/snapshots/seattlerb/do_lambda.txt @@ -4,6 +4,8 @@ ProgramNode(0...11)( [LambdaNode(0...11)( [], (0...2), + (5...7), + (8...11), BlockParametersNode(2...4)(nil, [], (2...3), (3...4)), nil )] diff --git a/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt b/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt index 1c9e2efc33940f..c7f83a9b512023 100644 --- a/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt +++ b/test/yarp/snapshots/seattlerb/lambda_do_vs_brace.txt @@ -6,7 +6,9 @@ ProgramNode(0...46)( nil, (0...1), nil, - ArgumentsNode(2...11)([LambdaNode(2...11)([], (2...4), nil, nil)]), + ArgumentsNode(2...11)( + [LambdaNode(2...11)([], (2...4), (5...7), (8...11), nil, nil)] + ), nil, nil, 0, @@ -17,7 +19,9 @@ ProgramNode(0...46)( nil, (13...14), nil, - ArgumentsNode(15...20)([LambdaNode(15...20)([], (15...17), nil, nil)]), + ArgumentsNode(15...20)( + [LambdaNode(15...20)([], (15...17), (18...19), (19...20), nil, nil)] + ), nil, nil, 0, @@ -32,6 +36,8 @@ ProgramNode(0...46)( [LambdaNode(24...35)( [], (24...26), + (29...31), + (32...35), BlockParametersNode(26...28)(nil, [], (26...27), (27...28)), nil )] @@ -50,6 +56,8 @@ ProgramNode(0...46)( [LambdaNode(39...46)( [], (39...41), + (44...45), + (45...46), BlockParametersNode(41...43)(nil, [], (41...42), (42...43)), nil )] diff --git a/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt b/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt index bbbd0a3fcf5bda..d999e4ea4729ba 100644 --- a/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt +++ b/test/yarp/snapshots/seattlerb/stabby_arg_no_paren.txt @@ -4,6 +4,8 @@ ProgramNode(0...5)( [LambdaNode(0...5)( [:a], (0...2), + (3...4), + (4...5), BlockParametersNode(2...3)( ParametersNode(2...3)( [RequiredParameterNode(2...3)(:a)], diff --git a/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt b/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt index 9475ba9c3f960e..0f64173a920c99 100644 --- a/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt +++ b/test/yarp/snapshots/seattlerb/stabby_arg_opt_splat_arg_block_omfg.txt @@ -4,6 +4,8 @@ ProgramNode(0...23)( [LambdaNode(0...23)( [:b, :c, :d, :e, :f], (0...2), + (21...22), + (22...23), BlockParametersNode(2...21)( ParametersNode(3...20)( [RequiredParameterNode(3...4)(:b)], diff --git a/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt b/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt index f84e8227f954e0..719c895b99b89c 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_iter_call.txt @@ -10,6 +10,8 @@ ProgramNode(0...25)( [LambdaNode(2...25)( [], (2...4), + (8...10), + (22...25), BlockParametersNode(5...7)(nil, [], (5...6), (6...7)), StatementsNode(11...21)( [CallNode(11...21)( diff --git a/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt b/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt index c24b3c63c75e1e..14b7a3eb982ed5 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_iter_call_no_target_with_arg.txt @@ -10,6 +10,8 @@ ProgramNode(0...26)( [LambdaNode(2...26)( [], (2...4), + (8...10), + (23...26), BlockParametersNode(5...7)(nil, [], (5...6), (6...7)), StatementsNode(11...22)( [CallNode(11...22)( diff --git a/test/yarp/snapshots/seattlerb/stabby_block_kw.txt b/test/yarp/snapshots/seattlerb/stabby_block_kw.txt index 9554c9d37a6983..8df95715c9f04e 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_kw.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_kw.txt @@ -4,6 +4,8 @@ ProgramNode(0...13)( [LambdaNode(0...13)( [:k], (0...2), + (10...11), + (12...13), BlockParametersNode(3...9)( ParametersNode(4...8)( [], diff --git a/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt b/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt index e2bdaf8124cd6b..8d3e73af8b7766 100644 --- a/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt +++ b/test/yarp/snapshots/seattlerb/stabby_block_kw__required.txt @@ -4,6 +4,8 @@ ProgramNode(0...11)( [LambdaNode(0...11)( [:k], (0...2), + (8...9), + (10...11), BlockParametersNode(3...7)( ParametersNode(4...6)( [], diff --git a/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt b/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt index d8aef65941e5b0..c4594997ebd59e 100644 --- a/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt +++ b/test/yarp/snapshots/seattlerb/stabby_proc_scope.txt @@ -4,6 +4,8 @@ ProgramNode(0...11)( [LambdaNode(0...11)( [:a, :b], (0...2), + (9...10), + (10...11), BlockParametersNode(2...8)( ParametersNode(3...4)( [RequiredParameterNode(3...4)(:a)], diff --git a/test/yarp/snapshots/unparser/corpus/literal/lambda.txt b/test/yarp/snapshots/unparser/corpus/literal/lambda.txt index 66300fd726cda4..13161f9c881262 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/lambda.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/lambda.txt @@ -46,12 +46,16 @@ ProgramNode(0...80)( LambdaNode(33...41)( [], (33...35), + (38...39), + (40...41), BlockParametersNode(35...37)(nil, [], (35...36), (36...37)), nil ), LambdaNode(42...51)( [:a], (42...44), + (48...49), + (50...51), BlockParametersNode(44...47)( ParametersNode(45...46)( [RequiredParameterNode(45...46)(:a)], @@ -71,6 +75,8 @@ ProgramNode(0...80)( LambdaNode(52...64)( [:a, :b], (52...54), + (61...62), + (63...64), BlockParametersNode(54...60)( ParametersNode(55...59)( [RequiredParameterNode(55...56)(:a), @@ -91,6 +97,8 @@ ProgramNode(0...80)( LambdaNode(65...80)( [:a, :b, :c], (65...67), + (77...78), + (79...80), BlockParametersNode(67...76)( ParametersNode(68...72)( [RequiredParameterNode(68...69)(:a), diff --git a/test/yarp/snapshots/unparser/corpus/literal/since/27.txt b/test/yarp/snapshots/unparser/corpus/literal/since/27.txt index cccfbc64b47061..f8709ae21af282 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/since/27.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/since/27.txt @@ -4,6 +4,8 @@ ProgramNode(0...22)( [LambdaNode(0...16)( [], (0...2), + (3...4), + (15...16), nil, StatementsNode(7...14)( [CallNode(7...14)( diff --git a/test/yarp/snapshots/whitequark/bug_435.txt b/test/yarp/snapshots/whitequark/bug_435.txt index ed5a985e3a65ac..7d74c8c8a2af64 100644 --- a/test/yarp/snapshots/whitequark/bug_435.txt +++ b/test/yarp/snapshots/whitequark/bug_435.txt @@ -9,6 +9,8 @@ ProgramNode(0...14)( [LambdaNode(3...12)( [:foo], (3...5), + (10...11), + (11...12), BlockParametersNode(6...9)( ParametersNode(6...9)( [RequiredParameterNode(6...9)(:foo)], diff --git a/test/yarp/snapshots/whitequark/bug_cmdarg.txt b/test/yarp/snapshots/whitequark/bug_cmdarg.txt index f953e561456ef8..b3f79aafd460fc 100644 --- a/test/yarp/snapshots/whitequark/bug_cmdarg.txt +++ b/test/yarp/snapshots/whitequark/bug_cmdarg.txt @@ -55,6 +55,8 @@ ProgramNode(0...56)( LambdaNode(35...56)( [], (35...37), + (38...40), + (53...56), nil, StatementsNode(41...52)( [CallNode(41...52)( diff --git a/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt b/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt index c59109d0a95c16..7792fa495af6ce 100644 --- a/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt +++ b/test/yarp/snapshots/whitequark/bug_lambda_leakage.txt @@ -4,6 +4,8 @@ ProgramNode(0...19)( [LambdaNode(0...12)( [:scope], (0...2), + (10...11), + (11...12), BlockParametersNode(2...9)( ParametersNode(3...8)( [RequiredParameterNode(3...8)(:scope)], diff --git a/test/yarp/snapshots/whitequark/kwnilarg.txt b/test/yarp/snapshots/whitequark/kwnilarg.txt index d7ca2692996c3c..0995a597c26dfa 100644 --- a/test/yarp/snapshots/whitequark/kwnilarg.txt +++ b/test/yarp/snapshots/whitequark/kwnilarg.txt @@ -4,6 +4,8 @@ ProgramNode(0...46)( [LambdaNode(0...12)( [], (0...2), + (10...11), + (11...12), BlockParametersNode(2...9)( ParametersNode(3...8)( [], diff --git a/test/yarp/snapshots/whitequark/numbered_args_after_27.txt b/test/yarp/snapshots/whitequark/numbered_args_after_27.txt index 45decfc3b708e5..c2a67b4cc32dc0 100644 --- a/test/yarp/snapshots/whitequark/numbered_args_after_27.txt +++ b/test/yarp/snapshots/whitequark/numbered_args_after_27.txt @@ -4,6 +4,8 @@ ProgramNode(0...65)( [LambdaNode(0...17)( [], (0...2), + (3...5), + (14...17), nil, StatementsNode(6...13)( [CallNode(6...13)( @@ -34,6 +36,8 @@ ProgramNode(0...65)( LambdaNode(19...32)( [], (19...21), + (22...23), + (31...32), nil, StatementsNode(24...31)( [CallNode(24...31)( diff --git a/test/yarp/snapshots/whitequark/parser_bug_507.txt b/test/yarp/snapshots/whitequark/parser_bug_507.txt index e0b69e4c8b80df..4d6d91a763f225 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_507.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_507.txt @@ -7,6 +7,8 @@ ProgramNode(0...19)( LambdaNode(4...19)( [:args], (4...6), + (13...15), + (16...19), BlockParametersNode(7...12)( ParametersNode(7...12)( [], diff --git a/test/yarp/snapshots/whitequark/parser_bug_645.txt b/test/yarp/snapshots/whitequark/parser_bug_645.txt index 22ae40a011c68a..27680b00aecf85 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_645.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_645.txt @@ -4,6 +4,8 @@ ProgramNode(0...14)( [LambdaNode(0...14)( [:arg], (0...2), + (12...13), + (13...14), BlockParametersNode(3...11)( ParametersNode(4...10)( [], diff --git a/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt b/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt index 475b72dbe80e4e..89825a7ac412ac 100644 --- a/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt +++ b/test/yarp/snapshots/whitequark/rescue_in_lambda_block.txt @@ -4,6 +4,8 @@ ProgramNode(0...17)( [LambdaNode(0...17)( [], (0...2), + (3...5), + (14...17), nil, BeginNode(6...17)( nil, diff --git a/test/yarp/snapshots/whitequark/ruby_bug_11107.txt b/test/yarp/snapshots/whitequark/ruby_bug_11107.txt index 0f5b129c5cde37..c61ee9ef80d316 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_11107.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_11107.txt @@ -10,6 +10,8 @@ ProgramNode(0...24)( [LambdaNode(2...24)( [], (2...4), + (7...9), + (21...24), BlockParametersNode(4...6)(nil, [], (4...5), (5...6)), StatementsNode(10...20)( [CallNode(10...20)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_11380.txt b/test/yarp/snapshots/whitequark/ruby_bug_11380.txt index 5962097a6a4075..9e89161e7817e9 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_11380.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_11380.txt @@ -10,6 +10,8 @@ ProgramNode(0...28)( [LambdaNode(2...15)( [], (2...4), + (5...6), + (14...15), nil, StatementsNode(7...13)( [SymbolNode(7...13)((7...8), (8...13), nil, "hello")] diff --git a/test/yarp/snapshots/whitequark/ruby_bug_15789.txt b/test/yarp/snapshots/whitequark/ruby_bug_15789.txt index 331c1e5cd858ad..0621e71c8cc3f5 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_15789.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_15789.txt @@ -10,6 +10,8 @@ ProgramNode(0...41)( [LambdaNode(2...20)( [:a], (2...4), + (17...18), + (19...20), BlockParametersNode(4...16)( ParametersNode(5...15)( [], @@ -20,6 +22,8 @@ ProgramNode(0...41)( LambdaNode(9...15)( [], (9...11), + (11...12), + (14...15), nil, StatementsNode(12...14)( [CallNode(12...14)( @@ -63,6 +67,8 @@ ProgramNode(0...41)( [LambdaNode(24...41)( [:a], (24...26), + (38...39), + (40...41), BlockParametersNode(26...37)( ParametersNode(27...36)( [], @@ -74,6 +80,8 @@ ProgramNode(0...41)( LambdaNode(30...36)( [], (30...32), + (32...33), + (35...36), nil, StatementsNode(33...35)( [CallNode(33...35)( diff --git a/test/yarp/snapshots/whitequark/send_lambda.txt b/test/yarp/snapshots/whitequark/send_lambda.txt index 3c9f4f56cb47c4..c30ec09249f4e6 100644 --- a/test/yarp/snapshots/whitequark/send_lambda.txt +++ b/test/yarp/snapshots/whitequark/send_lambda.txt @@ -4,6 +4,8 @@ ProgramNode(0...26)( [LambdaNode(0...8)( [:*], (0...2), + (5...6), + (7...8), BlockParametersNode(3...4)( ParametersNode(3...4)( [], @@ -20,7 +22,7 @@ ProgramNode(0...26)( ), nil ), - LambdaNode(10...19)([], (10...12), nil, nil), - LambdaNode(21...26)([], (21...23), nil, nil)] + LambdaNode(10...19)([], (10...12), (13...15), (16...19), nil, nil), + LambdaNode(21...26)([], (21...23), (23...24), (25...26), nil, nil)] ) ) diff --git a/test/yarp/snapshots/whitequark/send_lambda_args.txt b/test/yarp/snapshots/whitequark/send_lambda_args.txt index f904384781ba55..ae41cfa463fa6f 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_args.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_args.txt @@ -4,6 +4,8 @@ ProgramNode(0...21)( [LambdaNode(0...10)( [:a], (0...2), + (7...8), + (9...10), BlockParametersNode(3...6)( ParametersNode(4...5)( [RequiredParameterNode(4...5)(:a)], @@ -23,6 +25,8 @@ ProgramNode(0...21)( LambdaNode(12...21)( [:a], (12...14), + (18...19), + (20...21), BlockParametersNode(14...17)( ParametersNode(15...16)( [RequiredParameterNode(15...16)(:a)], diff --git a/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt b/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt index 8767cf48fac00d..ebb6fb106518f6 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_args_noparen.txt @@ -4,6 +4,8 @@ ProgramNode(0...22)( [LambdaNode(0...11)( [:a], (0...2), + (8...9), + (10...11), BlockParametersNode(3...7)( ParametersNode(3...7)( [], @@ -23,6 +25,8 @@ ProgramNode(0...22)( LambdaNode(13...22)( [:a], (13...15), + (19...20), + (21...22), BlockParametersNode(16...18)( ParametersNode(16...18)( [], diff --git a/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt b/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt index a2e92a67dcc056..9a6c888a930c90 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_args_shadow.txt @@ -4,6 +4,8 @@ ProgramNode(0...19)( [LambdaNode(0...19)( [:a, :foo, :bar], (0...2), + (16...17), + (18...19), BlockParametersNode(2...15)( ParametersNode(3...4)( [RequiredParameterNode(3...4)(:a)], diff --git a/test/yarp/snapshots/whitequark/send_lambda_legacy.txt b/test/yarp/snapshots/whitequark/send_lambda_legacy.txt index fdc30c30dcb16e..57b827cb7edd27 100644 --- a/test/yarp/snapshots/whitequark/send_lambda_legacy.txt +++ b/test/yarp/snapshots/whitequark/send_lambda_legacy.txt @@ -1,4 +1,6 @@ ProgramNode(0...5)( [], - StatementsNode(0...5)([LambdaNode(0...5)([], (0...2), nil, nil)]) + StatementsNode(0...5)( + [LambdaNode(0...5)([], (0...2), (2...3), (4...5), nil, nil)] + ) ) diff --git a/yarp/config.yml b/yarp/config.yml index 62a3c2249c2e98..c7fec6f8468fae 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -1468,8 +1468,12 @@ nodes: child_nodes: - name: locals type: constant[] + - name: operator_loc + type: location - name: opening_loc type: location + - name: closing_loc + type: location - name: parameters type: node? kind: BlockParametersNode diff --git a/yarp/yarp.c b/yarp/yarp.c index 46a57fe4e25abf..87d6a16cfa84a1 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2931,10 +2931,11 @@ static yp_lambda_node_t * yp_lambda_node_create( yp_parser_t *parser, yp_constant_id_list_t *locals, + const yp_token_t *operator, const yp_token_t *opening, + const yp_token_t *closing, yp_block_parameters_node_t *parameters, - yp_node_t *body, - const yp_token_t *closing + yp_node_t *body ) { yp_lambda_node_t *node = YP_ALLOC_NODE(parser, yp_lambda_node_t); @@ -2942,12 +2943,14 @@ yp_lambda_node_create( { .type = YP_NODE_LAMBDA_NODE, .location = { - .start = opening->start, + .start = operator->start, .end = closing->end }, }, .locals = *locals, + .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), + .closing_loc = YP_LOCATION_TOKEN_VALUE(closing), .parameters = parameters, .body = body }; @@ -12424,25 +12427,25 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_accepts_block_stack_push(parser, true); parser_lex(parser); - yp_token_t opening = parser->previous; + yp_token_t operator = parser->previous; yp_parser_scope_push(parser, false); yp_block_parameters_node_t *params; switch (parser->current.type) { case YP_TOKEN_PARENTHESIS_LEFT: { - yp_token_t block_parameters_opening = parser->current; + yp_token_t opening = parser->current; parser_lex(parser); if (match_type_p(parser, YP_TOKEN_PARENTHESIS_RIGHT)) { - params = yp_block_parameters_node_create(parser, NULL, &block_parameters_opening); + params = yp_block_parameters_node_create(parser, NULL, &opening); } else { - params = parse_block_parameters(parser, false, &block_parameters_opening, true); + params = parse_block_parameters(parser, false, &opening, true); } accept(parser, YP_TOKEN_NEWLINE); expect(parser, YP_TOKEN_PARENTHESIS_RIGHT, "Expected ')' after left parenthesis."); - yp_block_parameters_node_closing_set(params, &parser->previous); + yp_block_parameters_node_closing_set(params, &parser->previous); break; } case YP_CASE_PARAMETER: { @@ -12458,16 +12461,20 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { } } + yp_token_t opening; yp_node_t *body = NULL; parser->lambda_enclosure_nesting = previous_lambda_enclosure_nesting; if (accept(parser, YP_TOKEN_LAMBDA_BEGIN)) { + opening = parser->previous; + if (!accept(parser, YP_TOKEN_BRACE_RIGHT)) { body = (yp_node_t *) parse_statements(parser, YP_CONTEXT_LAMBDA_BRACES); expect(parser, YP_TOKEN_BRACE_RIGHT, "Expecting '}' to close lambda block."); } } else { expect(parser, YP_TOKEN_KEYWORD_DO, "Expected a 'do' keyword or a '{' to open lambda block."); + opening = parser->previous; if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_END, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { body = (yp_node_t *) parse_statements(parser, YP_CONTEXT_LAMBDA_DO_END); @@ -12484,7 +12491,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_constant_id_list_t locals = parser->current_scope->locals; yp_parser_scope_pop(parser); yp_accepts_block_stack_pop(parser); - return (yp_node_t *) yp_lambda_node_create(parser, &locals, &opening, params, body, &parser->previous); + return (yp_node_t *) yp_lambda_node_create(parser, &locals, &operator, &opening, &parser->previous, params, body); } case YP_TOKEN_UPLUS: { parser_lex(parser); From 55a8add304cdc88a466d396588f1f2d0022f7c45 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Aug 2023 12:20:08 -0400 Subject: [PATCH 019/208] [ruby/yarp] Fix rational parsing https://github.com/ruby/yarp/commit/c8f31eb5b6 --- lib/yarp.rb | 2 +- test/yarp/ruby_api_test.rb | 24 +++++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/yarp.rb b/lib/yarp.rb index 5a176e9e62d7c4..f43aaa3d12b3d9 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -343,7 +343,7 @@ def value class RationalNode < Node def value - Rational(numeric.value) + Rational(slice.chomp("r")) end end diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index a26b971b824732..1f66084ad2a38c 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -8,7 +8,6 @@ def test_ruby_api source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value - assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) serialized = YARP.dump(source, filepath) @@ -21,13 +20,20 @@ def test_ruby_api end def test_literal_value_method - assert_equal 123, YARP.parse("123").value.statements.body.first.value - assert_equal 3.14, YARP.parse("3.14").value.statements.body.first.value - assert_equal 42i, YARP.parse("42i").value.statements.body.first.value - assert_equal 3.14i, YARP.parse("3.14i").value.statements.body.first.value - assert_equal 42r, YARP.parse("42r").value.statements.body.first.value - assert_equal 0.5r, YARP.parse("0.5r").value.statements.body.first.value - assert_equal 42ri, YARP.parse("42ri").value.statements.body.first.value - assert_equal 0.5ri, YARP.parse("0.5ri").value.statements.body.first.value + assert_equal 123, parse_expression("123").value + assert_equal 3.14, parse_expression("3.14").value + assert_equal 42i, parse_expression("42i").value + assert_equal 42.1ri, parse_expression("42.1ri").value + assert_equal 3.14i, parse_expression("3.14i").value + assert_equal 42r, parse_expression("42r").value + assert_equal 0.5r, parse_expression("0.5r").value + assert_equal 42ri, parse_expression("42ri").value + assert_equal 0.5ri, parse_expression("0.5ri").value + end + + private + + def parse_expression(source) + YARP.parse(source).value.statements.body.first end end From 20cf9e3ae8ef05d7fc44e2fda8c5b8233e8eb03e Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Aug 2023 12:21:35 -0400 Subject: [PATCH 020/208] [ruby/yarp] Add a Location#to method for combining them https://github.com/ruby/yarp/commit/1db2de98ac --- lib/yarp.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/yarp.rb b/lib/yarp.rb index f43aaa3d12b3d9..4c3cf268ce35a3 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -112,6 +112,11 @@ def ==(other) other.end_offset == end_offset end + # Returns a new location that is the union of this location and the other. + def to(other) + Location.new(source, start_offset, other.end_offset - start_offset) + end + def self.null new(0, 0) end From 0c1a749eef476045e0b098467a564e888ef03c47 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 22 Aug 2023 13:26:26 -0400 Subject: [PATCH 021/208] [ruby/yarp] Fix nested multi assignment locations https://github.com/ruby/yarp/commit/9a65f002dc --- test/yarp/snapshots/lambda.txt | 4 ++ .../seattlerb/masgn_double_paren.txt | 8 ++-- test/yarp/snapshots/seattlerb/masgn_paren.txt | 6 +-- .../unparser/corpus/literal/assignment.txt | 40 +++++++++---------- .../unparser/corpus/literal/defined.txt | 4 +- .../unparser/corpus/literal/send.txt | 4 +- test/yarp/snapshots/variables.txt | 2 +- test/yarp/snapshots/whitequark/masgn.txt | 6 +-- .../snapshots/whitequark/masgn_nested.txt | 10 ++--- yarp/yarp.c | 18 ++++++--- 10 files changed, 57 insertions(+), 45 deletions(-) diff --git a/test/yarp/snapshots/lambda.txt b/test/yarp/snapshots/lambda.txt index 0832abf4e8077d..6640b585c32140 100644 --- a/test/yarp/snapshots/lambda.txt +++ b/test/yarp/snapshots/lambda.txt @@ -114,6 +114,8 @@ ProgramNode(0...92)( LambdaNode(53...72)( [:foo], (53...55), + (66...68), + (69...72), BlockParametersNode(56...65)( ParametersNode(56...65)( [], @@ -148,6 +150,8 @@ ProgramNode(0...92)( LambdaNode(74...92)( [:foo], (74...76), + (86...88), + (89...92), BlockParametersNode(77...85)( ParametersNode(77...85)( [], diff --git a/test/yarp/snapshots/seattlerb/masgn_double_paren.txt b/test/yarp/snapshots/seattlerb/masgn_double_paren.txt index 72f0237aba9465..3d0c2cf7ab6fbd 100644 --- a/test/yarp/snapshots/seattlerb/masgn_double_paren.txt +++ b/test/yarp/snapshots/seattlerb/masgn_double_paren.txt @@ -1,8 +1,8 @@ -ProgramNode(2...9)( +ProgramNode(0...9)( [:a, :b], - StatementsNode(2...9)( - [MultiWriteNode(2...9)( - [MultiWriteNode(2...5)( + StatementsNode(0...9)( + [MultiWriteNode(0...9)( + [MultiWriteNode(1...6)( [LocalVariableTargetNode(2...3)(:a, 0), LocalVariableTargetNode(4...5)(:b, 0)], nil, diff --git a/test/yarp/snapshots/seattlerb/masgn_paren.txt b/test/yarp/snapshots/seattlerb/masgn_paren.txt index 20301dff96d676..91dd386d8c4a35 100644 --- a/test/yarp/snapshots/seattlerb/masgn_paren.txt +++ b/test/yarp/snapshots/seattlerb/masgn_paren.txt @@ -1,7 +1,7 @@ -ProgramNode(1...12)( +ProgramNode(0...12)( [:a, :b], - StatementsNode(1...12)( - [MultiWriteNode(1...12)( + StatementsNode(0...12)( + [MultiWriteNode(0...12)( [LocalVariableTargetNode(1...2)(:a, 0), LocalVariableTargetNode(4...5)(:b, 0)], (7...8), diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 6ee980d7c06511..bcf853886f0d8c 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -2,7 +2,7 @@ ProgramNode(0...704)( [:a, :b, :foo, :c, :x], StatementsNode(0...704)( [GlobalVariableWriteNode(0...6)((0...2), (3...4), IntegerNode(5...6)()), - MultiWriteNode(8...24)( + MultiWriteNode(7...24)( [GlobalVariableTargetNode(8...10)(), GlobalVariableTargetNode(12...14)()], (16...17), @@ -14,8 +14,8 @@ ProgramNode(0...704)( (7...8), (14...15) ), - MultiWriteNode(27...38)( - [MultiWriteNode(27...29)( + MultiWriteNode(25...38)( + [MultiWriteNode(26...30)( [LocalVariableTargetNode(27...28)(:a, 0), SplatNode(28...29)((28...29), nil)], nil, @@ -29,7 +29,7 @@ ProgramNode(0...704)( (25...26), (33...34) ), - MultiWriteNode(40...48)( + MultiWriteNode(39...48)( [SplatNode(40...42)( (40...41), LocalVariableTargetNode(41...42)(:a, 0) @@ -39,7 +39,7 @@ ProgramNode(0...704)( (39...40), (42...43) ), - MultiWriteNode(50...64)( + MultiWriteNode(49...64)( [SplatNode(50...54)( (50...51), LocalVariableTargetNode(51...54)(:foo, 0) @@ -53,7 +53,7 @@ ProgramNode(0...704)( (49...50), (54...55) ), - MultiWriteNode(66...84)( + MultiWriteNode(65...84)( [ClassVariableTargetNode(66...69)(), ClassVariableTargetNode(71...74)()], (76...77), @@ -65,7 +65,7 @@ ProgramNode(0...704)( (65...66), (74...75) ), - MultiWriteNode(86...102)( + MultiWriteNode(85...102)( [InstanceVariableTargetNode(86...88)(), InstanceVariableTargetNode(90...92)()], (94...95), @@ -77,9 +77,9 @@ ProgramNode(0...704)( (85...86), (92...93) ), - MultiWriteNode(104...128)( + MultiWriteNode(103...128)( [LocalVariableTargetNode(104...105)(:a, 0), - MultiWriteNode(108...113)( + MultiWriteNode(107...113)( [LocalVariableTargetNode(108...109)(:b, 0), LocalVariableTargetNode(111...112)(:c, 0)], nil, @@ -101,7 +101,7 @@ ProgramNode(0...704)( (103...104), (113...114) ), - MultiWriteNode(130...144)( + MultiWriteNode(129...144)( [LocalVariableTargetNode(130...131)(:a, 0), SplatNode(133...134)((133...134), nil)], (136...137), @@ -113,7 +113,7 @@ ProgramNode(0...704)( (129...130), (134...135) ), - MultiWriteNode(146...163)( + MultiWriteNode(145...163)( [LocalVariableTargetNode(146...147)(:a, 0), SplatNode(149...153)( (149...150), @@ -128,7 +128,7 @@ ProgramNode(0...704)( (145...146), (153...154) ), - MultiWriteNode(165...179)( + MultiWriteNode(164...179)( [LocalVariableTargetNode(165...166)(:a, 0), LocalVariableTargetNode(168...169)(:b, 0)], (171...172), @@ -140,7 +140,7 @@ ProgramNode(0...704)( (164...165), (169...170) ), - MultiWriteNode(181...192)( + MultiWriteNode(180...192)( [LocalVariableTargetNode(181...182)(:a, 0), LocalVariableTargetNode(184...185)(:b, 0)], (187...188), @@ -148,7 +148,7 @@ ProgramNode(0...704)( (180...181), (185...186) ), - MultiWriteNode(194...203)( + MultiWriteNode(193...203)( [LocalVariableTargetNode(194...195)(:a, 0), SplatNode(195...196)((195...196), nil)], (198...199), @@ -156,7 +156,7 @@ ProgramNode(0...704)( (193...194), (196...197) ), - MultiWriteNode(205...227)( + MultiWriteNode(204...227)( [CallNode(205...210)( LocalVariableReadNode(205...206)(:a, 0), (206...207), @@ -188,7 +188,7 @@ ProgramNode(0...704)( (204...205), (217...218) ), - MultiWriteNode(229...252)( + MultiWriteNode(228...252)( [CallNode(229...236)( LocalVariableReadNode(229...230)(:a, 0), nil, @@ -225,7 +225,7 @@ ProgramNode(0...704)( (228...229), (242...243) ), - MultiWriteNode(254...274)( + MultiWriteNode(253...274)( [CallNode(254...258)( LocalVariableReadNode(254...255)(:a, 0), nil, @@ -257,7 +257,7 @@ ProgramNode(0...704)( (253...254), (264...265) ), - MultiWriteNode(276...287)( + MultiWriteNode(275...287)( [SplatNode(276...282)( (276...277), CallNode(277...282)( @@ -322,8 +322,8 @@ ProgramNode(0...704)( :a, 0, ParenthesesNode(355...367)( - StatementsNode(357...366)( - [MultiWriteNode(357...366)( + StatementsNode(356...366)( + [MultiWriteNode(356...366)( [LocalVariableTargetNode(357...358)(:b, 0), LocalVariableTargetNode(360...361)(:c, 0)], (363...364), diff --git a/test/yarp/snapshots/unparser/corpus/literal/defined.txt b/test/yarp/snapshots/unparser/corpus/literal/defined.txt index 7ba02da908979e..c2d6a71bb1b132 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/defined.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/defined.txt @@ -16,8 +16,8 @@ ProgramNode(0...56)( DefinedNode(29...56)( (37...38), ParenthesesNode(38...55)( - StatementsNode(40...54)( - [MultiWriteNode(40...54)( + StatementsNode(39...54)( + [MultiWriteNode(39...54)( [LocalVariableTargetNode(40...41)(:a, 0), LocalVariableTargetNode(43...44)(:b, 0)], (46...47), diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index ca65b7d58d4b12..1d3cd313b706dc 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -10,8 +10,8 @@ ProgramNode(0...999)( (11...14), (15...18), ParenthesesNode(19...31)( - StatementsNode(21...30)( - [MultiWriteNode(21...30)( + StatementsNode(20...30)( + [MultiWriteNode(20...30)( [LocalVariableTargetNode(21...22)(:a, 0), LocalVariableTargetNode(24...25)(:_, 0)], (27...28), diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index 5d96122709915d..35b90dbae3b484 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -148,7 +148,7 @@ ProgramNode(0...293)( ), MultiWriteNode(231...258)( [LocalVariableTargetNode(231...234)(:foo, 0), - MultiWriteNode(237...246)( + MultiWriteNode(236...246)( [LocalVariableTargetNode(237...240)(:bar, 0), LocalVariableTargetNode(242...245)(:baz, 0)], nil, diff --git a/test/yarp/snapshots/whitequark/masgn.txt b/test/yarp/snapshots/whitequark/masgn.txt index 466758f3ab6ca2..69667e1a38b5a9 100644 --- a/test/yarp/snapshots/whitequark/masgn.txt +++ b/test/yarp/snapshots/whitequark/masgn.txt @@ -1,7 +1,7 @@ -ProgramNode(1...56)( +ProgramNode(0...56)( [:foo, :bar, :baz], - StatementsNode(1...56)( - [MultiWriteNode(1...17)( + StatementsNode(0...56)( + [MultiWriteNode(0...17)( [LocalVariableTargetNode(1...4)(:foo, 0), LocalVariableTargetNode(6...9)(:bar, 0)], (11...12), diff --git a/test/yarp/snapshots/whitequark/masgn_nested.txt b/test/yarp/snapshots/whitequark/masgn_nested.txt index 3b087771354563..b1601b8aa7436a 100644 --- a/test/yarp/snapshots/whitequark/masgn_nested.txt +++ b/test/yarp/snapshots/whitequark/masgn_nested.txt @@ -1,8 +1,8 @@ -ProgramNode(2...30)( +ProgramNode(0...30)( [:b, :a, :c], - StatementsNode(2...30)( - [MultiWriteNode(2...13)( - [MultiWriteNode(2...4)( + StatementsNode(0...30)( + [MultiWriteNode(0...13)( + [MultiWriteNode(1...6)( [LocalVariableTargetNode(2...3)(:b, 0), SplatNode(3...4)((3...4), nil)], nil, @@ -17,7 +17,7 @@ ProgramNode(2...30)( ), MultiWriteNode(15...30)( [LocalVariableTargetNode(15...16)(:a, 0), - MultiWriteNode(19...24)( + MultiWriteNode(18...24)( [LocalVariableTargetNode(19...20)(:b, 0), LocalVariableTargetNode(22...23)(:c, 0)], nil, diff --git a/yarp/yarp.c b/yarp/yarp.c index 87d6a16cfa84a1..497a660ed98363 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -3164,7 +3164,10 @@ yp_multi_write_node_create(yp_parser_t *parser, const yp_token_t *operator, yp_n *node = (yp_multi_write_node_t) { { .type = YP_NODE_MULTI_WRITE_NODE, - .location = { .start = NULL, .end = NULL }, + .location = { + .start = lparen_loc->start, + .end = value == NULL ? rparen_loc->end : value->location.end + }, }, .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value, @@ -8202,6 +8205,8 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b if (YP_NODE_TYPE_P(child_target, YP_NODE_MULTI_WRITE_NODE)) { target = (yp_multi_write_node_t *) child_target; + target->base.location.start = lparen.start; + target->base.location.end = rparen.end; target->lparen_loc = (yp_location_t) { .start = lparen.start, .end = lparen.end }; target->rparen_loc = (yp_location_t) { .start = rparen.start, .end = rparen.end }; } else { @@ -8218,6 +8223,7 @@ parse_targets(yp_parser_t *parser, yp_node_t *first_target, yp_binding_power_t b yp_multi_write_node_targets_append(target, child_target); } + target->base.location.start = lparen.start; target->base.location.end = rparen.end; yp_multi_write_node_targets_append(result, (yp_node_t *) target); } @@ -10690,10 +10696,8 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { } case YP_TOKEN_PARENTHESIS_LEFT: case YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES: { - yp_token_type_t current_token_type = parser->current.type; + yp_token_t opening = parser->current; parser_lex(parser); - - yp_token_t opening = parser->previous; while (accept_any(parser, 2, YP_TOKEN_SEMICOLON, YP_TOKEN_NEWLINE)); // If this is the end of the file or we match a right parenthesis, then @@ -10712,7 +10716,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // If we hit a right parenthesis, then we're done parsing the parentheses // node, and we can check which kind of node we should return. if (match_type_p(parser, YP_TOKEN_PARENTHESIS_RIGHT)) { - if (current_token_type == YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES) { + if (opening.type == YP_TOKEN_PARENTHESIS_LEFT_PARENTHESES) { lex_state_set(parser, YP_LEX_STATE_ENDARG); } parser_lex(parser); @@ -10730,6 +10734,8 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { if (multi_statement->lparen_loc.start == NULL) { multi_write = (yp_multi_write_node_t *) statement; + multi_write->base.location.start = lparen_loc.start; + multi_write->base.location.end = rparen_loc.end; multi_write->lparen_loc = lparen_loc; multi_write->rparen_loc = rparen_loc; } else { @@ -10781,6 +10787,8 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_accepts_block_stack_pop(parser); expect(parser, YP_TOKEN_PARENTHESIS_RIGHT, "Expected a closing parenthesis."); + + return (yp_node_t *) yp_parentheses_node_create(parser, &opening, (yp_node_t *) statements, &parser->previous); } case YP_TOKEN_BRACE_LEFT: { From 481388769407b533879e97510dc8160d094356e0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 23 Aug 2023 10:53:08 -0400 Subject: [PATCH 022/208] [ruby/yarp] Accept a block to parse and parse_file to get lexer output as well https://github.com/ruby/yarp/commit/40fbf61a8d --- yarp/extension.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++- yarp/yarp.c | 2 -- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/yarp/extension.c b/yarp/extension.c index 3c678902001ca4..7e735b195649fe 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -316,16 +316,65 @@ lex_file(VALUE self, VALUE filepath) { /* Parsing Ruby code */ /******************************************************************************/ +// This is passed as a callback to the parser. It gets called every time a new +// token is found from within a call to parse that accepted a block. +static void +parse_token(void *data, yp_parser_t *parser, yp_token_t *token) { + lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; + rb_yield_values(2, yp_token_new(parser, token, lex_data->encoding, lex_data->source), INT2FIX(parser->lex_state)); +} + // Parse the given input and return a ParseResult instance. static VALUE parse_input(yp_string_t *input, const char *filepath) { yp_parser_t parser; yp_parser_init(&parser, yp_string_source(input), yp_string_length(input), filepath); + VALUE offsets; + VALUE source; + + // If a block was given to the parse method, then we're going to register a + // lex callback that will yield the tokens to the block. This means you can + // get the lexer and the parser output in one method call instead of having + // to parse twice. + if (rb_block_given_p()) { + offsets = rb_ary_new(); + + VALUE source_argv[] = { rb_str_new(yp_string_source(input), yp_string_length(input)), offsets }; + source = rb_class_new_instance(2, source_argv, rb_cYARPSource); + + lex_data_t lex_data = { + .source = source, + .tokens = Qnil, + .encoding = rb_utf8_encoding() + }; + + lex_data_t *data = &lex_data; + yp_lex_callback_t lex_callback = (yp_lex_callback_t) { + .data = (void *) data, + .callback = parse_token, + }; + + parser.lex_callback = &lex_callback; + yp_parser_register_encoding_changed_callback(&parser, lex_encoding_changed_callback); + } + yp_node_t *node = yp_parse(&parser); rb_encoding *encoding = rb_enc_find(parser.encoding.name); - VALUE source = yp_source_new(&parser); + if (rb_block_given_p()) { + // Here we need to update the source range to have the correct newline + // offsets. We do it here because we've already created the object and + // given it over to all of the tokens. + for (size_t index = 0; index < parser.newline_list.size; index++) { + rb_ary_push(offsets, INT2FIX(parser.newline_list.offsets[index])); + } + } else { + // Since a block was not given, we can just create the source now the + // regular way. + source = yp_source_new(&parser); + } + VALUE result_argv[] = { yp_ast_new(&parser, node, encoding), parser_comments(&parser, source), diff --git a/yarp/yarp.c b/yarp/yarp.c index 497a660ed98363..c6b7e1c0190043 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -10787,8 +10787,6 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_accepts_block_stack_pop(parser); expect(parser, YP_TOKEN_PARENTHESIS_RIGHT, "Expected a closing parenthesis."); - - return (yp_node_t *) yp_parentheses_node_create(parser, &opening, (yp_node_t *) statements, &parser->previous); } case YP_TOKEN_BRACE_LEFT: { From 76512d78fcde99458db211c0f958bd39cb23dd98 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 14:32:32 -0400 Subject: [PATCH 023/208] [ruby/yarp] Rename Location#to to Location#join, include checks https://github.com/ruby/yarp/commit/de8924e3ec --- lib/yarp.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/yarp.rb b/lib/yarp.rb index 4c3cf268ce35a3..d44f80b9a05b72 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -37,7 +37,7 @@ def compute_offsets(code) class Location # A Source object that is used to determine more information from the given # offset and length. - private attr_reader :source + protected attr_reader :source # The byte offset from the beginning of the source where this location # starts. @@ -112,8 +112,13 @@ def ==(other) other.end_offset == end_offset end - # Returns a new location that is the union of this location and the other. - def to(other) + # Returns a new location that stretches from this location to the given + # other location. Raises an error if this location is not before the other + # location or if they don't share the same source. + def join(other) + raise "Incompatible sources" if source != other.source + raise "Incompatible locations" if start_offset > other.start_offset + Location.new(source, start_offset, other.end_offset - start_offset) end From 9b8602dd903b2515463a1a314cb8fdf735a354aa Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 15:22:42 -0400 Subject: [PATCH 024/208] [ruby/yarp] Introduce parse_lex instead of asking for a block https://github.com/ruby/yarp/commit/7e70339fe1 --- lib/yarp/ffi.rb | 28 ++++- test/yarp/encoding_test.rb | 5 + test/yarp/parse_test.rb | 14 +++ yarp/extension.c | 143 +++++++++++------------ yarp/templates/lib/yarp/serialize.rb.erb | 45 ++++--- yarp/templates/src/serialize.c.erb | 37 +++++- yarp/yarp.c | 2 +- yarp/yarp.h | 6 + 8 files changed, 182 insertions(+), 98 deletions(-) diff --git a/lib/yarp/ffi.rb b/lib/yarp/ffi.rb index 31c1ce3cc718b1..73e5d60dfac237 100644 --- a/lib/yarp/ffi.rb +++ b/lib/yarp/ffi.rb @@ -70,7 +70,8 @@ def self.load_exported_functions_from(header, *functions) "yarp.h", "yp_version", "yp_parse_serialize", - "yp_lex_serialize" + "yp_lex_serialize", + "yp_parse_lex_serialize" ) load_exported_functions_from( @@ -225,4 +226,29 @@ def self.parse_file(filepath) parse(string.read, filepath) end end + + # Mirror the YARP.parse_lex API by using the serialization API. + def self.parse_lex(code, filepath = nil) + LibRubyParser::YPBuffer.with do |buffer| + metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath + LibRubyParser.yp_parse_lex_serialize(code, code.bytesize, buffer.pointer, metadata) + + source = Source.new(code) + loader = Serialize::Loader.new(source, buffer.read) + + tokens = loader.load_tokens + node, comments, errors, warnings = loader.load_nodes + + tokens.each { |token,| token.value.force_encoding(loader.encoding) } + + ParseResult.new([node, tokens], comments, errors, warnings, source) + end + end + + # Mirror the YARP.parse_lex_file API by using the serialization API. + def self.parse_lex_file(filepath) + LibRubyParser::YPString.with(filepath) do |string| + parse_lex(string.read, filepath) + end + end end diff --git a/test/yarp/encoding_test.rb b/test/yarp/encoding_test.rb index c96a08e60e6c23..7bf99ece9270f0 100644 --- a/test/yarp/encoding_test.rb +++ b/test/yarp/encoding_test.rb @@ -90,4 +90,9 @@ def test_utf_8_variations assert_equal Encoding.find("utf-8"), actual end end + + def test_first_lexed_token + encoding = YARP.lex("# encoding: ascii-8bit").value[0][0].value.encoding + assert_equal Encoding.find("ascii-8bit"), encoding + end end diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index f8c1fe12d13443..1f8b1374f6d98d 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -30,6 +30,20 @@ def test_parse_takes_file_path assert_equal filepath, find_source_file_node(result.value).filepath end + def test_parse_lex + node, tokens = YARP.parse_lex("def foo; end").value + + assert_kind_of YARP::ProgramNode, node + assert_equal 5, tokens.length + end + + def test_parse_lex_file + node, tokens = YARP.parse_lex_file(__FILE__).value + + assert_kind_of YARP::ProgramNode, node + refute_empty tokens + end + # To accurately compare against Ripper, we need to make sure that we're # running on Ruby 3.2+. check_ripper = RUBY_VERSION >= "3.2.0" diff --git a/yarp/extension.c b/yarp/extension.c index 7e735b195649fe..b59ccc1bcad942 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -198,66 +198,67 @@ typedef struct { VALUE source; VALUE tokens; rb_encoding *encoding; -} lex_data_t; +} parse_lex_data_t; // This is passed as a callback to the parser. It gets called every time a new // token is found. Once found, we initialize a new instance of Token and push it // onto the tokens array. static void -lex_token(void *data, yp_parser_t *parser, yp_token_t *token) { - lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; +parse_lex_token(void *data, yp_parser_t *parser, yp_token_t *token) { + parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data; VALUE yields = rb_ary_new_capa(2); - rb_ary_push(yields, yp_token_new(parser, token, lex_data->encoding, lex_data->source)); + rb_ary_push(yields, yp_token_new(parser, token, parse_lex_data->encoding, parse_lex_data->source)); rb_ary_push(yields, INT2FIX(parser->lex_state)); - rb_ary_push(lex_data->tokens, yields); + rb_ary_push(parse_lex_data->tokens, yields); } // This is called whenever the encoding changes based on the magic comment at // the top of the file. We use it to update the encoding that we are using to // create tokens. static void -lex_encoding_changed_callback(yp_parser_t *parser) { - lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; - lex_data->encoding = rb_enc_find(parser->encoding.name); +parse_lex_encoding_changed_callback(yp_parser_t *parser) { + parse_lex_data_t *parse_lex_data = (parse_lex_data_t *) parser->lex_callback->data; + parse_lex_data->encoding = rb_enc_find(parser->encoding.name); - // Since we got a new encoding, we need to go back and change the encoding - // of the tokens that we've already lexed. This should be a tiny amount - // since encoding magic comments need to be the first or second line of the + // Since the encoding changed, we need to go back and change the encoding of + // the tokens that were already lexed. This is only going to end up being + // one or two tokens, since the encoding can only change at the top of the // file. - VALUE tokens = lex_data->tokens; + VALUE tokens = parse_lex_data->tokens; for (long index = 0; index < RARRAY_LEN(tokens); index++) { VALUE yields = rb_ary_entry(tokens, index); VALUE token = rb_ary_entry(yields, 0); VALUE value = rb_ivar_get(token, rb_intern("@value")); - rb_enc_associate(value, lex_data->encoding); + rb_enc_associate(value, parse_lex_data->encoding); ENC_CODERANGE_CLEAR(value); } } -// Return an array of tokens corresponding to the given source. +// Parse the given input and return a ParseResult containing just the tokens or +// the nodes and tokens. static VALUE -lex_input(yp_string_t *input, const char *filepath) { +parse_lex_input(yp_string_t *input, const char *filepath, bool return_nodes) { yp_parser_t parser; yp_parser_init(&parser, yp_string_source(input), yp_string_length(input), filepath); - yp_parser_register_encoding_changed_callback(&parser, lex_encoding_changed_callback); + yp_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback); VALUE offsets = rb_ary_new(); VALUE source_argv[] = { rb_str_new(yp_string_source(input), yp_string_length(input)), offsets }; VALUE source = rb_class_new_instance(2, source_argv, rb_cYARPSource); - lex_data_t lex_data = { + parse_lex_data_t parse_lex_data = { .source = source, .tokens = rb_ary_new(), .encoding = rb_utf8_encoding() }; - lex_data_t *data = &lex_data; + parse_lex_data_t *data = &parse_lex_data; yp_lex_callback_t lex_callback = (yp_lex_callback_t) { .data = (void *) data, - .callback = lex_token, + .callback = parse_lex_token, }; parser.lex_callback = &lex_callback; @@ -270,20 +271,26 @@ lex_input(yp_string_t *input, const char *filepath) { rb_ary_push(offsets, INT2FIX(parser.newline_list.offsets[index])); } + VALUE value; + if (return_nodes) { + value = rb_ary_new_capa(2); + rb_ary_push(value, yp_ast_new(&parser, node, parse_lex_data.encoding)); + rb_ary_push(value, parse_lex_data.tokens); + } else { + value = parse_lex_data.tokens; + } + VALUE result_argv[] = { - lex_data.tokens, + value, parser_comments(&parser, source), - parser_errors(&parser, lex_data.encoding, source), - parser_warnings(&parser, lex_data.encoding, source), + parser_errors(&parser, parse_lex_data.encoding, source), + parser_warnings(&parser, parse_lex_data.encoding, source), source }; - VALUE result = rb_class_new_instance(5, result_argv, rb_cYARPParseResult); - yp_node_destroy(&parser, node); yp_parser_free(&parser); - - return result; + return rb_class_new_instance(5, result_argv, rb_cYARPParseResult); } // Return an array of tokens corresponding to the given string. @@ -295,7 +302,8 @@ lex(int argc, VALUE *argv, VALUE self) { yp_string_t input; input_load_string(&input, string); - return lex_input(&input, check_string(filepath)); + + return parse_lex_input(&input, check_string(filepath), false); } // Return an array of tokens corresponding to the given file. @@ -306,7 +314,7 @@ lex_file(VALUE self, VALUE filepath) { const char *checked = check_string(filepath); if (!yp_string_mapped_init(&input, checked)) return Qnil; - VALUE value = lex_input(&input, checked); + VALUE value = parse_lex_input(&input, checked, false); yp_string_free(&input); return value; @@ -316,65 +324,16 @@ lex_file(VALUE self, VALUE filepath) { /* Parsing Ruby code */ /******************************************************************************/ -// This is passed as a callback to the parser. It gets called every time a new -// token is found from within a call to parse that accepted a block. -static void -parse_token(void *data, yp_parser_t *parser, yp_token_t *token) { - lex_data_t *lex_data = (lex_data_t *) parser->lex_callback->data; - rb_yield_values(2, yp_token_new(parser, token, lex_data->encoding, lex_data->source), INT2FIX(parser->lex_state)); -} - // Parse the given input and return a ParseResult instance. static VALUE parse_input(yp_string_t *input, const char *filepath) { yp_parser_t parser; yp_parser_init(&parser, yp_string_source(input), yp_string_length(input), filepath); - VALUE offsets; - VALUE source; - - // If a block was given to the parse method, then we're going to register a - // lex callback that will yield the tokens to the block. This means you can - // get the lexer and the parser output in one method call instead of having - // to parse twice. - if (rb_block_given_p()) { - offsets = rb_ary_new(); - - VALUE source_argv[] = { rb_str_new(yp_string_source(input), yp_string_length(input)), offsets }; - source = rb_class_new_instance(2, source_argv, rb_cYARPSource); - - lex_data_t lex_data = { - .source = source, - .tokens = Qnil, - .encoding = rb_utf8_encoding() - }; - - lex_data_t *data = &lex_data; - yp_lex_callback_t lex_callback = (yp_lex_callback_t) { - .data = (void *) data, - .callback = parse_token, - }; - - parser.lex_callback = &lex_callback; - yp_parser_register_encoding_changed_callback(&parser, lex_encoding_changed_callback); - } - yp_node_t *node = yp_parse(&parser); rb_encoding *encoding = rb_enc_find(parser.encoding.name); - if (rb_block_given_p()) { - // Here we need to update the source range to have the correct newline - // offsets. We do it here because we've already created the object and - // given it over to all of the tokens. - for (size_t index = 0; index < parser.newline_list.size; index++) { - rb_ary_push(offsets, INT2FIX(parser.newline_list.offsets[index])); - } - } else { - // Since a block was not given, we can just create the source now the - // regular way. - source = yp_source_new(&parser); - } - + VALUE source = yp_source_new(&parser); VALUE result_argv[] = { yp_ast_new(&parser, node, encoding), parser_comments(&parser, source), @@ -431,6 +390,32 @@ parse_file(VALUE self, VALUE filepath) { return value; } +// Parse the given string and return a ParseResult instance. +static VALUE +parse_lex(int argc, VALUE *argv, VALUE self) { + VALUE string; + VALUE filepath; + rb_scan_args(argc, argv, "11", &string, &filepath); + + yp_string_t input; + input_load_string(&input, string); + return parse_lex_input(&input, check_string(filepath), true); +} + +// Parse and lex the given file and return a ParseResult instance. +static VALUE +parse_lex_file(VALUE self, VALUE filepath) { + yp_string_t input; + + const char *checked = check_string(filepath); + if (!yp_string_mapped_init(&input, checked)) return Qnil; + + VALUE value = parse_lex_input(&input, checked, true); + yp_string_free(&input); + + return value; +} + /******************************************************************************/ /* Utility functions exposed to make testing easier */ /******************************************************************************/ @@ -590,6 +575,8 @@ Init_yarp(void) { rb_define_singleton_method(rb_cYARP, "lex_file", lex_file, 1); rb_define_singleton_method(rb_cYARP, "parse", parse, -1); rb_define_singleton_method(rb_cYARP, "parse_file", parse_file, 1); + rb_define_singleton_method(rb_cYARP, "parse_lex", parse_lex, -1); + rb_define_singleton_method(rb_cYARP, "parse_lex_file", parse_lex_file, 1); // Next, the functions that will be called by the parser to perform various // internal tasks. We expose these to make them easier to test. diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index 524405d2edd15c..ee9285fdbc5732 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -14,11 +14,11 @@ end module YARP module Serialize def self.load(input, serialized) - Loader.new(Source.new(input), serialized).load + Loader.new(Source.new(input), serialized).load_result end def self.load_tokens(source, serialized) - Loader.new(source, serialized).load_tokens + Loader.new(source, serialized).load_tokens_result end class Loader @@ -39,6 +39,17 @@ module YARP @source = source end + def load_encoding + Encoding.find(io.read(load_varint)) + end + + def load_metadata + comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } + errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } + warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } + [comments, errors, warnings] + end + def load_tokens tokens = [] while type = TOKEN_TYPES.fetch(load_varint) @@ -49,34 +60,42 @@ module YARP tokens << [YARP::Token.new(type, location.slice, location), lex_state] end - comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } - errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } - warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } + tokens + end - raise "Expected to consume all bytes while deserializing" unless @io.eof? + def load_tokens_result + tokens = load_tokens + encoding = load_encoding + comments, errors, warnings = load_metadata + + if encoding != @encoding + tokens.each { |token,| token.value.force_encoding(encoding) } + end + raise "Expected to consume all bytes while deserializing" unless @io.eof? YARP::ParseResult.new(tokens, comments, errors, warnings, @source) end - def load + def load_nodes raise "Invalid serialization" if io.read(4) != "YARP" if io.read(3).unpack("C3") != [<%= YARP_VERSION_MAJOR %>, <%= YARP_VERSION_MINOR %>, <%= YARP_VERSION_PATCH %>] raise "Invalid serialization version" end - @encoding = Encoding.find(io.read(load_varint)) + @encoding = load_encoding @input = input.force_encoding(@encoding).freeze - comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(io.getbyte), load_location) } - errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } - warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } + comments, errors, warnings = load_metadata @constant_pool_offset = io.read(4).unpack1("L") @constant_pool = Array.new(load_varint, nil) - ast = load_node + [load_node, comments, errors, warnings] + end - YARP::ParseResult.new(ast, comments, errors, warnings, @source) + def load_result + node, comments, errors, warnings = load_nodes + YARP::ParseResult.new(node, comments, errors, warnings, @source) end private diff --git a/yarp/templates/src/serialize.c.erb b/yarp/templates/src/serialize.c.erb index 9b49540566c435..73ad0c9fe77657 100644 --- a/yarp/templates/src/serialize.c.erb +++ b/yarp/templates/src/serialize.c.erb @@ -170,14 +170,17 @@ yp_serialize_diagnostic_list(yp_parser_t *parser, yp_list_t *list, yp_buffer_t * } } +static void +yp_serialize_encoding(yp_encoding_t *encoding, yp_buffer_t *buffer) { + size_t encoding_length = strlen(encoding->name); + yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length)); + yp_buffer_append_str(buffer, encoding->name, encoding_length); +} + #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { - // First, serialize the encoding of the parser. - size_t encoding_length = strlen(parser->encoding.name); - yp_buffer_append_u32(buffer, yp_sizet_to_u32(encoding_length)); - yp_buffer_append_str(buffer, parser->encoding.name, encoding_length); - + yp_serialize_encoding(&parser->encoding, buffer); yp_serialize_comment_list(parser, &parser->comment_list, buffer); yp_serialize_diagnostic_list(parser, &parser->error_list, buffer); yp_serialize_diagnostic_list(parser, &parser->warning_list, buffer); @@ -246,6 +249,7 @@ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffe // Append 0 to mark end of tokens yp_buffer_append_u8(buffer, 0); + yp_serialize_encoding(&parser.encoding, buffer); yp_serialize_comment_list(&parser, &parser.comment_list, buffer); yp_serialize_diagnostic_list(&parser, &parser.error_list, buffer); yp_serialize_diagnostic_list(&parser, &parser.warning_list, buffer); @@ -253,3 +257,26 @@ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffe yp_node_destroy(&parser, node); yp_parser_free(&parser); } + +// Parse and serialize both the AST and the tokens represented by the given +// source to the given buffer. +YP_EXPORTED_FUNCTION void +yp_parse_lex_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata) { + yp_parser_t parser; + yp_parser_init(&parser, source, size, NULL); + if (metadata) yp_parser_metadata(&parser, metadata); + + yp_lex_callback_t lex_callback = (yp_lex_callback_t) { + .data = (void *) buffer, + .callback = serialize_token, + }; + + parser.lex_callback = &lex_callback; + yp_node_t *node = yp_parse(&parser); + + yp_buffer_append_u8(buffer, 0); + yp_serialize(&parser, node, buffer); + + yp_node_destroy(&parser, node); + yp_parser_free(&parser); +} diff --git a/yarp/yarp.c b/yarp/yarp.c index c6b7e1c0190043..69e1c8ebb9d225 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -13482,7 +13482,7 @@ yp_metadata_read_u32(const char *ptr) { // ]* // ] // ``` -static void +void yp_parser_metadata(yp_parser_t *parser, const char *metadata) { uint32_t filepath_size = yp_metadata_read_u32(metadata); metadata += 4; diff --git a/yarp/yarp.h b/yarp/yarp.h index b0879a44545428..1a8a47725a9adf 100644 --- a/yarp/yarp.h +++ b/yarp/yarp.h @@ -31,6 +31,8 @@ void yp_serialize_content(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buf void yp_print_node(yp_parser_t *parser, yp_node_t *node); +void yp_parser_metadata(yp_parser_t *parser, const char *metadata); + // Generate a scope node from the given node. void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest); @@ -69,6 +71,10 @@ YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp // Lex the given source and serialize to the given buffer. YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer); +// Parse and serialize both the AST and the tokens represented by the given +// source to the given buffer. +YP_EXPORTED_FUNCTION void yp_parse_lex_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata); + // Returns a string representation of the given token type. YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type); From 439f069b4b421964dab39b0a01e0afdae89ea26c Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 09:23:46 -0400 Subject: [PATCH 025/208] [ruby/yarp] Move tests from test/* to test/yarp/* to match CRuby This should make it easier on the sync to determine what changed and hopefully result in fewer merge conflicts that have to be manually resolved. https://github.com/ruby/yarp/commit/17d82afbfc --- test/yarp/bom_test.rb | 2 +- test/yarp/comments_test.rb | 2 +- test/{ => yarp}/desugar_visitor_test.rb | 2 +- test/yarp/encoding_test.rb | 2 +- test/yarp/errors_test.rb | 2 +- test/yarp/heredoc_dedent_test.rb | 2 +- test/yarp/library_symbols_test.rb | 2 +- test/yarp/locals_test.rb | 2 +- test/yarp/location_test.rb | 2 +- test/yarp/memsize_test.rb | 2 +- test/yarp/newline_test.rb | 2 +- test/yarp/parse_serialize_test.rb | 2 +- test/yarp/parse_test.rb | 2 +- test/yarp/regexp_test.rb | 2 +- test/yarp/ripper_compat_test.rb | 2 +- test/yarp/ruby_api_test.rb | 2 +- test/yarp/test_helper.rb | 100 ++++++++++++++++++++++++ test/yarp/unescape_test.rb | 2 +- test/yarp/version_test.rb | 2 +- 19 files changed, 118 insertions(+), 18 deletions(-) rename test/{ => yarp}/desugar_visitor_test.rb (99%) create mode 100644 test/yarp/test_helper.rb diff --git a/test/yarp/bom_test.rb b/test/yarp/bom_test.rb index 7dc7eabe92409d..b386a5d9a33b20 100644 --- a/test/yarp/bom_test.rb +++ b/test/yarp/bom_test.rb @@ -4,7 +4,7 @@ # test. return if RUBY_ENGINE == "jruby" || RUBY_ENGINE == "truffleruby" -require "yarp_test_helper" +require_relative "test_helper" class BOMTest < Test::Unit::TestCase def test_ident diff --git a/test/yarp/comments_test.rb b/test/yarp/comments_test.rb index fdb70045ca9462..13bf819a4eb75d 100644 --- a/test/yarp/comments_test.rb +++ b/test/yarp/comments_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class CommentsTest < Test::Unit::TestCase include ::YARP::DSL diff --git a/test/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb similarity index 99% rename from test/desugar_visitor_test.rb rename to test/yarp/desugar_visitor_test.rb index 3af3d9deb43a08..a893422a7c3fa6 100644 --- a/test/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class DesugarVisitorTest < Test::Unit::TestCase def test_and_write diff --git a/test/yarp/encoding_test.rb b/test/yarp/encoding_test.rb index 7bf99ece9270f0..2ee084cd3803ca 100644 --- a/test/yarp/encoding_test.rb +++ b/test/yarp/encoding_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class EncodingTest < Test::Unit::TestCase %w[ diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index 7b20ceadabe0af..074dd7129e6e68 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class ErrorsTest < Test::Unit::TestCase include ::YARP::DSL diff --git a/test/yarp/heredoc_dedent_test.rb b/test/yarp/heredoc_dedent_test.rb index 64627c564f2d68..2744b930ed3f4c 100644 --- a/test/yarp/heredoc_dedent_test.rb +++ b/test/yarp/heredoc_dedent_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" module YARP class HeredocDedentTest < Test::Unit::TestCase diff --git a/test/yarp/library_symbols_test.rb b/test/yarp/library_symbols_test.rb index 766adf16b77cdd..c5927c2a89edea 100644 --- a/test/yarp/library_symbols_test.rb +++ b/test/yarp/library_symbols_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" if RUBY_PLATFORM =~ /linux/ # diff --git a/test/yarp/locals_test.rb b/test/yarp/locals_test.rb index 4d9cea94a80a22..42fb72df789b88 100644 --- a/test/yarp/locals_test.rb +++ b/test/yarp/locals_test.rb @@ -13,7 +13,7 @@ # Ruby is handling large ISeqs on 32-bit machines return if RUBY_PLATFORM =~ /i686/ -require "yarp_test_helper" +require_relative "test_helper" class LocalsTest < Test::Unit::TestCase invalid = [] diff --git a/test/yarp/location_test.rb b/test/yarp/location_test.rb index 424c9778f8b86e..8e357fa193c3c1 100644 --- a/test/yarp/location_test.rb +++ b/test/yarp/location_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" module YARP class LocationTest < Test::Unit::TestCase diff --git a/test/yarp/memsize_test.rb b/test/yarp/memsize_test.rb index 30de1085cf275e..9ff670c11869bd 100644 --- a/test/yarp/memsize_test.rb +++ b/test/yarp/memsize_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI diff --git a/test/yarp/newline_test.rb b/test/yarp/newline_test.rb index 80f6329d94d6c2..2eaaefc61ea9aa 100644 --- a/test/yarp/newline_test.rb +++ b/test/yarp/newline_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return unless defined?(RubyVM::InstructionSequence) diff --git a/test/yarp/parse_serialize_test.rb b/test/yarp/parse_serialize_test.rb index 6789ba9c438499..daecbe1488d2d6 100644 --- a/test/yarp/parse_serialize_test.rb +++ b/test/yarp/parse_serialize_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index 1f8b1374f6d98d..b9f852abf934f4 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class ParseTest < Test::Unit::TestCase # When we pretty-print the trees to compare against the snapshots, we want to diff --git a/test/yarp/regexp_test.rb b/test/yarp/regexp_test.rb index bb236e2f1f9daf..241fcc862f931d 100644 --- a/test/yarp/regexp_test.rb +++ b/test/yarp/regexp_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI diff --git a/test/yarp/ripper_compat_test.rb b/test/yarp/ripper_compat_test.rb index 4350ba7f81f7f3..e13cef08b1fc6c 100644 --- a/test/yarp/ripper_compat_test.rb +++ b/test/yarp/ripper_compat_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" module YARP class RipperCompatTest < Test::Unit::TestCase diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 1f66084ad2a38c..f02026541de7de 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class YARPRubyAPITest < Test::Unit::TestCase def test_ruby_api diff --git a/test/yarp/test_helper.rb b/test/yarp/test_helper.rb new file mode 100644 index 00000000000000..0be0f51651f03b --- /dev/null +++ b/test/yarp/test_helper.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require "yarp" +require "ripper" +require "pp" +require "test/unit" +require "tempfile" + +puts "Using YARP backend: #{YARP::BACKEND}" if ENV["YARP_FFI_BACKEND"] + +module YARP + module Assertions + private + + def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) + assert_equal expected.class, actual.class + + case expected + when Array + assert_equal( + expected.size, + actual.size, + -> { "Arrays were different sizes. Parent: #{parent.pretty_inspect}" } + ) + + expected.zip(actual).each do |(expected_element, actual_element)| + assert_equal_nodes( + expected_element, + actual_element, + compare_location: compare_location, + parent: actual + ) + end + when YARP::SourceFileNode + deconstructed_expected = expected.deconstruct_keys(nil) + deconstructed_actual = actual.deconstruct_keys(nil) + assert_equal deconstructed_expected.keys, deconstructed_actual.keys + + # Filepaths can be different if test suites were run + # on different machines. + # We accommodate for this by comparing the basenames, + # and not the absolute filepaths + assert_equal deconstructed_expected.except(:filepath), deconstructed_actual.except(:filepath) + assert_equal File.basename(deconstructed_expected[:filepath]), File.basename(deconstructed_actual[:filepath]) + when YARP::Node + deconstructed_expected = expected.deconstruct_keys(nil) + deconstructed_actual = actual.deconstruct_keys(nil) + assert_equal deconstructed_expected.keys, deconstructed_actual.keys + + deconstructed_expected.each_key do |key| + assert_equal_nodes( + deconstructed_expected[key], + deconstructed_actual[key], + compare_location: compare_location, + parent: actual + ) + end + when YARP::Location + assert_operator actual.start_offset, :<=, actual.end_offset, -> { + "start_offset > end_offset for #{actual.inspect}, parent is #{parent.pretty_inspect}" + } + if compare_location + assert_equal( + expected.start_offset, + actual.start_offset, + -> { "Start locations were different. Parent: #{parent.pretty_inspect}" } + ) + + assert_equal( + expected.end_offset, + actual.end_offset, + -> { "End locations were different. Parent: #{parent.pretty_inspect}" } + ) + + end + else + assert_equal expected, actual + end + end + + def assert_valid_locations(value, parent: nil) + case value + when Array + value.each do |element| + assert_valid_locations(element, parent: value) + end + when YARP::Node + value.deconstruct_keys(nil).each_value do |field| + assert_valid_locations(field, parent: value) + end + when YARP::Location + assert_operator value.start_offset, :<=, value.end_offset, -> { + "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" + } + end + end + end +end + +Test::Unit::TestCase.include(YARP::Assertions) diff --git a/test/yarp/unescape_test.rb b/test/yarp/unescape_test.rb index 3b9e6652481a92..eef989ad23ab13 100644 --- a/test/yarp/unescape_test.rb +++ b/test/yarp/unescape_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" return if YARP::BACKEND == :FFI diff --git a/test/yarp/version_test.rb b/test/yarp/version_test.rb index f431157ae9264c..aaace0aa89adc1 100644 --- a/test/yarp/version_test.rb +++ b/test/yarp/version_test.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "yarp_test_helper" +require_relative "test_helper" class VersionTest < Test::Unit::TestCase def test_version_is_set From b2d1c720ece67b110ab4ade6b8c9f3b640216c8a Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Sat, 26 Aug 2023 09:59:45 +1200 Subject: [PATCH 026/208] Add documentation regarding how to build Ruby for debugging. (#8290) Co-authored-by: Nobuyoshi Nakada --- doc/contributing/building_ruby.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/contributing/building_ruby.md b/doc/contributing/building_ruby.md index 02efb2384fc47a..b96fa4bc0bd921 100644 --- a/doc/contributing/building_ruby.md +++ b/doc/contributing/building_ruby.md @@ -157,6 +157,14 @@ with the Ruby script you'd like to run. You can use the following make targets: * `make lldb-ruby`: Runs `test.rb` using Ruby in lldb * `make gdb-ruby`: Runs `test.rb` using Ruby in gdb +### Compiling for Debugging + +You should configure Ruby without optimization and other flags that may interfere with debugging: + +``` shell +./configure --enable-debug-env optflags="-O0 -fno-omit-frame-pointer" +``` + ### Building with Address Sanitizer Using the address sanitizer is a great way to detect memory issues. From aea7e9182835af4c9e1857038f03fc37a1474416 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 17:23:15 -0400 Subject: [PATCH 027/208] Fix sync script for YARP --- tool/sync_default_gems.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 29107fb48f317a..1cb58469c86c8e 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -405,7 +405,7 @@ def sync_default_gems(gem) # Run the YARP templating scripts cp_r("#{upstream}/ext/yarp", "yarp") cp_r("#{upstream}/lib/.", "lib") - cp_r("#{upstream}/test", "test/yarp") + cp_r("#{upstream}/test/yarp", "test") cp_r("#{upstream}/src/.", "yarp") cp_r("#{upstream}/yarp.gemspec", "lib/yarp") @@ -415,10 +415,7 @@ def sync_default_gems(gem) cp_r("#{upstream}/config.yml", "yarp/") cp_r("#{upstream}/templates", "yarp/") - rm_f("yarp/config.h") - File.write("yarp/config.h", "#include \"ruby/config.h\"\n") rm("yarp/extconf.rb") - mv("yarp_init.c", "yarp/") else sync_lib gem, upstream From ca9a44795b98f67cc3b51a59317c24dadb602bb5 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 17:54:01 -0400 Subject: [PATCH 028/208] Remove version templating in YARP --- lib/yarp.rb | 1 - lib/yarp/ffi.rb | 6 +- lib/yarp/yarp.gemspec | 11 +- test/yarp/yarp_test_helper.rb | 100 ------------------- yarp/config.h | 1 - yarp/extension.c | 7 +- yarp/extension.h | 2 + yarp/templates/include/yarp/version.h.erb | 9 -- yarp/templates/java/org/yarp/Loader.java.erb | 6 +- yarp/templates/lib/yarp/serialize.rb.erb | 8 +- yarp/templates/template.rb | 12 --- yarp/yarp.c | 1 - 12 files changed, 18 insertions(+), 146 deletions(-) delete mode 100644 test/yarp/yarp_test_helper.rb delete mode 100644 yarp/config.h delete mode 100644 yarp/templates/include/yarp/version.h.erb diff --git a/lib/yarp.rb b/lib/yarp.rb index d44f80b9a05b72..a40f27009178f7 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -533,7 +533,6 @@ def self.parse_serialize_file(filepath) require_relative "yarp/ripper_compat" require_relative "yarp/serialize" require_relative "yarp/pack" -require_relative "yarp/version" if RUBY_ENGINE == "ruby" and !ENV["YARP_FFI_BACKEND"] require "yarp/yarp" diff --git a/lib/yarp/ffi.rb b/lib/yarp/ffi.rb index 73e5d60dfac237..26b6019b27f894 100644 --- a/lib/yarp/ffi.rb +++ b/lib/yarp/ffi.rb @@ -172,10 +172,8 @@ def self.with(filepath, &block) # the YARP module. private_constant :LibRubyParser - library_version = LibRubyParser.yp_version.read_string - if library_version != YARP::VERSION - raise "The YARP library version (#{library_version}) does not match the expected version (#{YARP::VERSION})" - end + # The version constant is set by reading the result of calling yp_version. + VERSION = LibRubyParser.yp_version.read_string def self.dump_internal(source, source_size, filepath) LibRubyParser::YPBuffer.with do |buffer| diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index b04f893d046414..fceda5b1a0ac86 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -1,16 +1,8 @@ # frozen_string_literal: true -if File.exist?(File.expand_path("version.rb", __dir__)) - # CRuby - require_relative "version" -else - # Within the gem/local repository - require_relative "lib/yarp/version" -end - Gem::Specification.new do |spec| spec.name = "yarp" - spec.version = YARP::VERSION + spec.version = "0.8.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -75,7 +67,6 @@ Gem::Specification.new do |spec| "lib/yarp/pack.rb", "lib/yarp/ripper_compat.rb", "lib/yarp/serialize.rb", - "lib/yarp/version.rb", "src/diagnostic.c", "src/enc/yp_big5.c", "src/enc/yp_euc_jp.c", diff --git a/test/yarp/yarp_test_helper.rb b/test/yarp/yarp_test_helper.rb deleted file mode 100644 index 0be0f51651f03b..00000000000000 --- a/test/yarp/yarp_test_helper.rb +++ /dev/null @@ -1,100 +0,0 @@ -# frozen_string_literal: true - -require "yarp" -require "ripper" -require "pp" -require "test/unit" -require "tempfile" - -puts "Using YARP backend: #{YARP::BACKEND}" if ENV["YARP_FFI_BACKEND"] - -module YARP - module Assertions - private - - def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) - assert_equal expected.class, actual.class - - case expected - when Array - assert_equal( - expected.size, - actual.size, - -> { "Arrays were different sizes. Parent: #{parent.pretty_inspect}" } - ) - - expected.zip(actual).each do |(expected_element, actual_element)| - assert_equal_nodes( - expected_element, - actual_element, - compare_location: compare_location, - parent: actual - ) - end - when YARP::SourceFileNode - deconstructed_expected = expected.deconstruct_keys(nil) - deconstructed_actual = actual.deconstruct_keys(nil) - assert_equal deconstructed_expected.keys, deconstructed_actual.keys - - # Filepaths can be different if test suites were run - # on different machines. - # We accommodate for this by comparing the basenames, - # and not the absolute filepaths - assert_equal deconstructed_expected.except(:filepath), deconstructed_actual.except(:filepath) - assert_equal File.basename(deconstructed_expected[:filepath]), File.basename(deconstructed_actual[:filepath]) - when YARP::Node - deconstructed_expected = expected.deconstruct_keys(nil) - deconstructed_actual = actual.deconstruct_keys(nil) - assert_equal deconstructed_expected.keys, deconstructed_actual.keys - - deconstructed_expected.each_key do |key| - assert_equal_nodes( - deconstructed_expected[key], - deconstructed_actual[key], - compare_location: compare_location, - parent: actual - ) - end - when YARP::Location - assert_operator actual.start_offset, :<=, actual.end_offset, -> { - "start_offset > end_offset for #{actual.inspect}, parent is #{parent.pretty_inspect}" - } - if compare_location - assert_equal( - expected.start_offset, - actual.start_offset, - -> { "Start locations were different. Parent: #{parent.pretty_inspect}" } - ) - - assert_equal( - expected.end_offset, - actual.end_offset, - -> { "End locations were different. Parent: #{parent.pretty_inspect}" } - ) - - end - else - assert_equal expected, actual - end - end - - def assert_valid_locations(value, parent: nil) - case value - when Array - value.each do |element| - assert_valid_locations(element, parent: value) - end - when YARP::Node - value.deconstruct_keys(nil).each_value do |field| - assert_valid_locations(field, parent: value) - end - when YARP::Location - assert_operator value.start_offset, :<=, value.end_offset, -> { - "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" - } - end - end - end -end - -Test::Unit::TestCase.include(YARP::Assertions) diff --git a/yarp/config.h b/yarp/config.h deleted file mode 100644 index 5a5987fa4499f5..00000000000000 --- a/yarp/config.h +++ /dev/null @@ -1 +0,0 @@ -#include "ruby/config.h" diff --git a/yarp/extension.c b/yarp/extension.c index b59ccc1bcad942..b46a1f8226a2d8 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -545,12 +545,12 @@ RUBY_FUNC_EXPORTED void Init_yarp(void) { // Make sure that the YARP library version matches the expected version. // Otherwise something was compiled incorrectly. - if (strcmp(yp_version(), YP_VERSION) != 0) { + if (strcmp(yp_version(), EXPECTED_YARP_VERSION) != 0) { rb_raise( rb_eRuntimeError, "The YARP library version (%s) does not match the expected version (%s)", yp_version(), - YP_VERSION + EXPECTED_YARP_VERSION ); } @@ -566,6 +566,9 @@ Init_yarp(void) { rb_cYARPParseWarning = rb_define_class_under(rb_cYARP, "ParseWarning", rb_cObject); rb_cYARPParseResult = rb_define_class_under(rb_cYARP, "ParseResult", rb_cObject); + // Define the version string here so that we can use the constants defined + // in yarp.h. + rb_define_const(rb_cYARP, "VERSION", rb_str_new2(EXPECTED_YARP_VERSION)); rb_define_const(rb_cYARP, "BACKEND", ID2SYM(rb_intern("CExtension"))); // First, the functions that have to do with lexing and parsing. diff --git a/yarp/extension.h b/yarp/extension.h index 2796060fb1a085..fe004d16da5cc8 100644 --- a/yarp/extension.h +++ b/yarp/extension.h @@ -1,6 +1,8 @@ #ifndef YARP_EXT_NODE_H #define YARP_EXT_NODE_H +#define EXPECTED_YARP_VERSION "0.8.0" + #include #include #include "yarp.h" diff --git a/yarp/templates/include/yarp/version.h.erb b/yarp/templates/include/yarp/version.h.erb deleted file mode 100644 index 4ebcd69b5fc243..00000000000000 --- a/yarp/templates/include/yarp/version.h.erb +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef YARP_VERSION_H -#define YARP_VERSION_H - -#define YP_VERSION_MAJOR <%= YARP_VERSION_MAJOR %> -#define YP_VERSION_MINOR <%= YARP_VERSION_MINOR %> -#define YP_VERSION_PATCH <%= YARP_VERSION_PATCH %> -#define YP_VERSION "<%= YARP_VERSION %>" - -#endif // YARP_VERSION_H diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb index 3e7dcebafb2a5c..e3e29e446b84db 100644 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ b/yarp/templates/java/org/yarp/Loader.java.erb @@ -59,9 +59,9 @@ public class Loader { expect((byte) 'R'); expect((byte) 'P'); - expect((byte) <%= YARP_VERSION_MAJOR %>); - expect((byte) <%= YARP_VERSION_MINOR %>); - expect((byte) <%= YARP_VERSION_PATCH %>); + expect((byte) 0); + expect((byte) 8); + expect((byte) 0); // This loads the name of the encoding. We don't actually do anything // with it just yet. diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index ee9285fdbc5732..44ca3146fb385b 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -13,6 +13,10 @@ end module YARP module Serialize + MAJOR_VERSION = 0 + MINOR_VERSION = 8 + PATCH_VERSION = 0 + def self.load(input, serialized) Loader.new(Source.new(input), serialized).load_result end @@ -78,9 +82,7 @@ module YARP def load_nodes raise "Invalid serialization" if io.read(4) != "YARP" - if io.read(3).unpack("C3") != [<%= YARP_VERSION_MAJOR %>, <%= YARP_VERSION_MINOR %>, <%= YARP_VERSION_PATCH %>] - raise "Invalid serialization version" - end + raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] @encoding = load_encoding @input = input.force_encoding(@encoding).freeze diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index 944cb7edc547d4..e34d5b1a7be6ea 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -4,17 +4,6 @@ require "fileutils" require "yaml" -if File.exist?(File.expand_path("../lib/yarp/version.rb", __dir__)) - # Within the gem/local repository - require_relative "../lib/yarp/version" -else - # Within CRuby - require_relative "../../lib/yarp/version" -end - -YARP_VERSION = YARP::VERSION -YARP_VERSION_MAJOR, YARP_VERSION_MINOR, YARP_VERSION_PATCH = YARP_VERSION.split(".") - COMMON_FLAGS = 1 class Param @@ -323,7 +312,6 @@ def locals TEMPLATES = [ "ext/yarp/api_node.c", "include/yarp/ast.h", - "include/yarp/version.h", "java/org/yarp/Loader.java", "java/org/yarp/Nodes.java", "java/org/yarp/AbstractNodeVisitor.java", diff --git a/yarp/yarp.c b/yarp/yarp.c index 69e1c8ebb9d225..8758d365843b12 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -1,5 +1,4 @@ #include "yarp.h" -#include "yarp/version.h" // The YARP version and the serialization format. const char * From 293959a7452cb69d4c80e94b243d84b4de702707 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 18:06:38 -0400 Subject: [PATCH 029/208] Remove config.h from targets necessary for YARP --- .gitignore | 1 - common.mk | 57 +------------------------------------------------- yarp/version.h | 4 ++++ 3 files changed, 5 insertions(+), 57 deletions(-) create mode 100644 yarp/version.h diff --git a/.gitignore b/.gitignore index cb7c533bac1ac7..631a489eeff1f8 100644 --- a/.gitignore +++ b/.gitignore @@ -265,4 +265,3 @@ lcov*.info /yarp/prettyprint.c /yarp/serialize.c /yarp/token_type.c -/yarp/version.h diff --git a/common.mk b/common.mk index c193aaae493c1e..b569b7ebaca5df 100644 --- a/common.mk +++ b/common.mk @@ -225,10 +225,6 @@ srcs: yarp/ast.h yarp/ast.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/ast.h.erb $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/ast.h $@ -srcs: yarp/version.h -yarp/version.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/version.h.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/version.h $@ - srcs: yarp/node.c yarp/node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/node.c.erb $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/node.c $@ @@ -18862,7 +18858,6 @@ yarp/api_node.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_node.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/api_node.c yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19053,7 +19048,6 @@ yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/api_pack.c yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19240,143 +19234,115 @@ yarp/api_pack.$(OBJEXT): {$(VPATH)}onigmo.h yarp/api_pack.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/api_pack.$(OBJEXT): {$(VPATH)}st.h yarp/api_pack.$(OBJEXT): {$(VPATH)}subst.h -yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.c yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h yarp/diagnostic.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_ascii.c yarp/enc/yp_ascii.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_ascii.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_big5.c yarp/enc/yp_big5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_big5.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_euc_jp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_euc_jp.c yarp/enc/yp_euc_jp.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_gbk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_gbk.c yarp/enc/yp_gbk.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_1.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_1.c yarp/enc/yp_iso_8859_1.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_10.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_10.c yarp/enc/yp_iso_8859_10.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_11.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_11.c yarp/enc/yp_iso_8859_11.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_13.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_13.c yarp/enc/yp_iso_8859_13.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_14.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_14.c yarp/enc/yp_iso_8859_14.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_15.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_15.c yarp/enc/yp_iso_8859_15.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_16.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_16.c yarp/enc/yp_iso_8859_16.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_2.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_2.c yarp/enc/yp_iso_8859_2.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_3.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_3.c yarp/enc/yp_iso_8859_3.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_4.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_4.c yarp/enc/yp_iso_8859_4.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_5.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_5.c yarp/enc/yp_iso_8859_5.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_6.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_6.c yarp/enc/yp_iso_8859_6.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_7.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_7.c yarp/enc/yp_iso_8859_7.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_8.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_8.c yarp/enc/yp_iso_8859_8.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_iso_8859_9.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_iso_8859_9.c yarp/enc/yp_iso_8859_9.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_koi8_r.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_koi8_r.c yarp/enc/yp_koi8_r.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_shared.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_shared.c yarp/enc/yp_shared.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_shift_jis.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_shift_jis.c yarp/enc/yp_shift_jis.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_tables.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_tables.c yarp/enc/yp_tables.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_unicode.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_unicode.c yarp/enc/yp_unicode.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_windows_1251.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_1251.c yarp/enc/yp_windows_1251.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_windows_1252.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_1252.c yarp/enc/yp_windows_1252.$(OBJEXT): {$(VPATH)}config.h -yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_31j.c @@ -19384,7 +19350,6 @@ yarp/enc/yp_windows_31j.$(OBJEXT): {$(VPATH)}config.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/extension.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19573,7 +19538,6 @@ yarp/extension.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/extension.$(OBJEXT): {$(VPATH)}st.h yarp/extension.$(OBJEXT): {$(VPATH)}subst.h yarp/node.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/node.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19594,13 +19558,11 @@ yarp/node.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/node.$(OBJEXT): {$(VPATH)}config.h -yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.c yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/pack.$(OBJEXT): {$(VPATH)}config.h yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19613,7 +19575,6 @@ yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/prettyprint.$(OBJEXT): {$(VPATH)}config.h yarp/regexp.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19628,7 +19589,6 @@ yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/regexp.$(OBJEXT): {$(VPATH)}config.h yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19651,14 +19611,12 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/serialize.$(OBJEXT): {$(VPATH)}config.h yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/token_type.c yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/token_type.$(OBJEXT): {$(VPATH)}config.h yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19680,29 +19638,24 @@ yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/unescape.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.c yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/util/yp_buffer.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.c yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h yarp/util/yp_char.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_char.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.c yarp/util/yp_constant_pool.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/util/yp_constant_pool.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.c yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h yarp/util/yp_list.$(OBJEXT): {$(VPATH)}config.h yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19714,32 +19667,26 @@ yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.c yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_newline_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.c yarp/util/yp_state_stack.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/util/yp_state_stack.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.c yarp/util/yp_string.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_string.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.c yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/util/yp_string_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strncasecmp.c yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19752,7 +19699,6 @@ yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.c yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}config.h yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19771,14 +19717,13 @@ yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h -yarp/yarp.$(OBJEXT): {$(VPATH)}/yarp/version.h +yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/version.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.c yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/yarp.$(OBJEXT): {$(VPATH)}config.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/ast.h -yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/config.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h diff --git a/yarp/version.h b/yarp/version.h new file mode 100644 index 00000000000000..179543f54d751b --- /dev/null +++ b/yarp/version.h @@ -0,0 +1,4 @@ +#define YP_VERSION_MAJOR 0 +#define YP_VERSION_MINOR 8 +#define YP_VERSION_PATCH 0 +#define YP_VERSION "0.8.0" From 00ec8dedb1fc0aff44ba3a37271ed30751cccadf Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 25 Aug 2023 18:19:35 -0400 Subject: [PATCH 030/208] Update deps --- common.mk | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/common.mk b/common.mk index b569b7ebaca5df..c0d3e0a673471e 100644 --- a/common.mk +++ b/common.mk @@ -18856,8 +18856,6 @@ weakmap.$(OBJEXT): {$(VPATH)}vm_opts.h weakmap.$(OBJEXT): {$(VPATH)}weakmap.c yarp/api_node.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_node.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/api_node.c -yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/api_node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19044,10 +19042,12 @@ yarp/api_node.$(OBJEXT): {$(VPATH)}onigmo.h yarp/api_node.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/api_node.$(OBJEXT): {$(VPATH)}st.h yarp/api_node.$(OBJEXT): {$(VPATH)}subst.h +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/api_node.c +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/api_node.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby.h yarp/api_pack.$(OBJEXT): $(hdrdir)/ruby/ruby.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/api_pack.c -yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/api_pack.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19234,6 +19234,8 @@ yarp/api_pack.$(OBJEXT): {$(VPATH)}onigmo.h yarp/api_pack.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/api_pack.$(OBJEXT): {$(VPATH)}st.h yarp/api_pack.$(OBJEXT): {$(VPATH)}subst.h +yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/api_pack.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.c yarp/diagnostic.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h @@ -19349,7 +19351,6 @@ yarp/enc/yp_windows_31j.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_windows_31j.c yarp/enc/yp_windows_31j.$(OBJEXT): {$(VPATH)}config.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby.h yarp/extension.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/extension.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/extension.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19537,11 +19538,11 @@ yarp/extension.$(OBJEXT): {$(VPATH)}onigmo.h yarp/extension.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/extension.$(OBJEXT): {$(VPATH)}st.h yarp/extension.$(OBJEXT): {$(VPATH)}subst.h -yarp/node.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/extension.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/extension.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h -yarp/node.$(OBJEXT): {$(VPATH)}yarp/node.c yarp/node.$(OBJEXT): $(top_srcdir)/yarp/node.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19558,15 +19559,15 @@ yarp/node.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/node.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/node.$(OBJEXT): {$(VPATH)}config.h +yarp/node.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/node.$(OBJEXT): {$(VPATH)}yarp/node.c yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.c yarp/pack.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/pack.$(OBJEXT): {$(VPATH)}config.h -yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/parser.h -yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/prettyprint.c yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h @@ -19574,7 +19575,8 @@ yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/prettyprint.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/prettyprint.$(OBJEXT): {$(VPATH)}config.h -yarp/regexp.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/prettyprint.$(OBJEXT): {$(VPATH)}yarp/prettyprint.c yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19588,7 +19590,7 @@ yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/regexp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/regexp.$(OBJEXT): {$(VPATH)}config.h -yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/regexp.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19596,7 +19598,6 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/node.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/pack.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/parser.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/regexp.h -yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/serialize.c yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/unescape.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h @@ -19610,13 +19611,15 @@ yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/serialize.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/serialize.$(OBJEXT): {$(VPATH)}config.h -yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/serialize.c +yarp/serialize.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/defines.h -yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/token_type.c yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h yarp/token_type.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/token_type.$(OBJEXT): {$(VPATH)}config.h -yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/token_type.$(OBJEXT): {$(VPATH)}yarp/token_type.c yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19638,6 +19641,8 @@ yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/unescape.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/unescape.$(OBJEXT): {$(VPATH)}config.h +yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/unescape.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.c yarp/util/yp_buffer.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h @@ -19655,7 +19660,6 @@ yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.c yarp/util/yp_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h yarp/util/yp_list.$(OBJEXT): {$(VPATH)}config.h -yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19667,6 +19671,7 @@ yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h yarp/util/yp_memchr.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}config.h +yarp/util/yp_memchr.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.c yarp/util/yp_newline_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h @@ -19686,7 +19691,6 @@ yarp/util/yp_string_list.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h yarp/util/yp_string_list.$(OBJEXT): {$(VPATH)}config.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strncasecmp.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strncasecmp.c -yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/parser.h @@ -19698,7 +19702,7 @@ yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.c yarp/util/yp_strpbrk.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}config.h -yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/util/yp_strpbrk.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19721,9 +19725,10 @@ yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/version.h yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.c yarp/yarp.$(OBJEXT): $(top_srcdir)/yarp/yarp.h yarp/yarp.$(OBJEXT): {$(VPATH)}config.h +yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/yarp.$(OBJEXT): {$(VPATH)}yarp/version.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby.h yarp/yarp_init.$(OBJEXT): $(hdrdir)/ruby/ruby.h -yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/ast.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/defines.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h yarp/yarp_init.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h @@ -19911,6 +19916,8 @@ yarp/yarp_init.$(OBJEXT): {$(VPATH)}onigmo.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}oniguruma.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}st.h yarp/yarp_init.$(OBJEXT): {$(VPATH)}subst.h +yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/ast.h +yarp/yarp_init.$(OBJEXT): {$(VPATH)}yarp/version.h yjit.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h yjit.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h yjit.$(OBJEXT): $(CCAN_DIR)/list/list.h From 480aee4363d2bff26dcdd6b9280cf93ec0f1a06b Mon Sep 17 00:00:00 2001 From: Zack Deveau Date: Fri, 25 Aug 2023 18:27:56 -0400 Subject: [PATCH 031/208] Add Missing Counters to `rb_debug_counter_type` enum (#8297) Add missing counters to rb_debug_counter_type enum On master we have calls to the RB_DEBUG_COUNTER_INC macro for counters that are not getting defined in the rb_debug_counter_type enum. This commit adds those that are missing in order for compilation to pass with -DUSE_RUBY_DEBUG_LOG. --- debug_counter.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debug_counter.h b/debug_counter.h index 01b1a63f86312d..a8b95edded258b 100644 --- a/debug_counter.h +++ b/debug_counter.h @@ -100,6 +100,13 @@ RB_DEBUG_COUNTER(ccf_opt_block_call) RB_DEBUG_COUNTER(ccf_opt_struct_aref) RB_DEBUG_COUNTER(ccf_opt_struct_aset) RB_DEBUG_COUNTER(ccf_super_method) +RB_DEBUG_COUNTER(ccf_cfunc_other) +RB_DEBUG_COUNTER(ccf_cfunc_only_splat) +RB_DEBUG_COUNTER(ccf_cfunc_only_splat_kw) +RB_DEBUG_COUNTER(ccf_iseq_bmethod) +RB_DEBUG_COUNTER(ccf_noniseq_bmethod) +RB_DEBUG_COUNTER(ccf_opt_send_complex) +RB_DEBUG_COUNTER(ccf_opt_send_simple) /* * control frame push counts. From 96a809f621b742b6256545e85519002d0fd28a44 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 25 Aug 2023 15:32:13 -0700 Subject: [PATCH 032/208] Use require_relative in JSON tests to prevent them from conflicting with yarp/test_helper --- test/json/json_addition_test.rb | 2 +- test/json/json_common_interface_test.rb | 2 +- test/json/json_encoding_test.rb | 2 +- test/json/json_ext_parser_test.rb | 2 +- test/json/json_fixtures_test.rb | 2 +- test/json/json_generator_test.rb | 2 +- test/json/json_generic_object_test.rb | 2 +- test/json/json_parser_test.rb | 2 +- test/json/json_string_matching_test.rb | 2 +- test/json/ractor_test.rb | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/json/json_addition_test.rb b/test/json/json_addition_test.rb index d5dab78fed2bda..2877bef7d8e16d 100644 --- a/test/json/json_addition_test.rb +++ b/test/json/json_addition_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'json/add/core' require 'json/add/complex' require 'json/add/rational' diff --git a/test/json/json_common_interface_test.rb b/test/json/json_common_interface_test.rb index 9148b78c8ba13d..39d35fa7729346 100644 --- a/test/json/json_common_interface_test.rb +++ b/test/json/json_common_interface_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'stringio' require 'tempfile' diff --git a/test/json/json_encoding_test.rb b/test/json/json_encoding_test.rb index cc7b71553a3e02..7e7e5341b02cb6 100644 --- a/test/json/json_encoding_test.rb +++ b/test/json/json_encoding_test.rb @@ -1,6 +1,6 @@ # encoding: utf-8 #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONEncodingTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_ext_parser_test.rb b/test/json/json_ext_parser_test.rb index c5a030ea8a7bad..b5b18fb20b7086 100644 --- a/test/json/json_ext_parser_test.rb +++ b/test/json/json_ext_parser_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONExtParserTest < Test::Unit::TestCase if defined?(JSON::Ext::Parser) diff --git a/test/json/json_fixtures_test.rb b/test/json/json_fixtures_test.rb index 845abb48673ca7..acc87499654802 100644 --- a/test/json/json_fixtures_test.rb +++ b/test/json/json_fixtures_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONFixturesTest < Test::Unit::TestCase def setup diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index f71999b12c0560..fa15279905861d 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -2,7 +2,7 @@ # encoding: utf-8 # frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONGeneratorTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_generic_object_test.rb b/test/json/json_generic_object_test.rb index 82742dcd638967..c4d391208cc965 100644 --- a/test/json/json_generic_object_test.rb +++ b/test/json/json_generic_object_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONGenericObjectTest < Test::Unit::TestCase include JSON diff --git a/test/json/json_parser_test.rb b/test/json/json_parser_test.rb index 802408b2cb78ff..220826a426b1f2 100644 --- a/test/json/json_parser_test.rb +++ b/test/json/json_parser_test.rb @@ -1,6 +1,6 @@ # encoding: utf-8 # frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'stringio' require 'tempfile' require 'ostruct' diff --git a/test/json/json_string_matching_test.rb b/test/json/json_string_matching_test.rb index 5d55dc31b0c4e0..b9cf904aaaf7ab 100644 --- a/test/json/json_string_matching_test.rb +++ b/test/json/json_string_matching_test.rb @@ -1,5 +1,5 @@ #frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' require 'time' class JSONStringMatchingTest < Test::Unit::TestCase diff --git a/test/json/ractor_test.rb b/test/json/ractor_test.rb index 71105e55ecd22d..cca21b20f0b798 100644 --- a/test/json/ractor_test.rb +++ b/test/json/ractor_test.rb @@ -1,7 +1,7 @@ # encoding: utf-8 # frozen_string_literal: false -require 'test_helper' +require_relative 'test_helper' class JSONInRactorTest < Test::Unit::TestCase def test_generate From cfdbbd67268a77177e485263cdd8fb416315e9a6 Mon Sep 17 00:00:00 2001 From: yui-knk Date: Thu, 24 Aug 2023 21:58:51 +0900 Subject: [PATCH 033/208] kw_rest_arg nd_cflag has not been used since 9720136 --- parse.y | 1 - 1 file changed, 1 deletion(-) diff --git a/parse.y b/parse.y index ba146b7439ece5..ab416540eace4b 100644 --- a/parse.y +++ b/parse.y @@ -12737,7 +12737,6 @@ new_args_tail(struct parser_params *p, NODE *kw_args, ID kw_rest_arg, ID block, if (block) arg_var(p, block); args->kw_rest_arg = NEW_DVAR(kw_rest_arg, kw_rest_loc); - args->kw_rest_arg->nd_cflag = kw_bits; } else if (kw_rest_arg == idNil) { args->no_kwarg = 1; From 00ac3a64ba57ecd8f10bf54f03297cdec0c538d6 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 24 Aug 2023 18:32:46 +0900 Subject: [PATCH 034/208] Introduce `at_char_boundary` function --- file.c | 2 +- internal/string.h | 6 ++++++ io.c | 3 +-- string.c | 9 ++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/file.c b/file.c index e753e116de68f7..30dd1cc73b164a 100644 --- a/file.c +++ b/file.c @@ -4610,7 +4610,7 @@ rmext(const char *p, long l0, long l1, const char *e, long l2, rb_encoding *enc) if (l1 < l2) return l1; s = p+l1-l2; - if (rb_enc_left_char_head(p, s, p+l1, enc) != s) return 0; + if (!at_char_boundary(p, s, p+l1, enc)) return 0; #if CASEFOLD_FILESYSTEM #define fncomp strncasecmp #else diff --git a/internal/string.h b/internal/string.h index 5f59d9621b6b81..cfaf628e02140d 100644 --- a/internal/string.h +++ b/internal/string.h @@ -119,6 +119,12 @@ is_broken_string(VALUE str) return rb_enc_str_coderange(str) == ENC_CODERANGE_BROKEN; } +static inline bool +at_char_boundary(const char *s, const char *p, const char *e, rb_encoding *enc) +{ + return rb_enc_left_char_head(s, p, e, enc) == p; +} + /* expect tail call optimization */ // YJIT needs this function to never allocate and never raise static inline VALUE diff --git a/io.c b/io.c index 6e27ed6273ae35..bd1db9aa5d675d 100644 --- a/io.c +++ b/io.c @@ -4144,8 +4144,7 @@ rb_io_getline_0(VALUE rs, long limit, int chomp, rb_io_t *fptr) s = RSTRING_PTR(str); e = RSTRING_END(str); p = e - rslen; - pp = rb_enc_left_char_head(s, p, e, enc); - if (pp != p) continue; + if (!at_char_boundary(s, p, e, enc)) continue; if (!rspara) rscheck(rsptr, rslen, rs); if (memcmp(p, rsptr, rslen) == 0) { if (chomp) { diff --git a/string.c b/string.c index 0b9ede4e2ced40..b65868a226b13e 100644 --- a/string.c +++ b/string.c @@ -3930,8 +3930,7 @@ str_ensure_byte_pos(VALUE str, long pos) const char *s = RSTRING_PTR(str); const char *e = RSTRING_END(str); const char *p = s + pos; - const char *pp = rb_enc_left_char_head(s, p, e, rb_enc_get(str)); - if (p != pp) { + if (!at_char_boundary(s, p, e, rb_enc_get(str))) { rb_raise(rb_eIndexError, "offset %ld does not land on character boundary", pos); } @@ -9521,7 +9520,7 @@ chompped_length(VALUE str, VALUE rs) if (p[len-1] == newline && (rslen <= 1 || memcmp(rsptr, pp, rslen) == 0)) { - if (rb_enc_left_char_head(p, pp, e, enc) == pp) + if (at_char_boundary(p, pp, e, enc)) return len - rslen; RB_GC_GUARD(rs); } @@ -10497,7 +10496,7 @@ rb_str_end_with(int argc, VALUE *argv, VALUE str) p = RSTRING_PTR(str); e = p + slen; s = e - tlen; - if (rb_enc_left_char_head(p, s, e, enc) != s) + if (!at_char_boundary(p, s, e, enc)) continue; if (memcmp(s, RSTRING_PTR(tmp), RSTRING_LEN(tmp)) == 0) return Qtrue; @@ -10605,7 +10604,7 @@ deleted_suffix_length(VALUE str, VALUE suffix) suffixptr = RSTRING_PTR(suffix); s = strptr + olen - suffixlen; if (memcmp(s, suffixptr, suffixlen) != 0) return 0; - if (rb_enc_left_char_head(strptr, s, strptr + olen, enc) != s) return 0; + if (!at_char_boundary(strptr, s, strptr + olen, enc)) return 0; return suffixlen; } From 808b06708884bf928b2e9c23ed5dcbbdf6665972 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 25 Aug 2023 00:49:03 +0900 Subject: [PATCH 035/208] Split string tests --- test/ruby/test_string.rb | 64 ++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 6cc958332f1679..e69c66be408806 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -1941,7 +1941,9 @@ def test_start_with? bug5536 = '[ruby-core:40623]' assert_raise(TypeError, bug5536) {S("str").start_with? :not_convertible_to_string} + end + def test_start_with_regexp assert_equal(true, S("hello").start_with?(/hel/)) assert_equal("hel", $&) assert_equal(false, S("hello").start_with?(/el/)) @@ -2891,11 +2893,13 @@ def test_lstrip_bang end - def test_delete_prefix + def test_delete_prefix_type_error assert_raise(TypeError) { S('hello').delete_prefix(nil) } assert_raise(TypeError) { S('hello').delete_prefix(1) } assert_raise(TypeError) { S('hello').delete_prefix(/hel/) } + end + def test_delete_prefix s = S("hello") assert_equal("lo", s.delete_prefix('hel')) assert_equal("hello", s) @@ -2915,8 +2919,9 @@ def test_delete_prefix s = S("hello") assert_equal("hello", s.delete_prefix("\u{3053 3093}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_prefix_broken_encoding s = S("\xe3\x81\x82") assert_equal("\xe3\x81\x82", s.delete_prefix("\xe3")) assert_equal("\xe3\x81\x82", s) @@ -2925,23 +2930,28 @@ def test_delete_prefix assert_equal("\x95\x5c".force_encoding("Shift_JIS"), s.delete_prefix("\x95")) assert_equal("\x95\x5c".force_encoding("Shift_JIS"), s) - # clear coderange + end + + def test_delete_prefix_clear_coderange s = S("\u{3053 3093}hello") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_prefix("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_prefix_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("bba", s.delete_prefix(klass.new)) assert_equal("abba", s) end - def test_delete_prefix_bang + def test_delete_prefix_bang_type_error assert_raise(TypeError) { S('hello').delete_prefix!(nil) } assert_raise(TypeError) { S('hello').delete_prefix!(1) } assert_raise(TypeError) { S('hello').delete_prefix!(/hel/) } + end + def test_delete_prefix_bang s = S("hello") assert_equal("lo", s.delete_prefix!('hel')) assert_equal("lo", s) @@ -2961,23 +2971,29 @@ def test_delete_prefix_bang s = S("hello") assert_equal(nil, s.delete_prefix!("\u{3053 3093}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_prefix_bang_broken_encoding s = S("\xe3\x81\x82") assert_equal(nil, s.delete_prefix!("\xe3")) assert_equal("\xe3\x81\x82", s) - # clear coderange + end + + def test_delete_prefix_bang_clear_coderange s = S("\u{3053 3093}hello") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_prefix!("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_prefix_bang_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("bba", s.delete_prefix!(klass.new)) assert_equal("bba", s) + end + def test_delete_prefix_bang_frozen_error s = S("ax").freeze assert_raise_with_message(FrozenError, /frozen/) {s.delete_prefix!("a")} @@ -2990,11 +3006,13 @@ def o.to_str assert_raise_with_message(FrozenError, /frozen/) {s.delete_prefix!(o)} end - def test_delete_suffix + def test_delete_suffix_type_error assert_raise(TypeError) { S('hello').delete_suffix(nil) } assert_raise(TypeError) { S('hello').delete_suffix(1) } assert_raise(TypeError) { S('hello').delete_suffix(/hel/) } + end + def test_delete_suffix s = S("hello") assert_equal("hel", s.delete_suffix('lo')) assert_equal("hello", s) @@ -3014,23 +3032,28 @@ def test_delete_suffix s = S("hello") assert_equal("hello", s.delete_suffix("\u{3061 306f}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_suffix_broken_encoding s = S("\xe3\x81\x82") assert_equal("\xe3\x81\x82", s.delete_suffix("\x82")) assert_equal("\xe3\x81\x82", s) + end - # clear coderange + def test_delete_suffix_clear_coderange s = S("hello\u{3053 3093}") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_suffix("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_suffix_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("abb", s.delete_suffix(klass.new)) assert_equal("abba", s) + end + def test_delete_suffix_newline # chomp removes any of "\n", "\r\n", "\r" when "\n" is specified, # but delete_suffix does not s = "foo\n" @@ -3041,11 +3064,13 @@ def test_delete_suffix assert_equal("foo\r", s.delete_suffix("\n")) end - def test_delete_suffix_bang + def test_delete_suffix_bang_type_error assert_raise(TypeError) { S('hello').delete_suffix!(nil) } assert_raise(TypeError) { S('hello').delete_suffix!(1) } assert_raise(TypeError) { S('hello').delete_suffix!(/hel/) } + end + def test_delete_suffix_bang_frozen_error s = S("hello").freeze assert_raise_with_message(FrozenError, /frozen/) {s.delete_suffix!('lo')} @@ -3056,7 +3081,9 @@ def o.to_str "x" end assert_raise_with_message(FrozenError, /frozen/) {s.delete_suffix!(o)} + end + def test_delete_suffix_bang s = S("hello") assert_equal("hel", s.delete_suffix!('lo')) assert_equal("hel", s) @@ -3076,8 +3103,9 @@ def o.to_str s = S("hello") assert_equal(nil, s.delete_suffix!("\u{3061 306f}")) assert_equal("hello", s) + end - # skip if argument is a broken string + def test_delete_suffix_bang_broken_encoding s = S("\xe3\x81\x82") assert_equal(nil, s.delete_suffix!("\x82")) assert_equal("\xe3\x81\x82", s) @@ -3085,18 +3113,22 @@ def o.to_str s = S("\x95\x5c").force_encoding("Shift_JIS") assert_equal(nil, s.delete_suffix!("\x5c")) assert_equal("\x95\x5c".force_encoding("Shift_JIS"), s) + end - # clear coderange + def test_delete_suffix_bang_clear_coderange s = S("hello\u{3053 3093}") assert_not_predicate(s, :ascii_only?) assert_predicate(s.delete_suffix!("\u{3053 3093}"), :ascii_only?) + end - # argument should be converted to String + def test_delete_suffix_bang_argument_conversion klass = Class.new { def to_str; 'a'; end } s = S("abba") assert_equal("abb", s.delete_suffix!(klass.new)) assert_equal("abb", s) + end + def test_delete_suffix_bang_newline # chomp removes any of "\n", "\r\n", "\r" when "\n" is specified, # but delete_suffix does not s = "foo\n" From b054c2fe06598f1141fdc337b10046f41f0e227c Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 25 Aug 2023 00:52:53 +0900 Subject: [PATCH 036/208] [Bug #19784] Fix behaviors against prefix with broken encoding - String#start_with? - String#delete_prefix - String#delete_prefix! --- spec/ruby/core/string/start_with_spec.rb | 10 +++-- spec/ruby/shared/string/start_with.rb | 6 ++- string.c | 56 ++++++++++++++++++------ test/ruby/test_string.rb | 6 +++ 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/spec/ruby/core/string/start_with_spec.rb b/spec/ruby/core/string/start_with_spec.rb index 3833289f96d488..81eed47f96e800 100644 --- a/spec/ruby/core/string/start_with_spec.rb +++ b/spec/ruby/core/string/start_with_spec.rb @@ -7,12 +7,14 @@ it_behaves_like :start_with, :to_s # Here and not in the shared examples because this is invalid as a Symbol - it "does not check that we are not starting to match at the head of a character" do + it "matches part of a character with the same part" do "\xA9".should.start_with?("\xA9") # A9 is not a character head for UTF-8 end - it "does not check we are matching only part of a character" do - "\xe3\x81\x82".size.should == 1 - "\xe3\x81\x82".should.start_with?("\xe3") + ruby_bug "#19784", ""..."3.3" do + it "checks we are matching only part of a character" do + "\xe3\x81\x82".size.should == 1 + "\xe3\x81\x82".should_not.start_with?("\xe3") + end end end diff --git a/spec/ruby/shared/string/start_with.rb b/spec/ruby/shared/string/start_with.rb index 6932a017b6d621..91fc50c4cdd3e3 100644 --- a/spec/ruby/shared/string/start_with.rb +++ b/spec/ruby/shared/string/start_with.rb @@ -70,7 +70,9 @@ $1.should be_nil end - it "does not check that we are not matching part of a character" do - "\xC3\xA9".send(@method).should.start_with?("\xC3") + ruby_bug "#19784", ""..."3.3" do + it "checks that we are not matching part of a character" do + "\xC3\xA9".send(@method).should_not.start_with?("\xC3") + end end end diff --git a/string.c b/string.c index b65868a226b13e..5af5fc4a402156 100644 --- a/string.c +++ b/string.c @@ -10461,10 +10461,20 @@ rb_str_start_with(int argc, VALUE *argv, VALUE str) return Qtrue; } else { + const char *p, *s, *e; + long slen, tlen; + rb_encoding *enc; + StringValue(tmp); - rb_enc_check(str, tmp); - if (RSTRING_LEN(str) < RSTRING_LEN(tmp)) continue; - if (memcmp(RSTRING_PTR(str), RSTRING_PTR(tmp), RSTRING_LEN(tmp)) == 0) + enc = rb_enc_check(str, tmp); + if ((tlen = RSTRING_LEN(tmp)) == 0) return Qtrue; + if ((slen = RSTRING_LEN(str)) < tlen) continue; + p = RSTRING_PTR(str); + e = p + slen; + s = p + tlen; + if (!at_char_boundary(p, s, e, enc)) + continue; + if (memcmp(p, RSTRING_PTR(tmp), tlen) == 0) return Qtrue; } } @@ -10483,12 +10493,13 @@ static VALUE rb_str_end_with(int argc, VALUE *argv, VALUE str) { int i; - char *p, *s, *e; - rb_encoding *enc; for (i=0; i Date: Fri, 25 Aug 2023 20:17:14 -0400 Subject: [PATCH 037/208] [ruby/yarp] Bump to version 0.9.0 https://github.com/ruby/yarp/commit/b327e39527 --- lib/yarp/yarp.gemspec | 2 +- yarp/extension.h | 2 +- yarp/templates/java/org/yarp/Loader.java.erb | 2 +- yarp/templates/lib/yarp/serialize.rb.erb | 2 +- yarp/version.h | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/yarp/yarp.gemspec b/lib/yarp/yarp.gemspec index fceda5b1a0ac86..ceea5e8eb6e10b 100644 --- a/lib/yarp/yarp.gemspec +++ b/lib/yarp/yarp.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "yarp" - spec.version = "0.8.0" + spec.version = "0.9.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] diff --git a/yarp/extension.h b/yarp/extension.h index fe004d16da5cc8..fc24add420f2f1 100644 --- a/yarp/extension.h +++ b/yarp/extension.h @@ -1,7 +1,7 @@ #ifndef YARP_EXT_NODE_H #define YARP_EXT_NODE_H -#define EXPECTED_YARP_VERSION "0.8.0" +#define EXPECTED_YARP_VERSION "0.9.0" #include #include diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb index e3e29e446b84db..1c3a64354c0c11 100644 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ b/yarp/templates/java/org/yarp/Loader.java.erb @@ -60,7 +60,7 @@ public class Loader { expect((byte) 'P'); expect((byte) 0); - expect((byte) 8); + expect((byte) 9); expect((byte) 0); // This loads the name of the encoding. We don't actually do anything diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index 44ca3146fb385b..1959a71b62d19c 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -14,7 +14,7 @@ end module YARP module Serialize MAJOR_VERSION = 0 - MINOR_VERSION = 8 + MINOR_VERSION = 9 PATCH_VERSION = 0 def self.load(input, serialized) diff --git a/yarp/version.h b/yarp/version.h index 179543f54d751b..a364aec2474f03 100644 --- a/yarp/version.h +++ b/yarp/version.h @@ -1,4 +1,4 @@ #define YP_VERSION_MAJOR 0 -#define YP_VERSION_MINOR 8 +#define YP_VERSION_MINOR 9 #define YP_VERSION_PATCH 0 -#define YP_VERSION "0.8.0" +#define YP_VERSION "0.9.0" From 487d91fde18949e35d80ac1a4651a1f3124e1763 Mon Sep 17 00:00:00 2001 From: git Date: Sat, 26 Aug 2023 00:22:09 +0000 Subject: [PATCH 038/208] Update default gems list at e2b8eac767d41734520377b5cf6f44 [ci skip] --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index edad5de51e6ca1..1ce8995e4b9b42 100644 --- a/NEWS.md +++ b/NEWS.md @@ -83,7 +83,7 @@ The following default gems are updated. * time 0.2.2 * timeout 0.4.0 * uri 0.12.2 -* yarp 0.8.0 +* yarp 0.9.0 The following bundled gems are updated. From 9ea9f992487711fa1a66637edcaf1327d0cd5099 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 24 Aug 2023 14:23:19 -0400 Subject: [PATCH 039/208] [Feature #19785] Deprecate RUBY_GC_HEAP_INIT_SLOTS This environment variable is replaced by `RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS`, so it doesn't make sense to keep it. --- NEWS.md | 2 ++ gc.c | 9 --------- ruby.c | 7 +++++++ test/ruby/test_gc.rb | 35 ++++++----------------------------- 4 files changed, 15 insertions(+), 38 deletions(-) diff --git a/NEWS.md b/NEWS.md index 1ce8995e4b9b42..383e6ed04d2425 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,6 +12,7 @@ Note that each entry is kept to a minimum, see links for details. * A new `performance` warning category was introduced. They are not displayed by default even in verbose mode. Turn them on with `-W:performance` or `Warning[:performance] = true`. [[Feature #19538]] +* The `RUBY_GC_HEAP_INIT_SLOTS` environment variable has been deprecated and removed. Environment variables `RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS` should be used instead. [[Feature #19785]] ## Core classes updates @@ -164,3 +165,4 @@ changelog for details of the default gems or bundled gems. [Feature #19538]: https://bugs.ruby-lang.org/issues/19538 [Feature #19591]: https://bugs.ruby-lang.org/issues/19591 [Feature #19714]: https://bugs.ruby-lang.org/issues/19714 +[Feature #19785]: https://bugs.ruby-lang.org/issues/19785 diff --git a/gc.c b/gc.c index 758f65c2c8b0c8..6c4bffa95d0b4d 100644 --- a/gc.c +++ b/gc.c @@ -11615,8 +11615,6 @@ gc_set_initial_pages(rb_objspace_t *objspace) /* * GC tuning environment variables * - * * RUBY_GC_HEAP_INIT_SLOTS - * - Initial allocation slots. * * RUBY_GC_HEAP_FREE_SLOTS * - Prepare at least this amount of slots after GC. * - Allocate slots if there are not enough slots. @@ -11663,13 +11661,6 @@ ruby_gc_set_params(void) /* ok */ } - /* RUBY_GC_HEAP_INIT_SLOTS */ - size_t global_init_slots = GC_HEAP_INIT_SLOTS; - if (get_envparam_size("RUBY_GC_HEAP_INIT_SLOTS", &global_init_slots, 0)) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - gc_params.size_pool_init_slots[i] = global_init_slots; - } - } gc_set_initial_pages(objspace); get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE); diff --git a/ruby.c b/ruby.c index 2b4397c0011617..213425950f9131 100644 --- a/ruby.c +++ b/ruby.c @@ -1738,6 +1738,13 @@ ruby_opt_init(ruby_cmdline_options_t *opt) rb_warning_category_update(opt->warn.mask, opt->warn.set); + /* [Feature #19785] Warning for removed GC environment variable. + * Remove this in Ruby 3.4. */ + if (getenv("RUBY_GC_HEAP_INIT_SLOTS")) { + rb_warn_deprecated("The environment variable RUBY_GC_HEAP_INIT_SLOTS", + "environment variables RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS"); + } + #if USE_RJIT // rb_call_builtin_inits depends on RubyVM::RJIT.enabled? if (opt->rjit.on) diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index bfcfbbb72f6d3c..64a860fe651626 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -375,13 +375,9 @@ def test_gc_parameter env = { "RUBY_GC_HEAP_INIT_SLOTS" => "100" } - assert_in_out_err([env, "-W0", "-e", "exit"], "", [], [], "[Bug #19284]") - - env = { - "RUBY_GC_MALLOC_LIMIT" => "60000000", - "RUBY_GC_HEAP_INIT_SLOTS" => "100000" - } - assert_normal_exit("exit", "[ruby-core:39777]", :child_env => env) + assert_in_out_err([env, "-W0", "-e", "exit"], "", [], []) + assert_in_out_err([env, "-W:deprecated", "-e", "exit"], "", [], + /The environment variable RUBY_GC_HEAP_INIT_SLOTS is deprecated; use environment variables RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS instead/) env = {} GC.stat_heap.each do |_, s| @@ -389,26 +385,12 @@ def test_gc_parameter end assert_normal_exit("exit", "", :child_env => env) - env["RUBY_GC_HEAP_INIT_SLOTS"] = "100000" - assert_normal_exit("exit", "", :child_env => env) - env = {} GC.stat_heap.each do |_, s| env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = "0" end assert_normal_exit("exit", "", :child_env => env) - env = { - "RUBYOPT" => "", - "RUBY_GC_HEAP_INIT_SLOTS" => "100000" - } - assert_in_out_err([env, "-e", "exit"], "", [], [], "[ruby-core:39795]") - assert_in_out_err([env, "-W0", "-e", "exit"], "", [], [], "[ruby-core:39795]") - assert_in_out_err([env, "-W1", "-e", "exit"], "", [], [], "[ruby-core:39795]") - assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_INIT_SLOTS=100000/, "[ruby-core:39795]") - # Value of GC_HEAP_INIT_SLOTS is 10000 - assert_in_out_err([env, "-w", "-e", "exit"], "", [], /\(default value: 10000\)/) - env = { "RUBY_GC_HEAP_GROWTH_FACTOR" => "2.0", "RUBY_GC_HEAP_GROWTH_MAX_SLOTS" => "10000" @@ -417,15 +399,10 @@ def test_gc_parameter assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_GROWTH_FACTOR=2.0/, "") assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_GROWTH_MAX_SLOTS=10000/, "[ruby-core:57928]") - env = { - "RUBY_GC_HEAP_INIT_SLOTS" => "100000", - "RUBY_GC_HEAP_FREE_SLOTS" => "10000", - "RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR" => "0.4", - } - assert_normal_exit("exit", "", :child_env => env) - assert_in_out_err([env, "-w", "-e", "exit"], "", [], /RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR=0\.4/, "") - if use_rgengc? + env = { + "RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR" => "0.4", + } # always full GC when RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR < 1.0 assert_in_out_err([env, "--disable-gems", "-e", "GC.start; 1000_000.times{Object.new}; p(GC.stat[:minor_gc_count] < GC.stat[:major_gc_count])"], "", ['true'], //, "") end From a97dedf761c6931cbb2b9267faad2395481a3f57 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Sat, 26 Aug 2023 09:56:44 -0400 Subject: [PATCH 040/208] [ruby/irb] Fix deprecation test when ran multiple times (https://github.com/ruby/irb/pull/695) https://github.com/ruby/irb/commit/ae0e5bb80f --- test/irb/test_ruby_lex.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb index 336f4fd3660ddf..09344e1b0efed9 100644 --- a/test/irb/test_ruby_lex.rb +++ b/test/irb/test_ruby_lex.rb @@ -203,11 +203,10 @@ def test_assignment_expression_with_local_variable end def test_initialising_the_old_top_level_ruby_lex - _, err = capture_output do + assert_in_out_err(["--disable-gems", "-W:deprecated"], <<~RUBY, [], /warning: constant ::RubyLex is deprecated/) + require "irb" ::RubyLex.new(nil) - end - - assert_match(/warning: constant ::RubyLex is deprecated/, err) + RUBY end private From 5c98ee02d2ac7f20ab978be7645801adf03e4302 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 27 Aug 2023 00:12:03 +0900 Subject: [PATCH 041/208] Define bounds-checking interfaces macro for each file --- configure.ac | 3 --- util.c | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 05dd41e79c9f69..8447eba1051b09 100644 --- a/configure.ac +++ b/configure.ac @@ -2153,9 +2153,6 @@ AC_CHECK_FUNCS(__sinpi) AS_IF([test "x$ac_cv_member_struct_statx_stx_btime" = xyes], [AC_CHECK_FUNCS(statx)]) -AS_CASE(["$ac_cv_func_memset_s:$ac_cv_func_qsort_s"], [*yes*], - [RUBY_DEFINE_IF([!defined __STDC_WANT_LIB_EXT1__], [__STDC_WANT_LIB_EXT1__], 1)]) - AS_IF([test "$ac_cv_func_getcwd" = yes], [ AC_CACHE_CHECK(if getcwd allocates buffer if NULL is given, [rb_cv_getcwd_malloc], [AC_RUN_IFELSE([AC_LANG_SOURCE([[ diff --git a/util.c b/util.c index 1030c3ecbede16..f39e17b8b7b41d 100644 --- a/util.c +++ b/util.c @@ -13,6 +13,10 @@ # define MINGW_HAS_SECURE_API 1 #endif +#ifndef __STDC_WANT_LIB_EXT1__ +#define __STDC_WANT_LIB_EXT1__ 1 +#endif + #include "ruby/internal/config.h" #include From b7237e3bbd36e7c520c4cbaf1f866b6dcc265a99 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 21 Aug 2023 14:13:24 -0400 Subject: [PATCH 042/208] Free all empty heap pages in Process.warmup This commit adds `free_empty_pages` which frees all empty heap pages and moves the number of pages freed to the allocatable pages counter. This is used in Process.warmup to improve performance because page invalidation from copy-on-write is slower than allocating a new page. --- gc.c | 44 +++++++++++++++++++++++++++++++++++++++ process.c | 6 ++++-- test/ruby/test_process.rb | 22 ++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/gc.c b/gc.c index 6c4bffa95d0b4d..34334f3278c27f 100644 --- a/gc.c +++ b/gc.c @@ -9647,11 +9647,55 @@ gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE return Qnil; } +static void +free_empty_pages(void) +{ + rb_objspace_t *objspace = &rb_objspace; + + for (int i = 0; i < SIZE_POOL_COUNT; i++) { + /* Move all empty pages to the tomb heap for freeing. */ + rb_size_pool_t *size_pool = &size_pools[i]; + rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + rb_heap_t *tomb_heap = SIZE_POOL_TOMB_HEAP(size_pool); + + size_t freed_pages = 0; + + struct heap_page **next_page_ptr = &heap->free_pages; + struct heap_page *page = heap->free_pages; + while (page) { + /* All finalizers should have been ran in gc_start_internal, so there + * should be no objects that require finalization. */ + GC_ASSERT(page->final_slots == 0); + + struct heap_page *next_page = page->free_next; + + if (page->free_slots == page->total_slots) { + heap_unlink_page(objspace, heap, page); + heap_add_page(objspace, size_pool, tomb_heap, page); + freed_pages++; + } + else { + *next_page_ptr = page; + next_page_ptr = &page->free_next; + } + + page = next_page; + } + + *next_page_ptr = NULL; + + size_pool_allocatable_pages_set(objspace, size_pool, size_pool->allocatable_pages + freed_pages); + } + + heap_pages_free_unused_pages(objspace); +} + void rb_gc_prepare_heap(void) { rb_objspace_each_objects(gc_set_candidate_object_i, NULL); gc_start_internal(NULL, Qtrue, Qtrue, Qtrue, Qtrue, Qtrue); + free_empty_pages(); } static int diff --git a/process.c b/process.c index fa2ae7344ab20c..37dc5244153400 100644 --- a/process.c +++ b/process.c @@ -8671,10 +8671,12 @@ static VALUE rb_mProcID_Syscall; * * On CRuby, +Process.warmup+: * - * * Perform a major GC. + * * Performs a major GC. * * Compacts the heap. * * Promotes all surviving objects to the old generation. - * * Precompute the coderange of all strings. + * * Precomputes the coderange of all strings. + * * Frees all empty heap pages and increments the allocatable pages counter + * by the number of pages freed. */ static VALUE diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 38dcb8054fc1ca..095ab27f5d2c66 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -2725,4 +2725,26 @@ def test_warmup_precompute_string_coderange assert_include(ObjectSpace.dump(obj), '"coderange":"7bit"') end; end + + def test_warmup_frees_pages + assert_separately([{"RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO" => "1.0"}, "-W0"], "#{<<~"begin;"}\n#{<<~'end;'}") + begin; + TIMES = 10_000 + ary = Array.new(TIMES) + TIMES.times do |i| + ary[i] = Object.new + end + ary.clear + ary = nil + + total_pages_before = GC.stat(:heap_eden_pages) + GC.stat(:heap_allocatable_pages) + + Process.warmup + + # Number of pages freed should cause equal increase in number of allocatable pages. + assert_equal(total_pages_before, GC.stat(:heap_eden_pages) + GC.stat(:heap_allocatable_pages)) + assert_equal(0, GC.stat(:heap_tomb_pages)) + assert_operator(GC.stat(:total_freed_pages), :>, 0) + end; + end end From 78c5bb1136f7db559f14ba44eb04503b1493672d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 27 Aug 2023 23:21:20 +0900 Subject: [PATCH 043/208] Remove duplicate `#include ` [ci skip] --- missing/explicit_bzero.c | 5 +---- util.c | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/missing/explicit_bzero.c b/missing/explicit_bzero.c index 1220e5f9ad7c23..59417e158ebf10 100644 --- a/missing/explicit_bzero.c +++ b/missing/explicit_bzero.c @@ -1,12 +1,9 @@ #ifndef __STDC_WANT_LIB_EXT1__ -#define __STDC_WANT_LIB_EXT1__ 1 +#define __STDC_WANT_LIB_EXT1__ 1 /* for memset_s() */ #endif #include "ruby/missing.h" #include -#ifdef HAVE_MEMSET_S -# include -#endif #ifdef _WIN32 #include diff --git a/util.c b/util.c index f39e17b8b7b41d..3c08879ce58698 100644 --- a/util.c +++ b/util.c @@ -14,7 +14,7 @@ #endif #ifndef __STDC_WANT_LIB_EXT1__ -#define __STDC_WANT_LIB_EXT1__ 1 +#define __STDC_WANT_LIB_EXT1__ 1 /* for qsort_s() */ #endif #include "ruby/internal/config.h" From 412e586afe1efd137872610a9ed6e93a45cf4c08 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Sun, 27 Aug 2023 18:01:02 +0200 Subject: [PATCH 044/208] [ruby/yarp] Fix paths in library_symbols_test.rb https://github.com/ruby/yarp/commit/b5fba6d63f --- test/yarp/library_symbols_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/yarp/library_symbols_test.rb b/test/yarp/library_symbols_test.rb index c5927c2a89edea..9a52d189baff64 100644 --- a/test/yarp/library_symbols_test.rb +++ b/test/yarp/library_symbols_test.rb @@ -11,9 +11,9 @@ class LibrarySymbolsTest < Test::Unit::TestCase def setup super - @librubyparser_a = File.expand_path(File.join(__dir__, "..", "build", "librubyparser.a")) - @librubyparser_so = File.expand_path(File.join(__dir__, "..", "build", "librubyparser.so")) - @yarp_so = File.expand_path(File.join(__dir__, "..", "lib", "yarp", "yarp.so")) + @librubyparser_a = File.expand_path("../../build/librubyparser.a", __dir__) + @librubyparser_so = File.expand_path("../../build/librubyparser.so", __dir__) + @yarp_so = File.expand_path("../../lib/yarp/yarp.so", __dir__) end # objdump runner and helpers From 5937d01f7f90d7cb30b25c5c942cbf948e46c9d6 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Sun, 27 Aug 2023 17:56:53 +0200 Subject: [PATCH 045/208] [ruby/yarp] Rename constant pool fields to name or operator * `constant_id` and `operator_id` are confusing. * See https://github.com/ruby/yarp/issues/1296 https://github.com/ruby/yarp/commit/09d0a144df --- lib/yarp.rb | 10 +++++----- lib/yarp/desugar_visitor.rb | 10 +++++----- yarp/config.yml | 20 ++++++++++---------- yarp/yarp.c | 36 ++++++++++++++++++------------------ 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/lib/yarp.rb b/lib/yarp.rb index a40f27009178f7..316918b2db406d 100644 --- a/lib/yarp.rb +++ b/lib/yarp.rb @@ -462,10 +462,10 @@ def self.yarp_locals(source) # order here so that we can compare properly. if params sorted = [ - *params.requireds.grep(RequiredParameterNode).map(&:constant_id), - *params.optionals.map(&:constant_id), + *params.requireds.grep(RequiredParameterNode).map(&:name), + *params.optionals.map(&:name), *((params.rest.name ? params.rest.name.to_sym : :*) if params.rest && params.rest.operator != ","), - *params.posts.grep(RequiredParameterNode).map(&:constant_id), + *params.posts.grep(RequiredParameterNode).map(&:name), *params.keywords.reject(&:value).map { |param| param.name.chomp(":").to_sym }, *params.keywords.select(&:value).map { |param| param.name.chomp(":").to_sym } ] @@ -485,9 +485,9 @@ def self.yarp_locals(source) when RequiredDestructuredParameterNode param_stack.concat(param.parameters.reverse) when RequiredParameterNode - sorted << param.constant_id + sorted << param.name when SplatNode - sorted << param.expression.constant_id if param.expression + sorted << param.expression.name if param.expression end end diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb index 6e1831e98e8350..3f901630b0c0cc 100644 --- a/lib/yarp/desugar_visitor.rb +++ b/lib/yarp/desugar_visitor.rb @@ -210,8 +210,8 @@ def visit_instance_variable_operator_write_node(node) # foo && foo = bar def visit_local_variable_and_write_node(node) AndNode.new( - LocalVariableReadNode.new(node.constant_id, node.depth, node.name_loc), - LocalVariableWriteNode.new(node.constant_id, node.depth, node.name_loc, node.value, node.operator_loc, node.location), + LocalVariableReadNode.new(node.name, node.depth, node.name_loc), + LocalVariableWriteNode.new(node.name, node.depth, node.name_loc, node.value, node.operator_loc, node.location), node.operator_loc, node.location ) @@ -224,8 +224,8 @@ def visit_local_variable_and_write_node(node) # foo || foo = bar def visit_local_variable_or_write_node(node) OrNode.new( - LocalVariableReadNode.new(node.constant_id, node.depth, node.name_loc), - LocalVariableWriteNode.new(node.constant_id, node.depth, node.name_loc, node.value, node.operator_loc, node.location), + LocalVariableReadNode.new(node.name, node.depth, node.name_loc), + LocalVariableWriteNode.new(node.name, node.depth, node.name_loc, node.value, node.operator_loc, node.location), node.operator_loc, node.location ) @@ -237,7 +237,7 @@ def visit_local_variable_or_write_node(node) # # foo = foo + bar def visit_local_variable_operator_write_node(node) - desugar_operator_write_node(node, LocalVariableWriteNode, LocalVariableReadNode, arguments: [node.constant_id, node.depth]) + desugar_operator_write_node(node, LocalVariableWriteNode, LocalVariableReadNode, arguments: [node.name, node.depth]) end private diff --git a/yarp/config.yml b/yarp/config.yml index c7fec6f8468fae..d35436088abdda 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -662,7 +662,7 @@ nodes: type: location - name: value type: node - - name: operator_id + - name: operator type: constant comment: | Represents the use of an assignment operator on a call. @@ -1492,7 +1492,7 @@ nodes: type: location - name: value type: node - - name: constant_id + - name: name type: constant - name: depth type: uint32 @@ -1509,9 +1509,9 @@ nodes: type: location - name: value type: node - - name: constant_id + - name: name type: constant - - name: operator_id + - name: operator type: constant - name: depth type: uint32 @@ -1528,7 +1528,7 @@ nodes: type: location - name: value type: node - - name: constant_id + - name: name type: constant - name: depth type: uint32 @@ -1539,7 +1539,7 @@ nodes: ^^^^^^^^^^^^^^^^ - name: LocalVariableReadNode child_nodes: - - name: constant_id + - name: name type: constant - name: depth type: uint32 @@ -1552,7 +1552,7 @@ nodes: ^^^ - name: LocalVariableTargetNode child_nodes: - - name: constant_id + - name: name type: constant - name: depth type: uint32 @@ -1563,7 +1563,7 @@ nodes: ^^^ ^^^ - name: LocalVariableWriteNode child_nodes: - - name: constant_id + - name: name type: constant - name: depth type: uint32 @@ -1682,7 +1682,7 @@ nodes: ^^ - name: OptionalParameterNode child_nodes: - - name: constant_id + - name: name type: constant - name: name_loc type: location @@ -1883,7 +1883,7 @@ nodes: end - name: RequiredParameterNode child_nodes: - - name: constant_id + - name: name type: constant comment: | Represents a required parameter to a method, block, or lambda definition. diff --git a/yarp/yarp.c b/yarp/yarp.c index 8758d365843b12..ba8722a9250a80 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -1434,7 +1434,7 @@ yp_call_operator_write_node_create(yp_parser_t *parser, yp_call_node_t *target, .target = target, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, - .operator_id = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) }; return node; @@ -2959,7 +2959,7 @@ yp_lambda_node_create( // Allocate and initialize a new LocalVariableAndWriteNode node. static yp_local_variable_and_write_node_t * -yp_local_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id, uint32_t depth) { +yp_local_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t name, uint32_t depth) { assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); yp_local_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_and_write_node_t); @@ -2975,7 +2975,7 @@ yp_local_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, .name_loc = target->location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, - .constant_id = constant_id, + .name = name, .depth = depth }; @@ -2984,7 +2984,7 @@ yp_local_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, // Allocate and initialize a new LocalVariableOperatorWriteNode node. static yp_local_variable_operator_write_node_t * -yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id, uint32_t depth) { +yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t name, uint32_t depth) { yp_local_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_operator_write_node_t); *node = (yp_local_variable_operator_write_node_t) { @@ -2998,8 +2998,8 @@ yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *tar .name_loc = target->location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, - .constant_id = constant_id, - .operator_id = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), + .name = name, + .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1), .depth = depth }; @@ -3008,7 +3008,7 @@ yp_local_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *tar // Allocate and initialize a new LocalVariableOrWriteNode node. static yp_local_variable_or_write_node_t * -yp_local_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t constant_id, uint32_t depth) { +yp_local_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value, yp_constant_id_t name, uint32_t depth) { assert(YP_NODE_TYPE_P(target, YP_NODE_LOCAL_VARIABLE_READ_NODE) || YP_NODE_TYPE_P(target, YP_NODE_CALL_NODE)); assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); yp_local_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_or_write_node_t); @@ -3024,7 +3024,7 @@ yp_local_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, c .name_loc = target->location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, - .constant_id = constant_id, + .name = name, .depth = depth }; @@ -3041,7 +3041,7 @@ yp_local_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name, .type = YP_NODE_LOCAL_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(name) }, - .constant_id = yp_parser_constant_id_token(parser, name), + .name = yp_parser_constant_id_token(parser, name), .depth = depth }; @@ -3050,7 +3050,7 @@ yp_local_variable_read_node_create(yp_parser_t *parser, const yp_token_t *name, // Allocate and initialize a new LocalVariableWriteNode node. static yp_local_variable_write_node_t * -yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t constant_id, uint32_t depth, yp_node_t *value, const yp_location_t *name_loc, const yp_token_t *operator) { +yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t name, uint32_t depth, yp_node_t *value, const yp_location_t *name_loc, const yp_token_t *operator) { yp_local_variable_write_node_t *node = YP_ALLOC_NODE(parser, yp_local_variable_write_node_t); *node = (yp_local_variable_write_node_t) { @@ -3061,7 +3061,7 @@ yp_local_variable_write_node_create(yp_parser_t *parser, yp_constant_id_t consta .end = value->location.end } }, - .constant_id = constant_id, + .name = name, .depth = depth, .value = value, .name_loc = *name_loc, @@ -3081,7 +3081,7 @@ yp_local_variable_target_node_create(yp_parser_t *parser, const yp_token_t *name .type = YP_NODE_LOCAL_VARIABLE_TARGET_NODE, .location = YP_LOCATION_TOKEN_VALUE(name) }, - .constant_id = yp_parser_constant_id_token(parser, name), + .name = yp_parser_constant_id_token(parser, name), .depth = 0 }; @@ -3279,7 +3279,7 @@ yp_optional_parameter_node_create(yp_parser_t *parser, const yp_token_t *name, c .end = value->location.end } }, - .constant_id = yp_parser_constant_id_token(parser, name), + .name = yp_parser_constant_id_token(parser, name), .name_loc = YP_LOCATION_TOKEN_VALUE(name), .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value @@ -3630,7 +3630,7 @@ yp_required_parameter_node_create(yp_parser_t *parser, const yp_token_t *token) .type = YP_NODE_REQUIRED_PARAMETER_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) }, - .constant_id = yp_parser_constant_id_token(parser, token) + .name = yp_parser_constant_id_token(parser, token) }; return node; @@ -7987,7 +7987,7 @@ parse_write(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_nod case YP_NODE_LOCAL_VARIABLE_READ_NODE: { yp_local_variable_read_node_t *local_read = (yp_local_variable_read_node_t *) target; - yp_constant_id_t constant_id = local_read->constant_id; + yp_constant_id_t constant_id = local_read->name; uint32_t depth = local_read->depth; yp_location_t name_loc = target->location; @@ -12793,7 +12793,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_local_variable_and_write_node_create(parser, node, &token, value, cast->constant_id, cast->depth); + yp_node_t *result = (yp_node_t *) yp_local_variable_and_write_node_create(parser, node, &token, value, cast->name, cast->depth); yp_node_destroy(parser, node); return result; @@ -12894,7 +12894,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_local_variable_or_write_node_create(parser, node, &token, value, cast->constant_id, cast->depth); + yp_node_t *result = (yp_node_t *) yp_local_variable_or_write_node_create(parser, node, &token, value, cast->name, cast->depth); yp_node_destroy(parser, node); return result; @@ -13005,7 +13005,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, cast->constant_id, cast->depth); + yp_node_t *result = (yp_node_t *) yp_local_variable_operator_write_node_create(parser, node, &token, value, cast->name, cast->depth); yp_node_destroy(parser, node); return result; From 1cc700907d3ad3368272488a6f8960f2336bf26e Mon Sep 17 00:00:00 2001 From: Burdette Lamar Date: Sun, 27 Aug 2023 12:39:18 -0500 Subject: [PATCH 046/208] [DOC] More on method exec (#8302) --- process.c | 48 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/process.c b/process.c index 37dc5244153400..0de0b1f0a3ee19 100644 --- a/process.c +++ b/process.c @@ -3013,6 +3013,9 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * - Passing string +command_line+ to the shell. * - Invoking the executable at +exe_path+. * + * This method has potential security vulnerabilities if called with untrusted input; + * see {Command Injection}[rdoc-ref:command_injection.rdoc]. + * * The new process is created using the * {exec system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/execve.html]; * it may inherit some of its environment from the calling program @@ -3035,8 +3038,19 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * * \String argument +command_line+ is a command line to be passed to a shell; * it must begin with a shell reserved word, begin with a special built-in, - * or contain meta characters. - * It may also contain arguments and options for that command. + * or contain meta characters: + * + * exec('echo') # Built-in. + * exec('if true; then echo "Foo"; fi') # Shell reserved word. + * exec('date > date.tmp') # Contains meta character. + * + * The command line may also contain arguments and options for the command: + * + * exec('echo "Foo"') + * + * Output: + * + * Foo * * On a Unix-like system, the shell is /bin/sh; * otherwise the shell is determined by environment variable @@ -3046,7 +3060,13 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * the entire string +command_line+ is passed as an argument * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. * - * The shell performs normal shell expansion on the command line. + * The shell performs normal shell expansion on the command line: + * + * exec('echo C*') + * + * Output: + * + * CONTRIBUTING.md COPYING COPYING.ja * * Raises an exception if the new process fails to execute. * @@ -3058,10 +3078,28 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * - A 2-element array containing the path to an executable * and the string to be used as the name of the executing process. * - * Ruby invokes the executable directly, with no shell and no shell expansion. + * Example: + * + * exec('/usr/bin/date') + * + * Output: + * + * Sat Aug 26 09:38:00 AM CDT 2023 + * + * Ruby invokes the executable directly, with no shell and no shell expansion: + * + * exec('doesnt_exist') # Raises Errno::ENOENT * * If one or more +args+ is given, each is an argument or option - * to be passed to the executable. + * to be passed to the executable: + * + * exec('echo', 'C*') + * exec('echo', 'hello', 'world') + * + * Output: + * + * C* + * hello world * * Raises an exception if the new process fails to execute. */ From 0c9d0684e1ab75277f8543db769dc7bdaccaa332 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 28 Aug 2023 09:44:57 +0900 Subject: [PATCH 047/208] YARP: generated files using from templates depend on config.yml --- common.mk | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/common.mk b/common.mk index c0d3e0a673471e..1a85d0605af1e7 100644 --- a/common.mk +++ b/common.mk @@ -45,7 +45,8 @@ RUN_OPTS = --disable-gems # GITPULLOPTIONS = --no-tags -INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(srcdir) -I$(srcdir)/yarp -I$(UNICODE_HDR_DIR) +YARP_SRCDIR = $(srcdir)/yarp +INCFLAGS = -I. -I$(arch_hdrdir) -I$(hdrdir) -I$(srcdir) -I$(YARP_SRCDIR) -I$(UNICODE_HDR_DIR) GEM_HOME = GEM_PATH = @@ -204,42 +205,42 @@ $(YARP_BUILD_DIR)/.time $(YARP_BUILD_DIR)/enc/.time $(YARP_BUILD_DIR)/util/.time main: $(srcdir)/lib/yarp/mutation_visitor.rb srcs: $(srcdir)/lib/yarp/mutation_visitor.rb -$(srcdir)/lib/yarp/mutation_visitor.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/mutation_visitor.rb.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/mutation_visitor.rb $(srcdir)/lib/yarp/mutation_visitor.rb +$(srcdir)/lib/yarp/mutation_visitor.rb: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/lib/yarp/mutation_visitor.rb.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb lib/yarp/mutation_visitor.rb $(srcdir)/lib/yarp/mutation_visitor.rb main: $(srcdir)/lib/yarp/node.rb srcs: $(srcdir)/lib/yarp/node.rb -$(srcdir)/lib/yarp/node.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/node.rb.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/node.rb $(srcdir)/lib/yarp/node.rb +$(srcdir)/lib/yarp/node.rb: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/lib/yarp/node.rb.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb lib/yarp/node.rb $(srcdir)/lib/yarp/node.rb main: $(srcdir)/lib/yarp/serialize.rb srcs: $(srcdir)/lib/yarp/serialize.rb -$(srcdir)/lib/yarp/serialize.rb: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/lib/yarp/serialize.rb.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb lib/yarp/serialize.rb $(srcdir)/lib/yarp/serialize.rb +$(srcdir)/lib/yarp/serialize.rb: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/lib/yarp/serialize.rb.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb lib/yarp/serialize.rb $(srcdir)/lib/yarp/serialize.rb srcs: yarp/api_node.c -yarp/api_node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/ext/yarp/api_node.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb ext/yarp/api_node.c $@ +yarp/api_node.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/ext/yarp/api_node.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb ext/yarp/api_node.c $@ srcs: yarp/ast.h -yarp/ast.h: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/include/yarp/ast.h.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb include/yarp/ast.h $@ +yarp/ast.h: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/include/yarp/ast.h.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb include/yarp/ast.h $@ srcs: yarp/node.c -yarp/node.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/node.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/node.c $@ +yarp/node.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/node.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/node.c $@ srcs: yarp/prettyprint.c -yarp/prettyprint.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/prettyprint.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/prettyprint.c $@ +yarp/prettyprint.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/prettyprint.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/prettyprint.c $@ srcs: yarp/serialize.c -yarp/serialize.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/serialize.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/serialize.c $@ +yarp/serialize.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/serialize.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/serialize.c $@ srcs: yarp/token_type.c -yarp/token_type.c: $(srcdir)/yarp/templates/template.rb $(srcdir)/yarp/templates/src/token_type.c.erb - $(Q) $(BASERUBY) $(srcdir)/yarp/templates/template.rb src/token_type.c $@ +yarp/token_type.c: $(YARP_SRCDIR)/config.yml $(YARP_SRCDIR)/templates/template.rb $(YARP_SRCDIR)/templates/src/token_type.c.erb + $(Q) $(BASERUBY) $(YARP_SRCDIR)/templates/template.rb src/token_type.c $@ EXPORTOBJS = $(DLNOBJ) \ localeinit.$(OBJEXT) \ From 7bf5f780281edc2fca83a0657e3a8d256e6e7065 Mon Sep 17 00:00:00 2001 From: Martin Emde Date: Mon, 21 Aug 2023 14:05:57 -0700 Subject: [PATCH 048/208] [rubygems/rubygems] Refactor Fetcher#api_fetcher? and fetcher loading logic https://github.com/rubygems/rubygems/commit/f664d60114 --- lib/bundler/fetcher.rb | 61 ++++++++++++------------- lib/bundler/fetcher/compact_index.rb | 4 -- lib/bundler/resolver.rb | 13 ++++-- lib/bundler/source/rubygems.rb | 2 +- spec/bundler/bundler/fetcher_spec.rb | 66 ++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 38 deletions(-) diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index 2119799f688437..9b64a3c771b408 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -9,6 +9,7 @@ module Bundler # Handles all the fetching with the rubygems server class Fetcher + autoload :Base, File.expand_path("fetcher/base", __dir__) autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__) autoload :Downloader, File.expand_path("fetcher/downloader", __dir__) autoload :Dependency, File.expand_path("fetcher/dependency", __dir__) @@ -134,18 +135,7 @@ def specs_with_retry(gem_names, source) def specs(gem_names, source) index = Bundler::Index.new - if Bundler::Fetcher.disable_endpoint - @use_api = false - specs = fetchers.last.specs(gem_names) - else - specs = [] - @fetchers = fetchers.drop_while do |f| - !f.available? || (f.api_fetcher? && !gem_names) || !specs = f.specs(gem_names) - end - @use_api = false if fetchers.none?(&:api_fetcher?) - end - - specs.each do |name, version, platform, dependencies, metadata| + fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata| spec = if dependencies EndpointSpecification.new(name, version, platform, self, dependencies, metadata) else @@ -158,22 +148,10 @@ def specs(gem_names, source) index rescue CertificateFailureError - Bundler.ui.info "" if gem_names && use_api # newline after dots + Bundler.ui.info "" if gem_names && api_fetcher? # newline after dots raise end - def use_api - return @use_api if defined?(@use_api) - - fetchers.shift until fetchers.first.available? - - @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint - false - else - fetchers.first.api_fetcher? - end - end - def user_agent @user_agent ||= begin ruby = Bundler::RubyVersion.system @@ -209,10 +187,6 @@ def user_agent end end - def fetchers - @fetchers ||= FETCHERS.map {|f| f.new(downloader, @remote, uri) } - end - def http_proxy return unless uri = connection.proxy_uri uri.to_s @@ -222,9 +196,36 @@ def inspect "#<#{self.class}:0x#{object_id} uri=#{uri}>" end + def api_fetcher? + fetchers.first.api_fetcher? + end + private - FETCHERS = [CompactIndex, Dependency, Index].freeze + def available_fetchers + if Bundler::Fetcher.disable_endpoint + [Index] + elsif remote_uri.scheme == "file" + Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") + [Index] + else + [CompactIndex, Dependency, Index] + end + end + + def fetchers + @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri) }.drop_while {|f| !f.available? } + end + + def fetch_specs(gem_names) + fetchers.reject!(&:api_fetcher?) unless gem_names + fetchers.reject! do |f| + specs = f.specs(gem_names) + return specs if specs + true + end + [] + end def cis env_cis = { diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index d3160251cb0056..6786a841f56b02 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -60,10 +60,6 @@ def available? Bundler.ui.debug("FIPS mode is enabled, bundler can't use the CompactIndex API") return nil end - if fetch_uri.scheme == "file" - Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") - return false - end # Read info file checksums out of /versions, so we can know if gems are up to date compact_index_client.update_and_parse_checksums! rescue CompactIndexClient::Updater::MisMatchedChecksumError => e diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index 2ad35bc931a136..fa95ff879b981b 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -37,9 +37,9 @@ def setup_solver root_version = Resolver::Candidate.new(0) @all_specs = Hash.new do |specs, name| - specs[name] = source_for(name).specs.search(name).reject do |s| - s.dependencies.any? {|d| d.name == name && !d.requirement.satisfied_by?(s.version) } # ignore versions that depend on themselves incorrectly - end.sort_by {|s| [s.version, s.platform.to_s] } + matches = source_for(name).specs.search(name) + matches = filter_invalid_self_dependencies(matches, name) + specs[name] = matches.sort_by {|s| [s.version, s.platform.to_s] } end @sorted_versions = Hash.new do |candidates, package| @@ -318,6 +318,13 @@ def filter_prereleases(specs, package) specs.reject {|s| s.version.prerelease? } end + # Ignore versions that depend on themselves incorrectly + def filter_invalid_self_dependencies(specs, name) + specs.reject do |s| + s.dependencies.any? {|d| d.name == name && !d.requirement.satisfied_by?(s.version) } + end + end + def requirement_satisfied_by?(requirement, spec) requirement.satisfied_by?(spec.version) || spec.source.is_a?(Source::Gemspec) end diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 9790753204b91e..15bd8dacbbbe25 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -392,7 +392,7 @@ def cached_specs end def api_fetchers - fetchers.select {|f| f.use_api && f.fetchers.first.api_fetcher? } + fetchers.select(&:api_fetcher?) end def remote_specs diff --git a/spec/bundler/bundler/fetcher_spec.rb b/spec/bundler/bundler/fetcher_spec.rb index 27a63c476d321f..92790dc7d6f744 100644 --- a/spec/bundler/bundler/fetcher_spec.rb +++ b/spec/bundler/bundler/fetcher_spec.rb @@ -189,4 +189,70 @@ end end end + + describe "#specs_with_retry" do + let(:downloader) { double(:downloader) } + let(:remote) { double(:remote, :cache_slug => "slug", :uri => uri, :original_uri => nil, :anonymized_uri => uri) } + let(:compact_index) { double(Bundler::Fetcher::CompactIndex, :available? => true, :api_fetcher? => true) } + let(:dependency) { double(Bundler::Fetcher::Dependency, :available? => true, :api_fetcher? => true) } + let(:index) { double(Bundler::Fetcher::Index, :available? => true, :api_fetcher? => false) } + + before do + allow(Bundler::Fetcher::CompactIndex).to receive(:new).and_return(compact_index) + allow(Bundler::Fetcher::Dependency).to receive(:new).and_return(dependency) + allow(Bundler::Fetcher::Index).to receive(:new).and_return(index) + end + + it "picks the first fetcher that works" do + expect(compact_index).to receive(:specs).with("name").and_return([["name", "1.2.3", "ruby"]]) + expect(dependency).not_to receive(:specs) + expect(index).not_to receive(:specs) + fetcher.specs_with_retry("name", double(Bundler::Source::Rubygems)) + end + + context "when APIs are not available" do + before do + allow(compact_index).to receive(:available?).and_return(false) + allow(dependency).to receive(:available?).and_return(false) + end + + it "uses the index" do + expect(compact_index).not_to receive(:specs) + expect(dependency).not_to receive(:specs) + expect(index).to receive(:specs).with("name").and_return([["name", "1.2.3", "ruby"]]) + + fetcher.specs_with_retry("name", double(Bundler::Source::Rubygems)) + end + end + end + + describe "#api_fetcher?" do + let(:downloader) { double(:downloader) } + let(:remote) { double(:remote, :cache_slug => "slug", :uri => uri, :original_uri => nil, :anonymized_uri => uri) } + let(:compact_index) { double(Bundler::Fetcher::CompactIndex, :available? => false, :api_fetcher? => true) } + let(:dependency) { double(Bundler::Fetcher::Dependency, :available? => false, :api_fetcher? => true) } + let(:index) { double(Bundler::Fetcher::Index, :available? => true, :api_fetcher? => false) } + + before do + allow(Bundler::Fetcher::CompactIndex).to receive(:new).and_return(compact_index) + allow(Bundler::Fetcher::Dependency).to receive(:new).and_return(dependency) + allow(Bundler::Fetcher::Index).to receive(:new).and_return(index) + end + + context "when an api fetcher is available" do + before do + allow(compact_index).to receive(:available?).and_return(true) + end + + it "is truthy" do + expect(fetcher).to be_api_fetcher + end + end + + context "when only the index fetcher is available" do + it "is falsey" do + expect(fetcher).not_to be_api_fetcher + end + end + end end From 279dcfab7aea4474c6722fd39cbf65017edb4b2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Fri, 25 Aug 2023 09:55:37 +0200 Subject: [PATCH 049/208] [rubygems/rubygems] Fix standalone install crashing when using legacy multi remote gemfiles If a legacy multi remote Gemfile depends transitively on a default gem, then in standalone mode we'd fail to fetch the proper version from the source that includes it, since we were adding it to `specs` (instead of `remote_specs`), which was already including the default version of the gem, and thus preventing the remote version from "overwriting that" and being added to the index. We should add it to the `remote_specs` index directly instead. https://github.com/rubygems/rubygems/commit/05f4f9dfc0 --- lib/bundler/source/rubygems.rb | 4 ++- spec/bundler/install/gemfile/sources_spec.rb | 27 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 15bd8dacbbbe25..7a80ec2b00e52e 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -275,7 +275,9 @@ def double_check_for(unmet_dependency_names) Bundler.ui.debug "Double checking for #{unmet_dependency_names || "all specs (due to the size of the request)"} in #{self}" - fetch_names(api_fetchers, unmet_dependency_names, specs, false) + fetch_names(api_fetchers, unmet_dependency_names, remote_specs, false) + + specs.use(remote_specs, false) end def dependency_names_to_double_check diff --git a/spec/bundler/install/gemfile/sources_spec.rb b/spec/bundler/install/gemfile/sources_spec.rb index 1f89f9f0f9b933..8cb04768212e9b 100644 --- a/spec/bundler/install/gemfile/sources_spec.rb +++ b/spec/bundler/install/gemfile/sources_spec.rb @@ -78,6 +78,33 @@ end end + context "without source affinity, and a stdlib gem present in one of the sources", :ruby_repo do + let(:default_json_version) { ruby "gem 'json'; require 'json'; puts JSON::VERSION" } + + before do + build_repo2 do + build_gem "json", default_json_version + end + + build_repo4 do + build_gem "foo" do |s| + s.add_dependency "json", default_json_version + end + end + + gemfile <<-G + source "https://gem.repo2" + source "https://gem.repo4" + + gem "foo" + G + end + + it "works in standalone mode", :bundler => "< 3" do + bundle "install --standalone", :artifice => "compact_index" + end + end + context "with source affinity" do context "with sources given by a block" do before do From 2edf9fa23a78da59b5ff0d67a16037aecd38a00f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Fri, 25 Aug 2023 15:50:45 +0200 Subject: [PATCH 050/208] [rubygems/rubygems] Remove redundant checks https://github.com/rubygems/rubygems/commit/d66815633b --- lib/bundler/source/rubygems.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 7a80ec2b00e52e..7cd24092c93112 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -237,7 +237,7 @@ def add_remote(source) end def spec_names - if @allow_remote && dependency_api_available? + if dependency_api_available? remote_specs.spec_names else [] @@ -245,7 +245,7 @@ def spec_names end def unmet_deps - if @allow_remote && dependency_api_available? + if dependency_api_available? remote_specs.unmet_dependency_names else [] @@ -260,7 +260,6 @@ def fetchers end def double_check_for(unmet_dependency_names) - return unless @allow_remote return unless dependency_api_available? unmet_dependency_names = unmet_dependency_names.call From 80f35d96ae60fdf238ff62982094b4b4dbd13ecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Fri, 25 Aug 2023 16:18:01 +0200 Subject: [PATCH 051/208] [rubygems/rubygems] Don't check for circular deps on full index sources https://github.com/rubygems/rubygems/commit/d275cdccb1 --- lib/bundler/resolver.rb | 12 ++++++++++-- spec/bundler/resolver/basic_spec.rb | 23 +++++++++++++++++++++++ spec/bundler/support/indexes.rb | 10 ++++++++-- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index fa95ff879b981b..3263913b7ff9d4 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -37,8 +37,16 @@ def setup_solver root_version = Resolver::Candidate.new(0) @all_specs = Hash.new do |specs, name| - matches = source_for(name).specs.search(name) - matches = filter_invalid_self_dependencies(matches, name) + source = source_for(name) + matches = source.specs.search(name) + + # Don't bother to check for circular deps when no dependency API are + # available, since it's too slow to be usable. That edge case won't work + # but resolution other than that should work fine and reasonably fast. + if source.respond_to?(:dependency_api_available?) && source.dependency_api_available? + matches = filter_invalid_self_dependencies(matches, name) + end + specs[name] = matches.sort_by {|s| [s.version, s.platform.to_s] } end diff --git a/spec/bundler/resolver/basic_spec.rb b/spec/bundler/resolver/basic_spec.rb index f739f8c02bb2fd..151d10c61cd1e5 100644 --- a/spec/bundler/resolver/basic_spec.rb +++ b/spec/bundler/resolver/basic_spec.rb @@ -347,4 +347,27 @@ should_resolve_as %w[rack-3.0.0 standalone_migrations-1.0.13] end + + it "does not ignore versions that incorrectly depend on themselves when dependency_api is not available" do + @index = build_index do + gem "rack", "3.0.0" + + gem "standalone_migrations", "7.1.0" do + dep "rack", "~> 2.0" + end + + gem "standalone_migrations", "2.0.4" do + dep "standalone_migrations", ">= 2.0.5" + end + + gem "standalone_migrations", "1.0.13" do + dep "rack", ">= 0" + end + end + + dep "rack", "~> 3.0" + dep "standalone_migrations" + + should_resolve_without_dependency_api %w[rack-3.0.0 standalone_migrations-2.0.4] + end end diff --git a/spec/bundler/support/indexes.rb b/spec/bundler/support/indexes.rb index 78372302f1b7d8..14f515f870b084 100644 --- a/spec/bundler/support/indexes.rb +++ b/spec/bundler/support/indexes.rb @@ -14,9 +14,9 @@ def platform(*args) alias_method :platforms, :platform - def resolve(args = []) + def resolve(args = [], dependency_api_available: true) @platforms ||= ["ruby"] - default_source = instance_double("Bundler::Source::Rubygems", :specs => @index, :to_s => "locally install gems") + default_source = instance_double("Bundler::Source::Rubygems", :specs => @index, :to_s => "locally install gems", :dependency_api_available? => dependency_api_available) source_requirements = { :default => default_source } base = args[0] || Bundler::SpecSet.new([]) base.each {|ls| ls.source = default_source } @@ -41,6 +41,12 @@ def should_resolve_as(specs) expect(got).to eq(specs.sort) end + def should_resolve_without_dependency_api(specs) + got = resolve(:dependency_api_available => false) + got = got.map(&:full_name).sort + expect(got).to eq(specs.sort) + end + def should_resolve_and_include(specs, args = []) got = resolve(args) got = got.map(&:full_name).sort From 69d9fda9f5b579c6992621f4cd165cd3ca7b4b3e Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Tue, 22 Aug 2023 20:31:16 +0200 Subject: [PATCH 052/208] [ruby/openssl] Remove the pending logics by the pend_on_openssl_issue_21493. Because we will add a workaround to avoid this issue. https://github.com/ruby/openssl/commit/d157ba1d3b --- test/openssl/test_pkey.rb | 6 ------ test/openssl/utils.rb | 16 ---------------- 2 files changed, 22 deletions(-) diff --git a/test/openssl/test_pkey.rb b/test/openssl/test_pkey.rb index da3ae5d67d2ebf..5fe37e2d6426d1 100644 --- a/test/openssl/test_pkey.rb +++ b/test/openssl/test_pkey.rb @@ -82,8 +82,6 @@ def test_hmac_sign_verify end def test_ed25519 - pend_on_openssl_issue_21493 - # Test vector from RFC 8032 Section 7.1 TEST 2 priv_pem = <<~EOF -----BEGIN PRIVATE KEY----- @@ -148,8 +146,6 @@ def test_ed25519 end def test_x25519 - pend_on_openssl_issue_21493 - # Test vector from RFC 7748 Section 6.1 alice_pem = <<~EOF -----BEGIN PRIVATE KEY----- @@ -202,8 +198,6 @@ def raw_initialize end def test_compare? - pend_on_openssl_issue_21493 - key1 = Fixtures.pkey("rsa1024") key2 = Fixtures.pkey("rsa1024") key3 = Fixtures.pkey("rsa2048") diff --git a/test/openssl/utils.rb b/test/openssl/utils.rb index b5ffbe1c999e95..3d4d05fe02ebc7 100644 --- a/test/openssl/utils.rb +++ b/test/openssl/utils.rb @@ -144,22 +144,6 @@ def libressl?(major = nil, minor = nil, fix = nil) return false unless version !major || (version.map(&:to_i) <=> [major, minor, fix]) >= 0 end - - # OpenSSL 3: x25519 a decode from and then encode to a pem file corrupts the - # key if fips+base provider is used - # This issue happens in OpenSSL between 3.0,0 and 3.0.10 or between 3.1.0 and - # 3.1.2. - # https://github.com/openssl/openssl/issues/21493 - # https://github.com/openssl/openssl/pull/21519 - def pend_on_openssl_issue_21493 - if OpenSSL.fips_mode && - ( - (openssl?(3, 0, 0, 0) && !openssl?(3, 0, 0, 11)) || - (openssl?(3, 1, 0, 0) && !openssl?(3, 1, 0, 3)) - ) - pend('See ') - end - end end class OpenSSL::TestCase < Test::Unit::TestCase From b0ec1db8a72c530460abd9462ac75845362886bd Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Thu, 24 Aug 2023 19:04:23 +0200 Subject: [PATCH 053/208] [ruby/openssl] ossl_pkey.c: Workaround: Decode with non-zero selections. This is a workaround for the decoding issue in ossl_pkey_read_generic(). The issue happens in the case that a key management provider is different from a decoding provider. Try all the non-zero selections in order, instead of selection 0 for OpenSSL 3 to avoid the issue. https://github.com/ruby/openssl/commit/db688fa739 --- ext/openssl/ossl_pkey.c | 111 +++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c index 6d73d259c49ac5..013412c27f5329 100644 --- a/ext/openssl/ossl_pkey.c +++ b/ext/openssl/ossl_pkey.c @@ -82,30 +82,62 @@ ossl_pkey_new(EVP_PKEY *pkey) #if OSSL_OPENSSL_PREREQ(3, 0, 0) # include -EVP_PKEY * -ossl_pkey_read_generic(BIO *bio, VALUE pass) +static EVP_PKEY * +ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass) { void *ppass = (void *)pass; OSSL_DECODER_CTX *dctx; EVP_PKEY *pkey = NULL; int pos = 0, pos2; - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", NULL, NULL, 0, NULL, NULL); + dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, input_type, NULL, NULL, + selection, NULL, NULL); if (!dctx) goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) - goto out; - - /* First check DER */ - if (OSSL_DECODER_from_bio(dctx, bio) == 1) + if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, + ppass) != 1) goto out; + while (1) { + if (OSSL_DECODER_from_bio(dctx, bio) == 1) + goto out; + if (BIO_eof(bio)) + break; + pos2 = BIO_tell(bio); + if (pos2 < 0 || pos2 <= pos) + break; + ossl_clear_error(); + pos = pos2; + } + out: OSSL_BIO_reset(bio); + OSSL_DECODER_CTX_free(dctx); + return pkey; +} +EVP_PKEY * +ossl_pkey_read_generic(BIO *bio, VALUE pass) +{ + EVP_PKEY *pkey = NULL; + /* First check DER, then check PEM. */ + const char *input_types[] = {"DER", "PEM"}; + int input_type_num = (int)(sizeof(input_types) / sizeof(char *)); /* - * Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed. + * Non-zero selections to try to decode. + * + * See EVP_PKEY_fromdata(3) - Selections to see all the selections. * - * First check for private key formats. This is to keep compatibility with - * ruby/openssl < 3.0 which decoded the following as a private key. + * This is a workaround for the decoder failing to decode or returning + * bogus keys with selection 0, if a key management provider is different + * from a decoder provider. The workaround is to avoid using selection 0. + * + * Affected OpenSSL versions: >= 3.1.0, <= 3.1.2, or >= 3.0.0, <= 3.0.10 + * Fixed OpenSSL versions: 3.2, next release of the 3.1.z and 3.0.z + * + * See https://github.com/openssl/openssl/pull/21519 for details. + * + * First check for private key formats (EVP_PKEY_KEYPAIR). This is to keep + * compatibility with ruby/openssl < 3.0 which decoded the following as a + * private key. * * $ openssl ecparam -name prime256v1 -genkey -outform PEM * -----BEGIN EC PARAMETERS----- @@ -126,50 +158,25 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass) * * Note that we need to create the OSSL_DECODER_CTX variable each time when * we use the different selection as a workaround. - * https://github.com/openssl/openssl/issues/20657 + * See https://github.com/openssl/openssl/issues/20657 for details. */ - OSSL_DECODER_CTX_free(dctx); - dctx = NULL; - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, - EVP_PKEY_KEYPAIR, NULL, NULL); - if (!dctx) - goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) - goto out; - while (1) { - if (OSSL_DECODER_from_bio(dctx, bio) == 1) - goto out; - if (BIO_eof(bio)) - break; - pos2 = BIO_tell(bio); - if (pos2 < 0 || pos2 <= pos) - break; - ossl_clear_error(); - pos = pos2; - } - - OSSL_BIO_reset(bio); - OSSL_DECODER_CTX_free(dctx); - dctx = NULL; - dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, 0, NULL, NULL); - if (!dctx) - goto out; - if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1) - goto out; - while (1) { - if (OSSL_DECODER_from_bio(dctx, bio) == 1) - goto out; - if (BIO_eof(bio)) - break; - pos2 = BIO_tell(bio); - if (pos2 < 0 || pos2 <= pos) - break; - ossl_clear_error(); - pos = pos2; + int selections[] = { + EVP_PKEY_KEYPAIR, + EVP_PKEY_KEY_PARAMETERS, + EVP_PKEY_PUBLIC_KEY + }; + int selection_num = (int)(sizeof(selections) / sizeof(int)); + int i, j; + + for (i = 0; i < input_type_num; i++) { + for (j = 0; j < selection_num; j++) { + pkey = ossl_pkey_read(bio, input_types[i], selections[j], pass); + if (pkey) { + goto out; + } + } } - out: - OSSL_DECODER_CTX_free(dctx); return pkey; } #else From 23eb13d49daccd06e56d82884ac4bfd433f1b7be Mon Sep 17 00:00:00 2001 From: Imir Kiyamov Date: Sun, 27 Aug 2023 21:45:57 +0400 Subject: [PATCH 054/208] [rubygems/rubygems] Fixed malformed lockfile version on installing https://github.com/rubygems/rubygems/commit/c969a192bf --- lib/bundler/self_manager.rb | 2 ++ spec/bundler/runtime/self_management_spec.rb | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/lib/bundler/self_manager.rb b/lib/bundler/self_manager.rb index 5d7546e9262965..1925a266d977db 100644 --- a/lib/bundler/self_manager.rb +++ b/lib/bundler/self_manager.rb @@ -170,6 +170,8 @@ def lockfile_version parsed_version = Bundler::LockfileParser.bundled_with @lockfile_version = parsed_version ? Gem::Version.new(parsed_version) : nil + rescue ArgumentError + @lockfile_version = nil end def restart_version diff --git a/spec/bundler/runtime/self_management_spec.rb b/spec/bundler/runtime/self_management_spec.rb index 976437c332c553..3e49db4f32c3f7 100644 --- a/spec/bundler/runtime/self_management_spec.rb +++ b/spec/bundler/runtime/self_management_spec.rb @@ -129,6 +129,13 @@ expect(out).to eq(Bundler::VERSION[0] == "2" ? "Bundler version #{Bundler::VERSION}" : Bundler::VERSION) end + it "ignores malformed lockfile version" do + lockfile_bundled_with("2.3.") + + bundle "install --verbose" + expect(out).to include("Using bundler #{Bundler::VERSION}") + end + private def lockfile_bundled_with(version) From 4221d9695dd71707f3d4e648c1ff397d2515a793 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 28 Aug 2023 00:50:19 -0700 Subject: [PATCH 055/208] Sort repository names in sync_default_gems --- tool/sync_default_gems.rb | 102 +++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 1cb58469c86c8e..098e3873782b62 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -11,76 +11,76 @@ module SyncDefaultGems module_function REPOSITORIES = { - rubygems: 'rubygems/rubygems', - rdoc: 'ruby/rdoc', - reline: 'ruby/reline', - json: 'flori/json', - psych: 'ruby/psych', - fileutils: 'ruby/fileutils', - fiddle: 'ruby/fiddle', - stringio: 'ruby/stringio', "io-console": 'ruby/io-console', "io-nonblock": 'ruby/io-nonblock', "io-wait": 'ruby/io-wait', + "net-http": "ruby/net-http", + "net-protocol": "ruby/net-protocol", + "open-uri": "ruby/open-uri", + "resolv-replace": "ruby/resolv-replace", + English: "ruby/English", + abbrev: "ruby/abbrev", + base64: "ruby/base64", + benchmark: "ruby/benchmark", + bigdecimal: "ruby/bigdecimal", + cgi: "ruby/cgi", csv: 'ruby/csv', - etc: 'ruby/etc', date: 'ruby/date', - zlib: 'ruby/zlib', + delegate: "ruby/delegate", + did_you_mean: "ruby/did_you_mean", + digest: "ruby/digest", + drb: "ruby/drb", + erb: "ruby/erb", + error_highlight: "ruby/error_highlight", + etc: 'ruby/etc', fcntl: 'ruby/fcntl', - strscan: 'ruby/strscan', + fiddle: 'ruby/fiddle', + fileutils: 'ruby/fileutils', + find: "ruby/find", + forwardable: "ruby/forwardable", + getoptlong: "ruby/getoptlong", ipaddr: 'ruby/ipaddr', - logger: 'ruby/logger', - ostruct: 'ruby/ostruct', irb: 'ruby/irb', - forwardable: "ruby/forwardable", + json: 'flori/json', + logger: 'ruby/logger', mutex_m: "ruby/mutex_m", - singleton: "ruby/singleton", - open3: "ruby/open3", - getoptlong: "ruby/getoptlong", - pstore: "ruby/pstore", - delegate: "ruby/delegate", - benchmark: "ruby/benchmark", - cgi: "ruby/cgi", - readline: "ruby/readline", + nkf: "ruby/nkf", observer: "ruby/observer", - timeout: "ruby/timeout", - yaml: "ruby/yaml", - uri: "ruby/uri", + open3: "ruby/open3", openssl: "ruby/openssl", - did_you_mean: "ruby/did_you_mean", - weakref: "ruby/weakref", - tempfile: "ruby/tempfile", - tmpdir: "ruby/tmpdir", - English: "ruby/English", - "net-protocol": "ruby/net-protocol", - "net-http": "ruby/net-http", - bigdecimal: "ruby/bigdecimal", optparse: "ruby/optparse", - set: "ruby/set", - find: "ruby/find", + ostruct: 'ruby/ostruct', + pathname: "ruby/pathname", + pp: "ruby/pp", + prettyprint: "ruby/prettyprint", + pstore: "ruby/pstore", + psych: 'ruby/psych', + rdoc: 'ruby/rdoc', + readline: "ruby/readline", + reline: 'ruby/reline', + resolv: "ruby/resolv", rinda: "ruby/rinda", - erb: "ruby/erb", - nkf: "ruby/nkf", - tsort: "ruby/tsort", - abbrev: "ruby/abbrev", + rubygems: 'rubygems/rubygems', + securerandom: "ruby/securerandom", + set: "ruby/set", shellwords: "ruby/shellwords", - base64: "ruby/base64", + singleton: "ruby/singleton", + stringio: 'ruby/stringio', + strscan: 'ruby/strscan', + syntax_suggest: ["ruby/syntax_suggest", "main"], syslog: "ruby/syslog", - "open-uri": "ruby/open-uri", - securerandom: "ruby/securerandom", - resolv: "ruby/resolv", - "resolv-replace": "ruby/resolv-replace", + tempfile: "ruby/tempfile", time: "ruby/time", - pp: "ruby/pp", - prettyprint: "ruby/prettyprint", - drb: "ruby/drb", - pathname: "ruby/pathname", - digest: "ruby/digest", - error_highlight: "ruby/error_highlight", - syntax_suggest: ["ruby/syntax_suggest", "main"], + timeout: "ruby/timeout", + tmpdir: "ruby/tmpdir", + tsort: "ruby/tsort", un: "ruby/un", + uri: "ruby/uri", + weakref: "ruby/weakref", win32ole: "ruby/win32ole", + yaml: "ruby/yaml", yarp: ["ruby/yarp", "main"], + zlib: 'ruby/zlib', } CLASSICAL_DEFAULT_BRANCH = "master" From 94275d96a86a411d632d1746a8c017bc7939a972 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 28 Aug 2023 00:50:49 -0700 Subject: [PATCH 056/208] Add missing racc support to sync_default_gems --- tool/sync_default_gems.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 098e3873782b62..4320825c9da7b1 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -55,6 +55,7 @@ module SyncDefaultGems prettyprint: "ruby/prettyprint", pstore: "ruby/pstore", psych: 'ruby/psych', + racc: 'ruby/racc', rdoc: 'ruby/rdoc', readline: "ruby/readline", reline: 'ruby/reline', From f5da7c379bc287a1e19200358c917ddc5628351e Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 28 Aug 2023 00:52:52 -0700 Subject: [PATCH 057/208] Ignore test/regress/ for racc --- tool/sync_default_gems.rb | 62 ++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 4320825c9da7b1..457e927c03aa20 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -424,26 +424,43 @@ def sync_default_gems(gem) replace_rdoc_ref_all end - IGNORE_FILE_PATTERN = - /\A(?:[A-Z]\w*\.(?:md|txt) - |[^\/]+\.yml - |\.git.* - |[A-Z]\w+file - |COPYING - |Gemfile.lock - |bin\/.* - |rakelib\/.* - |test\/lib\/.* - )\z/mx - - YARP_IGNORE_FILE_PATTERN = - /\A(?:Makefile\.in - |configure\.ac - |fuzz\/.* - |rust\/.* - |tasks\/.* - |ext\/yarp\/extconf\.rb - )\z/mx + def ignore_file_pattern_for(gem) + patterns = [] + + # Common patterns + patterns << %r[\A(?: + [A-Z]\w*\.(?:md|txt) + |[^/]+\.yml + |\.git.* + |[A-Z]\w+file + |COPYING + |Gemfile.lock + |bin/.* + |rakelib/.* + |test/lib/.* + )\z]mx + + # Gem-specific patterns + case gem + when "racc" + %r[\A(?: + test/regress/.* + )\z]mx + when "yarp" + %r[\A(?: + Makefile\.in + |configure\.ac + |fuzz/.* + |rust/.* + |tasks/.* + |ext/yarp/extconf\.rb + )\z]mx + end&.tap do |pattern| + patterns << pattern + end + + Regexp.union(*patterns) + end def message_filter(repo, sha, input: ARGF) log = input.read @@ -515,10 +532,7 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) end # Ignore Merge commits and already-merged commits. - ignore_file_pattern = IGNORE_FILE_PATTERN - if gem == "yarp" - ignore_file_pattern = Regexp.union(ignore_file_pattern, YARP_IGNORE_FILE_PATTERN) - end + ignore_file_pattern = ignore_file_pattern_for(gem) commits.delete_if do |sha, subject| files = pipe_readlines(%W"git diff-tree -z --no-commit-id --name-only -r #{sha}") subject.start_with?("Merge", "Auto Merge") or files.all?(ignore_file_pattern) From 4963dd6b6432549b4685e31bf52b218a728cf50c Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 28 Aug 2023 01:14:57 -0700 Subject: [PATCH 058/208] Revert racc auto-sync support It's no longer a default gem actually. Fixed the webhook side instead https://github.com/ruby/git.ruby-lang.org/commit/3c27d860b4e39d10d63280b59d4db2cec518ae13. --- tool/sync_default_gems.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 457e927c03aa20..83a2c4cbaa5dbe 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -55,7 +55,6 @@ module SyncDefaultGems prettyprint: "ruby/prettyprint", pstore: "ruby/pstore", psych: 'ruby/psych', - racc: 'ruby/racc', rdoc: 'ruby/rdoc', readline: "ruby/readline", reline: 'ruby/reline', @@ -442,10 +441,6 @@ def ignore_file_pattern_for(gem) # Gem-specific patterns case gem - when "racc" - %r[\A(?: - test/regress/.* - )\z]mx when "yarp" %r[\A(?: Makefile\.in From 00439dbdb4909f0c78b37f6ed1e3f88c4cb05d46 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 26 Aug 2023 10:31:50 +0900 Subject: [PATCH 059/208] sync_default_gems.rb: convert keys of REPOSITORIES to strings Referencing always after conversion to a symbol, and yielded gem name is always converted to a string. --- tool/sync_default_gems.rb | 18 +++++++++--------- tool/test/test_sync_default_gems.rb | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 83a2c4cbaa5dbe..ad52c3faffb1a3 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -81,13 +81,13 @@ module SyncDefaultGems yaml: "ruby/yaml", yarp: ["ruby/yarp", "main"], zlib: 'ruby/zlib', - } + }.transform_keys {|k| k.to_s} CLASSICAL_DEFAULT_BRANCH = "master" class << REPOSITORIES def [](gem) - repo, branch = super + repo, branch = super(gem) return repo, branch || CLASSICAL_DEFAULT_BRANCH end @@ -131,7 +131,7 @@ def replace_rdoc_ref_all # We usually don't use this. Please consider using #sync_default_gems_with_commits instead. def sync_default_gems(gem) - repo, = REPOSITORIES[gem.to_sym] + repo, = REPOSITORIES[gem] puts "Sync #{repo}" upstream = File.join("..", "..", repo) @@ -497,7 +497,7 @@ def message_filter(repo, sha, input: ARGF) # @param ranges [Array] "before..after". Note that it will NOT sync "before" (but commits after that). # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem.to_sym] + repo, default_branch = REPOSITORIES[gem] puts "Sync #{repo} with commit history." # Fetch the repository to be synchronized @@ -740,7 +740,7 @@ def sync_lib(repo, upstream = nil) def update_default_gems(gem, release: false) - repository, default_branch = REPOSITORIES[gem.to_sym] + repository, default_branch = REPOSITORIES[gem] author, repository = repository.split('/') puts "Update #{author}/#{repository}" @@ -778,16 +778,16 @@ def update_default_gems(gem, release: false) if ARGV[1] update_default_gems(ARGV[1]) else - REPOSITORIES.each_key {|gem| update_default_gems(gem.to_s)} + REPOSITORIES.each_key {|gem| update_default_gems(gem)} end when "all" if ARGV[1] == "release" REPOSITORIES.each_key do |gem| - update_default_gems(gem.to_s, release: true) - sync_default_gems(gem.to_s) + update_default_gems(gem, release: true) + sync_default_gems(gem) end else - REPOSITORIES.each_key {|gem| sync_default_gems(gem.to_s)} + REPOSITORIES.each_key {|gem| sync_default_gems(gem)} end when "list" ARGV.shift diff --git a/tool/test/test_sync_default_gems.rb b/tool/test/test_sync_default_gems.rb index e6c654ed55d914..a73fc65d6e6b48 100755 --- a/tool/test/test_sync_default_gems.rb +++ b/tool/test/test_sync_default_gems.rb @@ -87,7 +87,7 @@ def setup system(*%W"git config --global user.name", "Ruby") system(*%W"git config --global init.defaultBranch default") @target = "sync-test" - SyncDefaultGems::REPOSITORIES[@target.to_sym] = ["ruby/#{@target}", "default"] + SyncDefaultGems::REPOSITORIES[@target] = ["ruby/#{@target}", "default"] @sha = {} @origdir = Dir.pwd Dir.chdir(@testdir) @@ -113,7 +113,7 @@ def setup def teardown if @target Dir.chdir(@origdir) - SyncDefaultGems::REPOSITORIES.delete(@target.to_sym) + SyncDefaultGems::REPOSITORIES.delete(@target) ENV.update(@git_config) FileUtils.rm_rf(@testdir) end From 06f5d8f3a04e147fd757f2ed15882592bb66e3a9 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 25 Aug 2023 16:04:55 -0400 Subject: [PATCH 060/208] [ruby/yarp] Improve how we declare ripper exceptions in parse_test.rb Specific files are named earlier in the block, and we now have the ability to skip just the lex matching, or skip ripper entirely (for files that don't parse). https://github.com/ruby/yarp/commit/dcd3806dca --- test/yarp/parse_test.rb | 44 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index b9f852abf934f4..70f061242ba1f2 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -46,7 +46,7 @@ def test_parse_lex_file # To accurately compare against Ripper, we need to make sure that we're # running on Ruby 3.2+. - check_ripper = RUBY_VERSION >= "3.2.0" + ripper_enabled = RUBY_VERSION >= "3.2.0" # The FOCUS environment variable allows you to specify one particular fixture # to test, instead of all of them. @@ -63,6 +63,22 @@ def test_parse_lex_file directory = File.dirname(snapshot) FileUtils.mkdir_p(directory) unless File.directory?(directory) + ripper_should_parse = ripper_should_match = ripper_enabled + + # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if + # we're on an earlier version. + ripper_should_match = false if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" + + # It seems like there are some oddities with nested heredocs and ripper. + # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. + ripper_should_match = false if relative == "seattlerb/heredoc_nested.txt" + + # Ripper seems to have a bug that the regex portions before and after the heredoc are combined + # into a single token. See https://bugs.ruby-lang.org/issues/19838. + # + # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. + ripper_should_match = false if relative == "wrapping_heredoc.txt" + define_method "test_filepath_#{relative}" do # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, # and explicitly set the external encoding to UTF-8 to override the binmode default. @@ -70,7 +86,7 @@ def test_parse_lex_file # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture # that is invalid Ruby. - refute_nil Ripper.sexp_raw(source) if check_ripper + refute_nil(Ripper.sexp_raw(source), "Ripper failed to parse") if ripper_should_parse # Next, assert that there were no errors during parsing. result = YARP.parse(source, relative) @@ -118,25 +134,13 @@ def test_parse_lex_file assert_equal expected_newlines, YARP.const_get(:Debug).newlines(source) - # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if - # we're on an earlier version. - return if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" - - # It seems like there are some oddities with nested heredocs and ripper. - # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. - return if relative == "seattlerb/heredoc_nested.txt" - - # Ripper seems to have a bug that the regex portions before and after the heredoc are combined - # into a single token. - return if relative == "wrapping_heredoc.txt" - - # Finally, assert that we can lex the source and get the same tokens as - # Ripper. - lex_result = YARP.lex_compat(source) - assert_equal [], lex_result.errors - tokens = lex_result.value + if ripper_should_parse && ripper_should_match + # Finally, assert that we can lex the source and get the same tokens as + # Ripper. + lex_result = YARP.lex_compat(source) + assert_equal [], lex_result.errors + tokens = lex_result.value - if check_ripper begin YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| assert_equal ripper, yarp From 2b9a05374032b3d287b91e6307d2817b6c962612 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 27 Aug 2023 15:54:45 -0400 Subject: [PATCH 061/208] [ruby/yarp] fix: yp_interpolated_symbol_node_append Made this function's behavior match the interpolated_string implementation. Previously, the start location was not set and left as 0. https://github.com/ruby/yarp/commit/87f348889f --- yarp/yarp.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yarp/yarp.c b/yarp/yarp.c index ba8722a9250a80..129a176decbdef 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2816,10 +2816,11 @@ yp_interpolated_symbol_node_create(yp_parser_t *parser, const yp_token_t *openin static inline void yp_interpolated_symbol_node_append(yp_interpolated_symbol_node_t *node, yp_node_t *part) { - yp_node_list_append(&node->parts, part); - if (!node->base.location.start) { + if (node->parts.size == 0 && node->opening_loc.start == NULL) { node->base.location.start = part->location.start; } + + yp_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } From 8926fd20f612aa6bae77844b8b6f5d14bf659cb1 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 27 Aug 2023 16:19:12 -0400 Subject: [PATCH 062/208] [ruby/yarp] test: backfill tests for %q/%Q spanning a heredoc Also rename the fixture file https://github.com/ruby/yarp/commit/c148d955fd --- test/yarp/fixtures/spanning_heredoc.txt | 25 +++++ test/yarp/fixtures/wrapping_heredoc.txt | 13 --- test/yarp/parse_test.rb | 2 +- test/yarp/snapshots/spanning_heredoc.txt | 115 +++++++++++++++++++++++ test/yarp/snapshots/wrapping_heredoc.txt | 80 ---------------- 5 files changed, 141 insertions(+), 94 deletions(-) create mode 100644 test/yarp/fixtures/spanning_heredoc.txt delete mode 100644 test/yarp/fixtures/wrapping_heredoc.txt create mode 100644 test/yarp/snapshots/spanning_heredoc.txt delete mode 100644 test/yarp/snapshots/wrapping_heredoc.txt diff --git a/test/yarp/fixtures/spanning_heredoc.txt b/test/yarp/fixtures/spanning_heredoc.txt new file mode 100644 index 00000000000000..5040840e7654e7 --- /dev/null +++ b/test/yarp/fixtures/spanning_heredoc.txt @@ -0,0 +1,25 @@ +# test regex, string, and lists that span a heredoc thanks to an escaped newline + +# ripper incorrectly creates a "b\nb" token instead of two separate string tokens +pp <<-A.gsub(/b\ +a +A +b/, "") + +# ripper incorrectly creates a "d\nd" token instead of two separate string tokens +pp <<-A, "d\ +c +A +d" + +# ripper gets this right +pp <<-A, %q[f\ +e +A +f] + +# ripper incorrectly creates a "h\nh" token instead of two separate string tokens +pp <<-A, %Q[h\ +g +A +h] diff --git a/test/yarp/fixtures/wrapping_heredoc.txt b/test/yarp/fixtures/wrapping_heredoc.txt deleted file mode 100644 index d5fc7101780c21..00000000000000 --- a/test/yarp/fixtures/wrapping_heredoc.txt +++ /dev/null @@ -1,13 +0,0 @@ -# test regex, string, and lists that wrap a heredoc thanks to an escaped newline - -# ripper incorrectly creates a "b\nc" string instead of two separate string tokens -pp <<-A.gsub(/b\ -a -A -c/, "") - -# ripper incorrectly creates a "e\nf" string instead of two separate string tokens -pp <<-A + "e\ -d -A -f" diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index 70f061242ba1f2..1eb0033208adc9 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -77,7 +77,7 @@ def test_parse_lex_file # into a single token. See https://bugs.ruby-lang.org/issues/19838. # # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. - ripper_should_match = false if relative == "wrapping_heredoc.txt" + ripper_should_match = false if relative == "spanning_heredoc.txt" define_method "test_filepath_#{relative}" do # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, diff --git a/test/yarp/snapshots/spanning_heredoc.txt b/test/yarp/snapshots/spanning_heredoc.txt new file mode 100644 index 00000000000000..244ced125b58e2 --- /dev/null +++ b/test/yarp/snapshots/spanning_heredoc.txt @@ -0,0 +1,115 @@ +ProgramNode(164...448)( + [], + StatementsNode(164...448)( + [CallNode(164...192)( + nil, + nil, + (164...166), + nil, + ArgumentsNode(167...192)( + [CallNode(167...192)( + InterpolatedStringNode(167...171)( + (167...171), + [StringNode(181...183)(nil, (181...183), nil, "a\n")], + (183...185) + ), + (171...172), + (172...176), + (176...177), + ArgumentsNode(177...191)( + [InterpolatedRegularExpressionNode(177...187)( + (177...178), + [StringNode(178...181)(nil, (178...181), nil, "b"), + StringNode(185...186)(nil, (185...186), nil, "b")], + (186...187), + 0 + ), + StringNode(189...191)( + (189...190), + (190...190), + (190...191), + "" + )] + ), + (191...192), + nil, + 0, + "gsub" + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(276...295)( + nil, + nil, + (276...278), + nil, + ArgumentsNode(279...295)( + [InterpolatedStringNode(279...283)( + (279...283), + [StringNode(289...291)(nil, (289...291), nil, "c\n")], + (291...293) + ), + InterpolatedStringNode(285...295)( + (285...286), + [StringNode(286...289)(nil, (286...289), nil, "d"), + StringNode(293...294)(nil, (293...294), nil, "d")], + (294...295) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(322...343)( + nil, + nil, + (322...324), + nil, + ArgumentsNode(325...343)( + [InterpolatedStringNode(325...329)( + (325...329), + [StringNode(337...339)(nil, (337...339), nil, "e\n")], + (339...341) + ), + InterpolatedStringNode(331...343)( + (331...334), + [StringNode(334...337)(nil, (334...337), nil, "f\\\n"), + StringNode(341...342)(nil, (341...342), nil, "f")], + (342...343) + )] + ), + nil, + nil, + 0, + "pp" + ), + CallNode(427...448)( + nil, + nil, + (427...429), + nil, + ArgumentsNode(430...448)( + [InterpolatedStringNode(430...434)( + (430...434), + [StringNode(442...444)(nil, (442...444), nil, "g\n")], + (444...446) + ), + InterpolatedStringNode(436...448)( + (436...439), + [StringNode(439...442)(nil, (439...442), nil, "h"), + StringNode(446...447)(nil, (446...447), nil, "h")], + (447...448) + )] + ), + nil, + nil, + 0, + "pp" + )] + ) +) diff --git a/test/yarp/snapshots/wrapping_heredoc.txt b/test/yarp/snapshots/wrapping_heredoc.txt deleted file mode 100644 index 674db56ed12546..00000000000000 --- a/test/yarp/snapshots/wrapping_heredoc.txt +++ /dev/null @@ -1,80 +0,0 @@ -ProgramNode(165...298)( - [], - StatementsNode(165...298)( - [CallNode(165...193)( - nil, - nil, - (165...167), - nil, - ArgumentsNode(168...193)( - [CallNode(168...193)( - InterpolatedStringNode(168...172)( - (168...172), - [StringNode(182...184)(nil, (182...184), nil, "a\n")], - (184...186) - ), - (172...173), - (173...177), - (177...178), - ArgumentsNode(178...192)( - [InterpolatedRegularExpressionNode(178...188)( - (178...179), - [StringNode(179...182)(nil, (179...182), nil, "b"), - StringNode(186...187)(nil, (186...187), nil, "c")], - (187...188), - 0 - ), - StringNode(190...192)( - (190...191), - (191...191), - (191...192), - "" - )] - ), - (192...193), - nil, - 0, - "gsub" - )] - ), - nil, - nil, - 0, - "pp" - ), - CallNode(278...298)( - nil, - nil, - (278...280), - nil, - ArgumentsNode(281...298)( - [CallNode(281...298)( - InterpolatedStringNode(281...285)( - (281...285), - [StringNode(292...294)(nil, (292...294), nil, "d\n")], - (294...296) - ), - nil, - (286...287), - nil, - ArgumentsNode(288...298)( - [InterpolatedStringNode(288...298)( - (288...289), - [StringNode(289...292)(nil, (289...292), nil, "e"), - StringNode(296...297)(nil, (296...297), nil, "f")], - (297...298) - )] - ), - nil, - nil, - 0, - "+" - )] - ), - nil, - nil, - 0, - "pp" - )] - ) -) From 77e971b6ec93810842a8ef320e412174267dab6d Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 27 Aug 2023 16:32:19 -0400 Subject: [PATCH 063/208] [ruby/yarp] fix: %w list spanning a heredoc Two fixes were necessary: - ensure we are handling newlines correctly - accept two consecutive string tokens without a separator https://github.com/ruby/yarp/commit/4e707937cb Co-authored-by: Kevin Newton --- test/yarp/fixtures/spanning_heredoc.txt | 6 ++++++ test/yarp/parse_test.rb | 2 +- test/yarp/snapshots/spanning_heredoc.txt | 27 ++++++++++++++++++++++-- yarp/yarp.c | 25 ++++++++++++++-------- 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/test/yarp/fixtures/spanning_heredoc.txt b/test/yarp/fixtures/spanning_heredoc.txt index 5040840e7654e7..1b17edac5d7fc3 100644 --- a/test/yarp/fixtures/spanning_heredoc.txt +++ b/test/yarp/fixtures/spanning_heredoc.txt @@ -23,3 +23,9 @@ pp <<-A, %Q[h\ g A h] + +# ripper can't parse this successfully, though ruby runs it correctly +pp <<-A, %w[j\ +i +A +j] diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index 1eb0033208adc9..b288d597b23eb9 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -77,7 +77,7 @@ def test_parse_lex_file # into a single token. See https://bugs.ruby-lang.org/issues/19838. # # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. - ripper_should_match = false if relative == "spanning_heredoc.txt" + ripper_should_parse = false if relative == "spanning_heredoc.txt" define_method "test_filepath_#{relative}" do # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, diff --git a/test/yarp/snapshots/spanning_heredoc.txt b/test/yarp/snapshots/spanning_heredoc.txt index 244ced125b58e2..f948e1e9d3d6ab 100644 --- a/test/yarp/snapshots/spanning_heredoc.txt +++ b/test/yarp/snapshots/spanning_heredoc.txt @@ -1,6 +1,6 @@ -ProgramNode(164...448)( +ProgramNode(164...541)( [], - StatementsNode(164...448)( + StatementsNode(164...541)( [CallNode(164...192)( nil, nil, @@ -110,6 +110,29 @@ ProgramNode(164...448)( nil, 0, "pp" + ), + CallNode(520...541)( + nil, + nil, + (520...522), + nil, + ArgumentsNode(523...541)( + [InterpolatedStringNode(523...527)( + (523...527), + [StringNode(535...537)(nil, (535...537), nil, "i\n")], + (537...539) + ), + ArrayNode(529...541)( + [StringNode(532...535)(nil, (532...535), nil, "j\\\n"), + StringNode(539...540)(nil, (539...540), nil, "j")], + (529...532), + (540...541) + )] + ), + nil, + nil, + 0, + "pp" )] ) ) diff --git a/yarp/yarp.c b/yarp/yarp.c index 129a176decbdef..a4b1c9be3ca8ee 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -6942,9 +6942,19 @@ parser_lex(yp_parser_t *parser) { yp_unescape_type_t unescape_type = lex_mode->as.list.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); - // If the result is an escaped newline, then we need to - // track that newline. - yp_newline_list_check_append(&parser->newline_list, breakpoint + difference - 1); + // If the result is an escaped newline ... + if (*(breakpoint + difference - 1) == '\n') { + if (parser->heredoc_end) { + // ... if we are on the same line as a heredoc, flush the heredoc and + // continue parsing after heredoc_end. + parser->current.end = breakpoint + difference; + parser_flush_heredoc_end(parser); + LEX(YP_TOKEN_STRING_CONTENT); + } else { + // ... else track the newline. + yp_newline_list_append(&parser->newline_list, breakpoint + difference - 1); + } + } breakpoint = yp_strpbrk(parser, breakpoint + difference, breakpoints, parser->end - (breakpoint + difference)); continue; @@ -12098,12 +12108,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { accept(parser, YP_TOKEN_WORDS_SEP); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - if (yp_array_node_size(array) == 0) { - accept(parser, YP_TOKEN_WORDS_SEP); - } else { - expect(parser, YP_TOKEN_WORDS_SEP, "Expected a separator for the strings in a `%w` list."); - if (match_type_p(parser, YP_TOKEN_STRING_END)) break; - } + accept(parser, YP_TOKEN_WORDS_SEP); + if (match_type_p(parser, YP_TOKEN_STRING_END)) break; + expect(parser, YP_TOKEN_STRING_CONTENT, "Expected a string in a `%w` list."); yp_token_t opening = not_provided(parser); From 74812df4963b1ee56a015babf3798d48bae447c6 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 27 Aug 2023 16:34:53 -0400 Subject: [PATCH 064/208] [ruby/yarp] fix: %W list spanning a heredoc Primarily this fix is to accept a string node and concatenate it onto an interpolated string. https://github.com/ruby/yarp/commit/6df729fe72 --- test/yarp/fixtures/spanning_heredoc.txt | 7 ++++++ test/yarp/snapshots/spanning_heredoc.txt | 31 ++++++++++++++++++++++-- yarp/yarp.c | 13 ++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/test/yarp/fixtures/spanning_heredoc.txt b/test/yarp/fixtures/spanning_heredoc.txt index 1b17edac5d7fc3..a81731a1815444 100644 --- a/test/yarp/fixtures/spanning_heredoc.txt +++ b/test/yarp/fixtures/spanning_heredoc.txt @@ -29,3 +29,10 @@ pp <<-A, %w[j\ i A j] + +# ripper can't parse this successfully, though ruby runs it correctly +# TODO: yarp does not include the "\n" in "l\nl" in the AST like ruby does +pp <<-A, %W[l\ +k +A +l] diff --git a/test/yarp/snapshots/spanning_heredoc.txt b/test/yarp/snapshots/spanning_heredoc.txt index f948e1e9d3d6ab..a0f58a97bfbbba 100644 --- a/test/yarp/snapshots/spanning_heredoc.txt +++ b/test/yarp/snapshots/spanning_heredoc.txt @@ -1,6 +1,6 @@ -ProgramNode(164...541)( +ProgramNode(164...709)( [], - StatementsNode(164...541)( + StatementsNode(164...709)( [CallNode(164...192)( nil, nil, @@ -133,6 +133,33 @@ ProgramNode(164...541)( nil, 0, "pp" + ), + CallNode(688...709)( + nil, + nil, + (688...690), + nil, + ArgumentsNode(691...709)( + [InterpolatedStringNode(691...695)( + (691...695), + [StringNode(703...705)(nil, (703...705), nil, "k\n")], + (705...707) + ), + ArrayNode(697...709)( + [InterpolatedStringNode(700...708)( + nil, + [StringNode(700...703)(nil, (700...703), nil, "l"), + StringNode(707...708)(nil, (707...708), nil, "l")], + nil + )], + (697...700), + (708...709) + )] + ), + nil, + nil, + 0, + "pp" )] ) ) diff --git a/yarp/yarp.c b/yarp/yarp.c index a4b1c9be3ca8ee..7306d9e8e5a6c4 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -12160,6 +12160,19 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // to the list of child nodes. yp_node_t *part = parse_string_part(parser); yp_interpolated_string_node_append((yp_interpolated_string_node_t *) current, part); + } else if (YP_NODE_TYPE_P(current, YP_NODE_STRING_NODE)) { + // If we hit string content and the current node is a string node, + // then we need to convert the current node into an interpolated + // string and add the string content to the list of child nodes. + yp_token_t opening = not_provided(parser); + yp_token_t closing = not_provided(parser); + yp_interpolated_string_node_t *interpolated = + yp_interpolated_string_node_create(parser, &opening, NULL, &closing); + yp_interpolated_string_node_append(interpolated, current); + + yp_node_t *part = parse_string_part(parser); + yp_interpolated_string_node_append(interpolated, part); + current = (yp_node_t *) interpolated; } else { assert(false && "unreachable"); } From 29c5b851281da753ea0ae204afbd3f9010b466bc Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 27 Aug 2023 16:40:18 -0400 Subject: [PATCH 065/208] [ruby/yarp] fix: %i list spanning a heredoc The fix here is similar to what we did in a previous commit for %w, to accept two consecutive string tokens without a separator. https://github.com/ruby/yarp/commit/f869fbdbe5 --- test/yarp/fixtures/spanning_heredoc.txt | 6 ++++++ test/yarp/snapshots/spanning_heredoc.txt | 27 ++++++++++++++++++++++-- yarp/yarp.c | 9 ++------ 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/test/yarp/fixtures/spanning_heredoc.txt b/test/yarp/fixtures/spanning_heredoc.txt index a81731a1815444..c1852e377a5081 100644 --- a/test/yarp/fixtures/spanning_heredoc.txt +++ b/test/yarp/fixtures/spanning_heredoc.txt @@ -36,3 +36,9 @@ pp <<-A, %W[l\ k A l] + +# ripper can't parse this successfully, though ruby runs it correctly +pp <<-A, %i[n\ +m +A +n] diff --git a/test/yarp/snapshots/spanning_heredoc.txt b/test/yarp/snapshots/spanning_heredoc.txt index a0f58a97bfbbba..3d223e5e4d3c27 100644 --- a/test/yarp/snapshots/spanning_heredoc.txt +++ b/test/yarp/snapshots/spanning_heredoc.txt @@ -1,6 +1,6 @@ -ProgramNode(164...709)( +ProgramNode(164...802)( [], - StatementsNode(164...709)( + StatementsNode(164...802)( [CallNode(164...192)( nil, nil, @@ -160,6 +160,29 @@ ProgramNode(164...709)( nil, 0, "pp" + ), + CallNode(781...802)( + nil, + nil, + (781...783), + nil, + ArgumentsNode(784...802)( + [InterpolatedStringNode(784...788)( + (784...788), + [StringNode(796...798)(nil, (796...798), nil, "m\n")], + (798...800) + ), + ArrayNode(790...802)( + [SymbolNode(793...796)(nil, (793...796), nil, "n\\\n"), + SymbolNode(800...801)(nil, (800...801), nil, "n")], + (790...793), + (801...802) + )] + ), + nil, + nil, + 0, + "pp" )] ) ) diff --git a/yarp/yarp.c b/yarp/yarp.c index 7306d9e8e5a6c4..1f707696bb2580 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -11944,14 +11944,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_array_node_t *array = yp_array_node_create(parser, &parser->previous); while (!match_any_type_p(parser, 2, YP_TOKEN_STRING_END, YP_TOKEN_EOF)) { - if (yp_array_node_size(array) == 0) { - accept(parser, YP_TOKEN_WORDS_SEP); - } else { - expect(parser, YP_TOKEN_WORDS_SEP, "Expected a separator for the symbols in a `%i` list."); - if (match_type_p(parser, YP_TOKEN_STRING_END)) break; - } - + accept(parser, YP_TOKEN_WORDS_SEP); if (match_type_p(parser, YP_TOKEN_STRING_END)) break; + expect(parser, YP_TOKEN_STRING_CONTENT, "Expected a symbol in a `%i` list."); yp_token_t opening = not_provided(parser); From 9b87518ea0eff0c30c22db9c2394bb61db913646 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Sun, 27 Aug 2023 16:43:27 -0400 Subject: [PATCH 066/208] [ruby/yarp] fix: %I list spanning a heredoc Similar to the previous %W fix, we accept a symbol node and concatenate it onto an interpolated symbol. https://github.com/ruby/yarp/commit/6b5911b95e --- test/yarp/fixtures/spanning_heredoc.txt | 7 ++++++ test/yarp/snapshots/spanning_heredoc.txt | 31 ++++++++++++++++++++++-- yarp/yarp.c | 13 ++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/test/yarp/fixtures/spanning_heredoc.txt b/test/yarp/fixtures/spanning_heredoc.txt index c1852e377a5081..a52a4c3c27b37a 100644 --- a/test/yarp/fixtures/spanning_heredoc.txt +++ b/test/yarp/fixtures/spanning_heredoc.txt @@ -42,3 +42,10 @@ pp <<-A, %i[n\ m A n] + +# ripper gets this one wrong in the same way that YARP does ... +# TODO: yarp does not include the "\n" in "p\np" in the AST like ruby does +pp <<-A, %I[p\ +o +A +p] diff --git a/test/yarp/snapshots/spanning_heredoc.txt b/test/yarp/snapshots/spanning_heredoc.txt index 3d223e5e4d3c27..301c70adf2a67c 100644 --- a/test/yarp/snapshots/spanning_heredoc.txt +++ b/test/yarp/snapshots/spanning_heredoc.txt @@ -1,6 +1,6 @@ -ProgramNode(164...802)( +ProgramNode(164...964)( [], - StatementsNode(164...802)( + StatementsNode(164...964)( [CallNode(164...192)( nil, nil, @@ -183,6 +183,33 @@ ProgramNode(164...802)( nil, 0, "pp" + ), + CallNode(943...964)( + nil, + nil, + (943...945), + nil, + ArgumentsNode(946...964)( + [InterpolatedStringNode(946...950)( + (946...950), + [StringNode(958...960)(nil, (958...960), nil, "o\n")], + (960...962) + ), + ArrayNode(952...964)( + [InterpolatedSymbolNode(955...963)( + nil, + [SymbolNode(955...958)(nil, (955...958), nil, "p"), + StringNode(962...963)(nil, (962...963), nil, "p")], + nil + )], + (952...955), + (963...964) + )] + ), + nil, + nil, + 0, + "pp" )] ) ) diff --git a/yarp/yarp.c b/yarp/yarp.c index 1f707696bb2580..6f66affd78b281 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -12001,6 +12001,19 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { // to the list of child nodes. yp_node_t *part = parse_string_part(parser); yp_interpolated_symbol_node_append((yp_interpolated_symbol_node_t *) current, part); + } else if (YP_NODE_TYPE_P(current, YP_NODE_SYMBOL_NODE)) { + // If we hit string content and the current node is a string node, + // then we need to convert the current node into an interpolated + // string and add the string content to the list of child nodes. + yp_token_t opening = not_provided(parser); + yp_token_t closing = not_provided(parser); + yp_interpolated_symbol_node_t *interpolated = + yp_interpolated_symbol_node_create(parser, &opening, NULL, &closing); + yp_interpolated_symbol_node_append(interpolated, current); + + yp_node_t *part = parse_string_part(parser); + yp_interpolated_symbol_node_append(interpolated, part); + current = (yp_node_t *) interpolated; } else { assert(false && "unreachable"); } From caf48487cafb8414c60f32b902decf0689ae2093 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 29 Aug 2023 00:50:05 +1200 Subject: [PATCH 067/208] Restore `HAVE_RB_IO_T` macro for compatibility with `kgio`, `unicorn`, etc. (#8286) --- include/ruby/io.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/ruby/io.h b/include/ruby/io.h index 60029fedb6f0d6..e9dfeda5b1214b 100644 --- a/include/ruby/io.h +++ b/include/ruby/io.h @@ -138,6 +138,7 @@ struct rb_io_encoding { }; #ifndef HAVE_RB_IO_T +#define HAVE_RB_IO_T 1 /** Ruby's IO, metadata and buffers. */ struct rb_io { /** The IO's Ruby level counterpart. */ From 91de37c23ec6b048e45df79cef1cb93a86316929 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Sat, 26 Aug 2023 19:31:46 -0400 Subject: [PATCH 068/208] Remove --disable-gems in assert_in_out_err assert_in_out_err adds --disable=gems so we don't need to add --disable-gems in the args list. --- test/-ext-/bug_reporter/test_bug_reporter.rb | 3 +- test/json/json_generator_test.rb | 2 +- test/ruby/test_gc.rb | 82 ++++++++++---------- test/ruby/test_require.rb | 10 +-- test/ruby/test_rubyoptions.rb | 10 +-- test/ruby/test_thread.rb | 2 +- test/test_tempfile.rb | 3 +- 7 files changed, 55 insertions(+), 57 deletions(-) diff --git a/test/-ext-/bug_reporter/test_bug_reporter.rb b/test/-ext-/bug_reporter/test_bug_reporter.rb index d4d0de7f46d0f5..1d3d253c04ca7f 100644 --- a/test/-ext-/bug_reporter/test_bug_reporter.rb +++ b/test/-ext-/bug_reporter/test_bug_reporter.rb @@ -22,8 +22,7 @@ def test_bug_reporter_add tmpdir = Dir.mktmpdir no_core = "Process.setrlimit(Process::RLIMIT_CORE, 0); " if defined?(Process.setrlimit) && defined?(Process::RLIMIT_CORE) - args = ["--disable-gems", "-r-test-/bug_reporter", - "-C", tmpdir] + args = ["-r-test-/bug_reporter", "-C", tmpdir] args.push("--yjit") if yjit_enabled? # We want the printed description to match this process's RUBY_DESCRIPTION args.unshift({"RUBY_ON_BUG" => nil}) stdin = "#{no_core}register_sample_bug_reporter(12345); Process.kill :SEGV, $$" diff --git a/test/json/json_generator_test.rb b/test/json/json_generator_test.rb index fa15279905861d..3c4aad6528b80a 100755 --- a/test/json/json_generator_test.rb +++ b/test/json/json_generator_test.rb @@ -233,7 +233,7 @@ def test_buffer_initial_length def test_gc if respond_to?(:assert_in_out_err) && !(RUBY_PLATFORM =~ /java/) - assert_in_out_err(%w[-rjson --disable-gems], <<-EOS, [], []) + assert_in_out_err(%w[-rjson], <<-EOS, [], []) bignum_too_long_to_embed_as_string = 1234567890123456789012345 expect = bignum_too_long_to_embed_as_string.to_s GC.stress = true diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 64a860fe651626..c5a9731f8a9ae8 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -228,12 +228,12 @@ def test_stat_heap_constraints def test_latest_gc_info omit 'stress' if GC.stress - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'eom' - GC.start - count = GC.stat(:heap_free_slots) + GC.stat(:heap_allocatable_pages) * GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] - count.times{ "a" + "b" } - assert_equal :newobj, GC.latest_gc_info[:gc_by] - eom + assert_separately([], __FILE__, __LINE__, <<-'RUBY') + GC.start + count = GC.stat(:heap_free_slots) + GC.stat(:heap_allocatable_pages) * GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] + count.times{ "a" + "b" } + assert_equal :newobj, GC.latest_gc_info[:gc_by] + RUBY GC.latest_gc_info(h = {}) # allocate hash and rehearsal GC.start @@ -335,7 +335,7 @@ def test_latest_gc_info_weak_references_count end def test_stress_compile_send - assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "") + assert_in_out_err([], <<-EOS, [], [], "") GC.stress = true begin eval("A::B.c(1, 1, d: 234)") @@ -345,7 +345,7 @@ def test_stress_compile_send end def test_singleton_method - assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "[ruby-dev:42832]") + assert_in_out_err([], <<-EOS, [], [], "[ruby-dev:42832]") GC.stress = true 10.times do obj = Object.new @@ -357,7 +357,7 @@ def obj.bar() raise "obj.foo is called, but this is obj.bar" end end def test_singleton_method_added - assert_in_out_err(%w[--disable-gems], <<-EOS, [], [], "[ruby-dev:44436]") + assert_in_out_err([], <<-EOS, [], [], "[ruby-dev:44436]") class BasicObject undef singleton_method_added def singleton_method_added(mid) @@ -404,7 +404,7 @@ def test_gc_parameter "RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR" => "0.4", } # always full GC when RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR < 1.0 - assert_in_out_err([env, "--disable-gems", "-e", "GC.start; 1000_000.times{Object.new}; p(GC.stat[:minor_gc_count] < GC.stat[:major_gc_count])"], "", ['true'], //, "") + assert_in_out_err([env, "-e", "GC.start; 1000_000.times{Object.new}; p(GC.stat[:minor_gc_count] < GC.stat[:major_gc_count])"], "", ['true'], //, "") end env = { @@ -505,19 +505,19 @@ def test_profiler_enabled def test_profiler_clear omit "for now" - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'eom', timeout: 30 - GC::Profiler.enable + assert_separately([], __FILE__, __LINE__, <<-'RUBY', timeout: 30) + GC::Profiler.enable - GC.start - assert_equal(1, GC::Profiler.raw_data.size) - GC::Profiler.clear - assert_equal(0, GC::Profiler.raw_data.size) - - 200.times{ GC.start } - assert_equal(200, GC::Profiler.raw_data.size) - GC::Profiler.clear - assert_equal(0, GC::Profiler.raw_data.size) - eom + GC.start + assert_equal(1, GC::Profiler.raw_data.size) + GC::Profiler.clear + assert_equal(0, GC::Profiler.raw_data.size) + + 200.times{ GC.start } + assert_equal(200, GC::Profiler.raw_data.size) + GC::Profiler.clear + assert_equal(0, GC::Profiler.raw_data.size) + RUBY end def test_profiler_total_time @@ -531,34 +531,34 @@ def test_profiler_total_time end def test_finalizing_main_thread - assert_in_out_err(%w[--disable-gems], <<-EOS, ["\"finalize\""], [], "[ruby-dev:46647]") + assert_in_out_err([], <<-EOS, ["\"finalize\""], [], "[ruby-dev:46647]") ObjectSpace.define_finalizer(Thread.main) { p 'finalize' } EOS end def test_expand_heap - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'eom' - GC.start - base_length = GC.stat[:heap_eden_pages] - (base_length * 500).times{ 'a' } - GC.start - base_length = GC.stat[:heap_eden_pages] - (base_length * 500).times{ 'a' } - GC.start - assert_in_epsilon base_length, (v = GC.stat[:heap_eden_pages]), 1/8r, - "invalid heap expanding (base_length: #{base_length}, GC.stat[:heap_eden_pages]: #{v})" + assert_separately([], __FILE__, __LINE__, <<~'RUBY') + GC.start + base_length = GC.stat[:heap_eden_pages] + (base_length * 500).times{ 'a' } + GC.start + base_length = GC.stat[:heap_eden_pages] + (base_length * 500).times{ 'a' } + GC.start + assert_in_epsilon base_length, (v = GC.stat[:heap_eden_pages]), 1/8r, + "invalid heap expanding (base_length: #{base_length}, GC.stat[:heap_eden_pages]: #{v})" - a = [] - (base_length * 500).times{ a << 'a'; nil } - GC.start - assert_operator base_length, :<, GC.stat[:heap_eden_pages] + 1 - eom + a = [] + (base_length * 500).times{ a << 'a'; nil } + GC.start + assert_operator base_length, :<, GC.stat[:heap_eden_pages] + 1 + RUBY end def test_thrashing_for_young_objects # This test prevents bugs like [Bug #18929] - assert_separately %w[--disable-gem], __FILE__, __LINE__, <<-'RUBY' + assert_separately([], __FILE__, __LINE__, <<-'RUBY') # Grow the heap @ary = 100_000.times.map { Object.new } @@ -648,11 +648,11 @@ def test_interrupt_in_finalizer end def test_finalizer_passed_object_id - assert_in_out_err(%w[--disable-gems], <<-EOS, ["true"], []) + assert_in_out_err([], <<~RUBY, ["true"], []) o = Object.new obj_id = o.object_id ObjectSpace.define_finalizer(o, ->(id){ p id == obj_id }) - EOS + RUBY end def test_verify_internal_consistency diff --git a/test/ruby/test_require.rb b/test/ruby/test_require.rb index 49c771c33cf6d2..cadab4f851ab3a 100644 --- a/test/ruby/test_require.rb +++ b/test/ruby/test_require.rb @@ -193,7 +193,7 @@ def test_require_twice File.write(req, "p :ok\n") assert_file.exist?(req) req[/.rb$/i] = "" - assert_in_out_err(['--disable-gems'], <<-INPUT, %w(:ok), []) + assert_in_out_err([], <<-INPUT, %w(:ok), []) require "#{req}" require "#{req}" INPUT @@ -681,7 +681,7 @@ def test_require_to_path_redefined_in_load_path Dir.mktmpdir {|tmp| Dir.chdir(tmp) { open("foo.rb", "w") {} - assert_in_out_err([{"RUBYOPT"=>nil}, '--disable-gems'], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) + assert_in_out_err([{"RUBYOPT"=>nil}], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) begin; $:.replace([IO::NULL]) a = Object.new @@ -709,7 +709,7 @@ def test_require_to_str_redefined_in_load_path Dir.mktmpdir {|tmp| Dir.chdir(tmp) { open("foo.rb", "w") {} - assert_in_out_err([{"RUBYOPT"=>nil}, '--disable-gems'], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) + assert_in_out_err([{"RUBYOPT"=>nil}], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7158) begin; $:.replace([IO::NULL]) a = Object.new @@ -739,7 +739,7 @@ def assert_require_with_shared_array_modified(add, del) open("foo.rb", "w") {} Dir.mkdir("a") open(File.join("a", "bar.rb"), "w") {} - assert_in_out_err(['--disable-gems'], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7383) + assert_in_out_err([], "#{<<~"begin;"}\n#{<<~"end;"}", %w(:ok), [], bug7383) begin; $:.replace([IO::NULL]) $:.#{add} "#{tmp}" @@ -963,7 +963,7 @@ def test_resolve_feature_path_with_missing_feature def test_require_with_public_method_missing # [Bug #19793] - assert_separately(["-W0", "--disable-gems", "-rtempfile"], __FILE__, __LINE__, <<~RUBY) + assert_separately(["-W0", "-rtempfile"], __FILE__, __LINE__, <<~RUBY) GC.stress = true class Object diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb index 3d98cd6f4e2526..20fa15604db9d1 100644 --- a/test/ruby/test_rubyoptions.rb +++ b/test/ruby/test_rubyoptions.rb @@ -135,12 +135,12 @@ def test_warning end def test_debug - assert_in_out_err(["--disable-gems", "-de", "p $DEBUG"], "", %w(true), []) + assert_in_out_err(["-de", "p $DEBUG"], "", %w(true), []) - assert_in_out_err(["--disable-gems", "--debug", "-e", "p $DEBUG"], + assert_in_out_err(["--debug", "-e", "p $DEBUG"], "", %w(true), []) - assert_in_out_err(["--disable-gems", "--debug-", "-e", "p $DEBUG"], "", %w(), /invalid option --debug-/) + assert_in_out_err(["--debug-", "-e", "p $DEBUG"], "", %w(), /invalid option --debug-/) end q = Regexp.method(:quote) @@ -211,9 +211,9 @@ def test_disable assert_in_out_err(%w(--disable foobarbazqux -e) + [""], "", [], /unknown argument for --disable: `foobarbazqux'/) assert_in_out_err(%w(--disable), "", [], /missing argument for --disable/) - assert_in_out_err(%w(--disable-gems -e) + ['p defined? Gem'], "", ["nil"], []) + assert_in_out_err(%w(-e) + ['p defined? Gem'], "", ["nil"], []) assert_in_out_err(%w(--disable-did_you_mean -e) + ['p defined? DidYouMean'], "", ["nil"], []) - assert_in_out_err(%w(--disable-gems -e) + ['p defined? DidYouMean'], "", ["nil"], []) + assert_in_out_err(%w(-e) + ['p defined? DidYouMean'], "", ["nil"], []) end def test_kanji diff --git a/test/ruby/test_thread.rb b/test/ruby/test_thread.rb index e6f347c4869fe3..703373b11ea75a 100644 --- a/test/ruby/test_thread.rb +++ b/test/ruby/test_thread.rb @@ -395,7 +395,7 @@ def test_abort_on_exception end INPUT - assert_in_out_err(%w(--disable-gems -d), <<-INPUT, %w(false 2), %r".+") + assert_in_out_err(%w(-d), <<-INPUT, %w(false 2), %r".+") p Thread.abort_on_exception begin t = Thread.new { raise } diff --git a/test/test_tempfile.rb b/test/test_tempfile.rb index 6b087f920742b4..87c1241df85f0f 100644 --- a/test/test_tempfile.rb +++ b/test/test_tempfile.rb @@ -209,8 +209,7 @@ def test_tempfile_is_unlinked_when_ruby_exits def test_tempfile_finalizer_does_not_run_if_unlinked bug8768 = '[ruby-core:56521] [Bug #8768]' - args = %w(--disable-gems -rtempfile) - assert_in_out_err(args, <<-'EOS') do |(filename), (error)| + assert_in_out_err(%w(-rtempfile), <<-'EOS') do |(filename), (error)| tmp = Tempfile.new('foo') puts tmp.path tmp.close From 325240d0b6878612326cd182023c08ce2f092e36 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Mon, 28 Aug 2023 14:13:59 -0400 Subject: [PATCH 069/208] [ruby/yarp] make `node.c` generated code more readable https://github.com/ruby/yarp/commit/0ffd61c87a --- yarp/templates/src/node.c.erb | 37 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/yarp/templates/src/node.c.erb b/yarp/templates/src/node.c.erb index d288628fffa666..f837ca0324a6f7 100644 --- a/yarp/templates/src/node.c.erb +++ b/yarp/templates/src/node.c.erb @@ -78,29 +78,33 @@ yp_node_destroy(yp_parser_t *parser, yp_node_t *node) { switch (YP_NODE_TYPE(node)) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" - case <%= node.type %>: + case <%= node.type %>: { + <%- if node.params.any? { |param| ![LocationParam, OptionalLocationParam, UInt32Param, FlagsParam, ConstantParam].include?(param.class) } -%> + yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; + <%- end -%> <%- node.params.each do |param| -%> <%- case param -%> <%- when LocationParam, OptionalLocationParam, UInt32Param, FlagsParam, ConstantParam -%> <%- when NodeParam -%> - yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); + yp_node_destroy(parser, (yp_node_t *)cast-><%= param.name %>); <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> != NULL) { - yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); + if (cast-><%= param.name %> != NULL) { + yp_node_destroy(parser, (yp_node_t *)cast-><%= param.name %>); } <%- when StringParam -%> - yp_string_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + yp_string_free(&cast-><%= param.name %>); <%- when NodeListParam -%> - yp_node_list_free(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>); + yp_node_list_free(parser, &cast-><%= param.name %>); <%- when LocationListParam -%> - yp_location_list_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + yp_location_list_free(&cast-><%= param.name %>); <%- when ConstantListParam -%> - yp_constant_id_list_free(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + yp_constant_id_list_free(&cast-><%= param.name %>); <%- else -%> <%- raise -%> <%- end -%> <%- end -%> break; + } <%- end -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" default: @@ -122,24 +126,25 @@ yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { - memsize->memsize += sizeof(yp_<%= node.human %>_t); + yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; + memsize->memsize += sizeof(*cast); <%- node.params.each do |param| -%> <%- case param -%> <%- when ConstantParam, UInt32Param, FlagsParam, LocationParam, OptionalLocationParam -%> <%- when NodeParam -%> - yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize); + yp_node_memsize_node((yp_node_t *)cast-><%= param.name %>, memsize); <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> != NULL) { - yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize); + if (cast-><%= param.name %> != NULL) { + yp_node_memsize_node((yp_node_t *)cast-><%= param.name %>, memsize); } <%- when StringParam -%> - memsize->memsize += yp_string_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + memsize->memsize += yp_string_memsize(&cast-><%= param.name %>); <%- when NodeListParam -%> - yp_node_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize); + yp_node_list_memsize(&cast-><%= param.name %>, memsize); <%- when LocationListParam -%> - memsize->memsize += yp_location_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + memsize->memsize += yp_location_list_memsize(&cast-><%= param.name %>); <%- when ConstantListParam -%> - memsize->memsize += yp_constant_id_list_memsize(&((yp_<%= node.human %>_t *)node)-><%= param.name %>); + memsize->memsize += yp_constant_id_list_memsize(&cast-><%= param.name %>); <%- else -%> <%- raise -%> <%- end -%> From f726ad97406dc1c3bccfe74a53931552b9db755f Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Mon, 28 Aug 2023 12:19:56 -0400 Subject: [PATCH 070/208] [ruby/yarp] use `memcmp` for block memory comparison https://github.com/ruby/yarp/commit/3563e5c5d5 --- yarp/util/yp_constant_pool.c | 2 +- yarp/yarp.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/yarp/util/yp_constant_pool.c b/yarp/util/yp_constant_pool.c index fdece2dabb25b0..df46c769921322 100644 --- a/yarp/util/yp_constant_pool.c +++ b/yarp/util/yp_constant_pool.c @@ -122,7 +122,7 @@ yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t leng // If there is a collision, then we need to check if the content is the // same as the content we are trying to insert. If it is, then we can // return the id of the existing constant. - if ((constant->length == length) && strncmp(constant->start, start, length) == 0) { + if ((constant->length == length) && memcmp(constant->start, start, length) == 0) { return pool->constants[index].id; } diff --git a/yarp/yarp.c b/yarp/yarp.c index 6f66affd78b281..ad433efc4b47d5 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -4693,7 +4693,7 @@ parser_lex_encoding_comment_start(yp_parser_t *parser, const char *cursor, ptrdi const char *cursor_limit = cursor + length - key_length + 1; while ((cursor = yp_memchr(cursor, 'c', (size_t) (cursor_limit - cursor), parser->encoding_changed, &parser->encoding)) != NULL) { - if (strncmp(cursor, "coding", key_length - 1) == 0) { + if (memcmp(cursor, "coding", key_length - 1) == 0) { size_t whitespace_after_coding = yp_strspn_inline_whitespace(cursor + key_length - 1, parser->end - (cursor + key_length - 1)); size_t cur_pos = key_length + whitespace_after_coding; @@ -5211,7 +5211,7 @@ lex_keyword(yp_parser_t *parser, const char *value, yp_lex_state_t state, yp_tok yp_lex_state_t last_state = parser->lex_state; const size_t vlen = strlen(value); - if (parser->current.start + vlen <= parser->end && strncmp(parser->current.start, value, vlen) == 0) { + if (parser->current.start + vlen <= parser->end && memcmp(parser->current.start, value, vlen) == 0) { if (parser->lex_state & YP_LEX_STATE_FNAME) { lex_state_set(parser, YP_LEX_STATE_ENDFN); } else { @@ -5648,7 +5648,7 @@ lex_embdoc(yp_parser_t *parser) { // If we've hit the end of the embedded documentation then we'll return that // token here. - if (strncmp(parser->current.end, "=end", 4) == 0 && + if (memcmp(parser->current.end, "=end", 4) == 0 && (parser->current.end + 4 == parser->end || yp_char_is_whitespace(parser->current.end[4]))) { const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); @@ -6151,7 +6151,7 @@ parser_lex(yp_parser_t *parser) { // = => =~ == === =begin case '=': - if (current_token_starts_line(parser) && strncmp(peek_string(parser, 5), "begin", 5) == 0 && yp_char_is_whitespace(peek_offset(parser, 5))) { + if (current_token_starts_line(parser) && memcmp(peek_string(parser, 5), "begin", 5) == 0 && yp_char_is_whitespace(peek_offset(parser, 5))) { yp_token_type_t type = lex_embdoc(parser); if (type == YP_TOKEN_EOF) { @@ -6816,7 +6816,7 @@ parser_lex(yp_parser_t *parser) { if ( ((parser->current.end - parser->current.start) == 7) && current_token_starts_line(parser) && - (strncmp(parser->current.start, "__END__", 7) == 0) && + (memcmp(parser->current.start, "__END__", 7) == 0) && (parser->current.end == parser->end || match_eol(parser)) ) { @@ -7294,7 +7294,7 @@ parser_lex(yp_parser_t *parser) { start += yp_strspn_inline_whitespace(start, parser->end - start); } - if ((start + ident_length <= parser->end) && (strncmp(start, ident_start, ident_length) == 0)) { + if ((start + ident_length <= parser->end) && (memcmp(start, ident_start, ident_length) == 0)) { bool matched = true; bool at_end = false; @@ -7364,7 +7364,7 @@ parser_lex(yp_parser_t *parser) { // again and return the end of the heredoc. if ( (start + ident_length <= parser->end) && - (strncmp(start, ident_start, ident_length) == 0) + (memcmp(start, ident_start, ident_length) == 0) ) { // Heredoc terminators must be followed by a newline, CRLF, or EOF to be valid. if ( From 3b815ed7da8261f45b84dcde2c900934f7379dac Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 28 Aug 2023 16:55:58 -0400 Subject: [PATCH 071/208] Add yarp/yarp_compiler.c (#8042) * Add yarp/yarp_compiler.c as stencil for compiling YARP This commit adds yarp/yarp_compiler.c, and changes the sync script to ensure that yarp/yarp_compiler.c will not get overwritten * [Misc #119772] Create and expose RubyVM::InstructionSequence.compile_yarp This commit creates the stencil for a compile_yarp function, which we will continue to fill out. It allows us to check the output of compiled YARP code against compiled code without using YARP. --- common.mk | 45 ++++++++++++++++++++++++++ compile.c | 19 +++++++++++ iseq.c | 67 ++++++++++++++++++++++++++++++++++++++- tool/sync_default_gems.rb | 2 ++ yarp/yarp_compiler.c | 18 +++++++++++ 5 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 yarp/yarp_compiler.c diff --git a/common.mk b/common.mk index 1a85d0605af1e7..d55d1788aa8e01 100644 --- a/common.mk +++ b/common.mk @@ -3182,6 +3182,26 @@ compile.$(OBJEXT): $(top_srcdir)/internal/thread.h compile.$(OBJEXT): $(top_srcdir)/internal/variable.h compile.$(OBJEXT): $(top_srcdir)/internal/vm.h compile.$(OBJEXT): $(top_srcdir)/internal/warnings.h +compile.$(OBJEXT): $(top_srcdir)/yarp/defines.h +compile.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h +compile.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h +compile.$(OBJEXT): $(top_srcdir)/yarp/node.h +compile.$(OBJEXT): $(top_srcdir)/yarp/pack.h +compile.$(OBJEXT): $(top_srcdir)/yarp/parser.h +compile.$(OBJEXT): $(top_srcdir)/yarp/regexp.h +compile.$(OBJEXT): $(top_srcdir)/yarp/unescape.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_memchr.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h +compile.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h +compile.$(OBJEXT): $(top_srcdir)/yarp/yarp.h +compile.$(OBJEXT): $(top_srcdir)/yarp/yarp_compiler.c compile.$(OBJEXT): {$(VPATH)}assert.h compile.$(OBJEXT): {$(VPATH)}atomic.h compile.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -3379,6 +3399,9 @@ compile.$(OBJEXT): {$(VPATH)}vm_callinfo.h compile.$(OBJEXT): {$(VPATH)}vm_core.h compile.$(OBJEXT): {$(VPATH)}vm_debug.h compile.$(OBJEXT): {$(VPATH)}vm_opts.h +compile.$(OBJEXT): {$(VPATH)}yarp/ast.h +compile.$(OBJEXT): {$(VPATH)}yarp/version.h +compile.$(OBJEXT): {$(VPATH)}yarp/yarp.h complex.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h complex.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h complex.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -8206,6 +8229,25 @@ iseq.$(OBJEXT): $(top_srcdir)/internal/thread.h iseq.$(OBJEXT): $(top_srcdir)/internal/variable.h iseq.$(OBJEXT): $(top_srcdir)/internal/vm.h iseq.$(OBJEXT): $(top_srcdir)/internal/warnings.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/defines.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/diagnostic.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/enc/yp_encoding.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/node.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/pack.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/parser.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/regexp.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/unescape.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_buffer.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_char.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_constant_pool.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_list.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_memchr.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_newline_list.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_state_stack.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_string_list.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/util/yp_strpbrk.h +iseq.$(OBJEXT): $(top_srcdir)/yarp/yarp.h iseq.$(OBJEXT): {$(VPATH)}assert.h iseq.$(OBJEXT): {$(VPATH)}atomic.h iseq.$(OBJEXT): {$(VPATH)}backward/2/assume.h @@ -8400,6 +8442,9 @@ iseq.$(OBJEXT): {$(VPATH)}util.h iseq.$(OBJEXT): {$(VPATH)}vm_callinfo.h iseq.$(OBJEXT): {$(VPATH)}vm_core.h iseq.$(OBJEXT): {$(VPATH)}vm_opts.h +iseq.$(OBJEXT): {$(VPATH)}yarp/ast.h +iseq.$(OBJEXT): {$(VPATH)}yarp/version.h +iseq.$(OBJEXT): {$(VPATH)}yarp/yarp.h iseq.$(OBJEXT): {$(VPATH)}yjit.h load.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h load.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h diff --git a/compile.c b/compile.c index 1b382765c768ee..d7f345d0af1dab 100644 --- a/compile.c +++ b/compile.c @@ -43,6 +43,9 @@ #include "builtin.h" #include "insns.inc" #include "insns_info.inc" +#include "yarp/yarp.h" + +VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer); #undef RUBY_UNTYPED_DATA_WARNING #define RUBY_UNTYPED_DATA_WARNING 0 @@ -855,6 +858,20 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node) return iseq_setup(iseq, ret); } +static VALUE rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret); + +VALUE +rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer) +{ + DECL_ANCHOR(ret); + INIT_ANCHOR(ret); + + CHECK(rb_translate_yarp(iseq, yarp_pointer, ret)); + + CHECK(iseq_setup_insn(iseq, ret)); + return iseq_setup(iseq, ret); +} + static int rb_iseq_translate_threaded_code(rb_iseq_t *iseq) { @@ -13288,3 +13305,5 @@ rb_iseq_ibf_load_extra_data(VALUE str) RB_GC_GUARD(loader_obj); return extra_str; } + +#include "yarp/yarp_compiler.c" diff --git a/iseq.c b/iseq.c index 80bd280a14b25e..0b7ea138fa7e10 100644 --- a/iseq.c +++ b/iseq.c @@ -43,6 +43,9 @@ #include "builtin.h" #include "insns.inc" #include "insns_info.inc" +#include "yarp/yarp.h" + +VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer); VALUE rb_cISeq; static VALUE iseqw_new(const rb_iseq_t *iseq); @@ -1326,7 +1329,7 @@ rb_iseqw_new(const rb_iseq_t *iseq) static VALUE iseqw_s_compile(int argc, VALUE *argv, VALUE self) { - VALUE src, file = Qnil, path = Qnil, line = INT2FIX(1), opt = Qnil; + VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; int i; i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); @@ -1348,6 +1351,67 @@ iseqw_s_compile(int argc, VALUE *argv, VALUE self) return iseqw_new(rb_iseq_compile_with_option(src, file, path, line, opt)); } +static VALUE +iseqw_s_compile_yarp(int argc, VALUE *argv, VALUE self) +{ + VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; + int i; + + i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); + if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); + switch (i) { + case 5: opt = argv[--i]; + case 4: line = argv[--i]; + case 3: path = argv[--i]; + case 2: file = argv[--i]; + } + + if (NIL_P(file)) file = rb_fstring_lit(""); + if (NIL_P(path)) path = file; + if (NIL_P(line)) line = INT2FIX(1); + + Check_Type(path, T_STRING); + Check_Type(file, T_STRING); + + rb_iseq_t *iseq = iseq_alloc(); + + yp_parser_t parser; + size_t len = RSTRING_LEN(src); + VALUE name = rb_fstring_lit(""); + + yp_parser_init(&parser, RSTRING_PTR(src), len, ""); + + yp_node_t *node = yp_parse(&parser); + + int first_lineno = NUM2INT(line); + yp_line_column_t start_loc = yp_newline_list_line_column(&parser.newline_list, node->location.start); + yp_line_column_t end_loc = yp_newline_list_line_column(&parser.newline_list, node->location.end); + + rb_code_location_t node_location; + node_location.beg_pos.lineno = (int)start_loc.line; + node_location.beg_pos.column = (int)start_loc.column; + node_location.end_pos.lineno = (int)end_loc.line; + node_location.end_pos.column = (int)end_loc.column; + + int node_id = 0; + + rb_iseq_t *parent = NULL; + enum rb_iseq_type iseq_type = ISEQ_TYPE_TOP; + rb_compile_option_t option; + + make_compile_option(&option, opt); + + prepare_iseq_build(iseq, name, file, path, first_lineno, &node_location, node_id, + parent, 0, (enum rb_iseq_type)iseq_type, Qnil, &option); + + rb_iseq_compile_yarp_node(iseq, node); + + yp_node_destroy(&parser, node); + yp_parser_free(&parser); + + return iseqw_new(iseq); +} + /* * call-seq: * InstructionSequence.compile_file(file[, options]) -> iseq @@ -3920,6 +3984,7 @@ Init_ISeq(void) (void)iseq_s_load; rb_define_singleton_method(rb_cISeq, "compile", iseqw_s_compile, -1); + rb_define_singleton_method(rb_cISeq, "compile_yarp", iseqw_s_compile_yarp, -1); rb_define_singleton_method(rb_cISeq, "new", iseqw_s_compile, -1); rb_define_singleton_method(rb_cISeq, "compile_file", iseqw_s_compile_file, -1); rb_define_singleton_method(rb_cISeq, "compile_option", iseqw_s_compile_option_get, 0); diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index ad52c3faffb1a3..9f1bde06a87c15 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -400,6 +400,7 @@ def sync_default_gems(gem) # We don't want to remove yarp_init.c, so we temporarily move it # out of the yarp dir, wipe the yarp dir, and then put it back mv("yarp/yarp_init.c", ".") if File.exist? "yarp/yarp_init.c" + mv("yarp/yarp_compiler.c", ".") if File.exist? "yarp/yarp_compiler.c" rm_rf(%w[test/yarp yarp]) # Run the YARP templating scripts @@ -417,6 +418,7 @@ def sync_default_gems(gem) rm("yarp/extconf.rb") mv("yarp_init.c", "yarp/") + mv("yarp_compiler.c", "yarp/") else sync_lib gem, upstream end diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c new file mode 100644 index 00000000000000..ae8575e33ac3fe --- /dev/null +++ b/yarp/yarp_compiler.c @@ -0,0 +1,18 @@ +#include "yarp.h" + +static void +yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped) { + return; +} + +static VALUE +rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret) +{ + RUBY_ASSERT(ISEQ_COMPILE_DATA(iseq)); + RUBY_ASSERT(node->type == YP_NODE_PROGRAM_NODE); + + yp_compile_node(iseq, node, ret, node->location.start, false); + iseq_set_sequence(iseq, ret); + + return Qnil; +} From 23c83d172c1e68a35e80548ea7efb64cc1c063b5 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Fri, 25 Aug 2023 13:32:04 -0400 Subject: [PATCH 072/208] YJIT: Remove Type::CArray and limit use of Type::CString These types are essentially claims about what `RBASIC_CLASS(obj)` returns. The field changes with singleton class creation, but we didn't consider so previously and elided guards where we actually needed them. Found running ruby/spec with --yjit-verify-ctx. The assertion interface makes extensive use of singleton classes. --- bootstraptest/test_yjit.rb | 25 +++++++++++++++++++++++++ yjit/src/codegen.rs | 19 ++++++++++++------- yjit/src/core.rs | 22 +++------------------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 4f73af89e8507d..3c53641f91e2fb 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -1,3 +1,28 @@ +# regression test for overly generous guard elision +assert_equal '[0, :sum, 0, :sum]', %q{ + # In faulty versions, the following happens: + # 1. YJIT puts object on the temp stack with type knowledge + # (CArray or CString) about RBASIC_CLASS(object). + # 2. In iter=0, due to the type knowledge, YJIT generates + # a call to sum() without any guard on RBASIC_CLASS(object). + # 3. In iter=1, a singleton class is added to the object, + # changing RBASIC_CLASS(object), falsifying the type knowledge. + # 4. Because the code from (1) has no class guard, it is incorrectly + # reused and the wrong method is invoked. + # Putting a literal is important for gaining type knowledge. + def carray(iter) + array = [] + array.sum(iter.times { def array.sum(_) = :sum }) + end + + def cstring(iter) + string = "" + string.sum(iter.times { def string.sum(_) = :sum }) + end + + [carray(0), carray(1), cstring(0), cstring(1)] +} + # regression test for return type of Integer#/ # It can return a T_BIGNUM when inputs are T_FIXNUM. assert_equal 0x3fffffffffffffff.to_s, %q{ diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 6ccb14264bdf9f..354b8e5fd213af 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -1272,7 +1272,7 @@ fn gen_newarray( ); asm.stack_pop(n.as_usize()); - let stack_ret = asm.stack_push(Type::CArray); + let stack_ret = asm.stack_push(Type::TArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1295,7 +1295,7 @@ fn gen_duparray( vec![ary.into()], ); - let stack_ret = asm.stack_push(Type::CArray); + let stack_ret = asm.stack_push(Type::TArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1926,7 +1926,7 @@ fn gen_putstring( vec![EC, put_val.into()] ); - let stack_top = asm.stack_push(Type::CString); + let stack_top = asm.stack_push(Type::TString); asm.mov(stack_top, str_opnd); Some(KeepCompiling) @@ -2722,7 +2722,7 @@ fn gen_concatstrings( ); asm.stack_pop(n); - let stack_ret = asm.stack_push(Type::CString); + let stack_ret = asm.stack_push(Type::TString); asm.mov(stack_ret, return_value); Some(KeepCompiling) @@ -4170,9 +4170,14 @@ fn jit_guard_known_klass( jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); if known_klass == unsafe { rb_cString } { - asm.ctx.upgrade_opnd_type(insn_opnd, Type::CString); + // Upgrading to Type::CString here is incorrect. + // The guard we put only checks RBASIC_CLASS(obj), + // which adding a singleton class can change. We + // additionally need to know the string is frozen + // to claim Type::CString. + asm.ctx.upgrade_opnd_type(insn_opnd, Type::TString); } else if known_klass == unsafe { rb_cArray } { - asm.ctx.upgrade_opnd_type(insn_opnd, Type::CArray); + asm.ctx.upgrade_opnd_type(insn_opnd, Type::TArray); } } } @@ -6196,7 +6201,7 @@ fn gen_send_iseq( let rest_param = if opts_missing == 0 { // All optionals are filled, the rest param goes at the top of the stack argc += 1; - asm.stack_push(Type::CArray) + asm.stack_push(Type::TArray) } else { // The top of the stack will be a missing optional, but the rest // parameter needs to be placed after all the missing optionals. diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 40646ebd342fdf..c0577f08e8266d 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -57,7 +57,6 @@ pub enum Type { TString, // An object with the T_STRING flag set, possibly an rb_cString CString, // An un-subclassed string of type rb_cString (can have instance vars in some cases) TArray, // An object with the T_ARRAY flag set, possibly an rb_cArray - CArray, // An un-subclassed string of type rb_cArray (can have instance vars in some cases) TProc, // A proc object. Could be an instance of a subclass of ::rb_cProc @@ -95,13 +94,9 @@ impl Type { // Core.rs can't reference rb_cString because it's linked by Rust-only tests. // But CString vs TString is only an optimisation and shouldn't affect correctness. #[cfg(not(test))] - if val.class_of() == unsafe { rb_cString } { + if val.class_of() == unsafe { rb_cString } && val.is_frozen() { return Type::CString; } - #[cfg(not(test))] - if val.class_of() == unsafe { rb_cArray } { - return Type::CArray; - } // We likewise can't reference rb_block_param_proxy, but it's again an optimisation; // we can just treat it as a normal Object. #[cfg(not(test))] @@ -153,7 +148,6 @@ impl Type { match self { Type::UnknownHeap => true, Type::TArray => true, - Type::CArray => true, Type::Hash => true, Type::HeapSymbol => true, Type::TString => true, @@ -166,11 +160,7 @@ impl Type { /// Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) pub fn is_array(&self) -> bool { - match self { - Type::TArray => true, - Type::CArray => true, - _ => false, - } + matches!(self, Type::TArray) } /// Check if it's a T_STRING object (both TString and CString are T_STRING) @@ -190,7 +180,7 @@ impl Type { Type::False => Some(RUBY_T_FALSE), Type::Fixnum => Some(RUBY_T_FIXNUM), Type::Flonum => Some(RUBY_T_FLOAT), - Type::TArray | Type::CArray => Some(RUBY_T_ARRAY), + Type::TArray => Some(RUBY_T_ARRAY), Type::Hash => Some(RUBY_T_HASH), Type::ImmSymbol | Type::HeapSymbol => Some(RUBY_T_SYMBOL), Type::TString | Type::CString => Some(RUBY_T_STRING), @@ -211,7 +201,6 @@ impl Type { Type::Flonum => Some(rb_cFloat), Type::ImmSymbol | Type::HeapSymbol => Some(rb_cSymbol), Type::CString => Some(rb_cString), - Type::CArray => Some(rb_cArray), _ => None, } } @@ -266,11 +255,6 @@ impl Type { return TypeDiff::Compatible(1); } - // A CArray is also a TArray. - if self == Type::CArray && dst == Type::TArray { - return TypeDiff::Compatible(1); - } - // Specific heap type into unknown heap type is imperfect but valid if self.is_heap() && dst == Type::UnknownHeap { return TypeDiff::Compatible(1); From 85aa28e8a65b0b0691bf18f4af13adf080638035 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Mon, 28 Aug 2023 15:29:46 -0400 Subject: [PATCH 073/208] RJIT: Remove Type::CArray and limit use of Type::CString See previous similar YJIT commit. --- lib/ruby_vm/rjit/insn_compiler.rb | 21 +++++++++++++-------- lib/ruby_vm/rjit/type.rb | 16 ++-------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb index 5b2beb68edb74e..96dfa55c69f383 100644 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ b/lib/ruby_vm/rjit/insn_compiler.rb @@ -794,7 +794,7 @@ def putstring(jit, ctx, asm) asm.mov(C_ARGS[1], to_value(put_val)) asm.call(C.rb_ec_str_resurrect) - stack_top = ctx.stack_push(Type::CString) + stack_top = ctx.stack_push(Type::TString) asm.mov(stack_top, C_RET) KeepCompiling @@ -817,7 +817,7 @@ def concatstrings(jit, ctx, asm) asm.call(C.rb_str_concat_literals) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CString) + stack_ret = ctx.stack_push(Type::TString) asm.mov(stack_ret, C_RET) KeepCompiling @@ -932,7 +932,7 @@ def newarray(jit, ctx, asm) asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling @@ -954,7 +954,7 @@ def duparray(jit, ctx, asm) asm.mov(C_ARGS[0], ary) asm.call(C.rb_ary_resurrect) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling @@ -3082,7 +3082,7 @@ def jit_rb_str_concat(jit, ctx, asm, argc, known_recv_class) asm.test(recv_reg, C::RUBY_ENCODING_MASK) # Push once, use the resulting operand in both branches below. - stack_ret = ctx.stack_push(Type::CString) + stack_ret = ctx.stack_push(Type::TString) enc_mismatch = asm.new_label('enc_mismatch') asm.jnz(enc_mismatch) @@ -3779,9 +3779,14 @@ def jit_guard_known_klass(jit, ctx, asm, known_klass, obj_opnd, insn_opnd, compt jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) if known_klass == C.rb_cString - ctx.upgrade_opnd_type(insn_opnd, Type::CString) + # Upgrading to Type::CString here is incorrect. + # The guard we put only checks RBASIC_CLASS(obj), + # which adding a singleton class can change. We + # additionally need to know the string is frozen + # to claim Type::CString. + ctx.upgrade_opnd_type(insn_opnd, Type::TString) elsif known_klass == C.rb_cArray - ctx.upgrade_opnd_type(insn_opnd, Type::CArray) + ctx.upgrade_opnd_type(insn_opnd, Type::TArray) end end end @@ -4723,7 +4728,7 @@ def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: n asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) end end diff --git a/lib/ruby_vm/rjit/type.rb b/lib/ruby_vm/rjit/type.rb index 155e141d63c8b8..119692014bfe02 100644 --- a/lib/ruby_vm/rjit/type.rb +++ b/lib/ruby_vm/rjit/type.rb @@ -35,7 +35,6 @@ def heap? case self in Type::UnknownHeap then true in Type::TArray then true - in Type::CArray then true in Type::Hash then true in Type::HeapSymbol then true in Type::TString then true @@ -45,11 +44,10 @@ def heap? end end - # Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) + # Check if it's a T_ARRAY object def array? case self in Type::TArray then true - in Type::CArray then true else false end end @@ -73,7 +71,6 @@ def known_class in Type::Flonum then C.rb_cFloat in Type::ImmSymbol | Type::HeapSymbol then C.rb_cSymbol in Type::CString then C.rb_cString - in Type::CArray then C.rb_cArray else nil end end @@ -115,11 +112,6 @@ def diff(dst) return TypeDiff::Compatible[1] end - # A CArray is also a TArray. - if self == Type::CArray && dst == Type::TArray - return TypeDiff::Compatible[1] - end - # Specific heap type into unknown heap type is imperfect but valid if self.heap? && dst == Type::UnknownHeap return TypeDiff::Compatible[1] @@ -169,12 +161,9 @@ def from(val) end else val_class = C.to_value(C.rb_class_of(val)) - if val_class == C.rb_cString + if val_class == C.rb_cString && C.rb_obj_frozen_p(val) return Type::CString end - if val_class == C.rb_cArray - return Type::CArray - end if C.to_value(val) == C.rb_block_param_proxy return Type::BlockParamProxy end @@ -222,7 +211,6 @@ def static_symbol?(obj) Type::TString = Type[:TString] # An object with the T_STRING flag set, possibly an rb_cString Type::CString = Type[:CString] # An un-subclassed string of type rb_cString (can have instance vars in some cases) Type::TArray = Type[:TArray] # An object with the T_ARRAY flag set, possibly an rb_cArray - Type::CArray = Type[:CArray] # An un-subclassed string of type rb_cArray (can have instance vars in some cases) Type::BlockParamProxy = Type[:BlockParamProxy] # A special sentinel value indicating the block parameter should be read from From 5485680244bc40a9dba52e468fc5705973f2f5bd Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 28 Aug 2023 16:33:28 -0400 Subject: [PATCH 074/208] Expose RVALUE_OLD_AGE in GC::INTERNAL_CONSTANTS --- gc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/gc.c b/gc.c index 34334f3278c27f..e57e8cf52ef002 100644 --- a/gc.c +++ b/gc.c @@ -13907,6 +13907,7 @@ Init_GC(void) rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(SIZE_POOL_COUNT)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(size_pool_slot_size(SIZE_POOL_COUNT - 1))); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE)); if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) { rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue); } From fd0df1f8c6845671c86eddd85c9bcced30501690 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 28 Aug 2023 16:34:27 -0400 Subject: [PATCH 075/208] Fix growth in minor GC when we have initial slots If initial slots is set, then during a minor GC, if we have allocatable pages but the heap is mostly full, then we will set `grow_heap` to true since `total_slots` does not count allocatable pages so it will be less than `init_slots`. This can cause `allocatable_pages` to grow to much higher than desired since it will appear that the heap is mostly full. --- gc.c | 11 +++++------ test/ruby/test_gc.rb | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/gc.c b/gc.c index e57e8cf52ef002..9178c9462fa913 100644 --- a/gc.c +++ b/gc.c @@ -5689,12 +5689,11 @@ gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) if (swept_slots < min_free_slots) { bool grow_heap = is_full_marking(objspace); - if (!is_full_marking(objspace)) { - /* The heap is a growth heap if it freed more slots than had empty - * slots and used up all of its allocatable pages. */ - bool is_growth_heap = (size_pool->empty_slots == 0 || - size_pool->freed_slots > size_pool->empty_slots) && - size_pool->allocatable_pages == 0; + /* Consider growing or starting a major GC if we are not currently in a + * major GC and we can't allocate any more pages. */ + if (!is_full_marking(objspace) && size_pool->allocatable_pages == 0) { + /* The heap is a growth heap if it freed more slots than had empty slots. */ + bool is_growth_heap = size_pool->empty_slots == 0 || size_pool->freed_slots > size_pool->empty_slots; /* Grow this heap if we haven't run at least RVALUE_OLD_AGE minor * GC since the last major GC or if this heap is smaller than the diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index c5a9731f8a9ae8..8fc511331ee0c3 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -492,6 +492,45 @@ def test_gc_parameter_init_slots assert_in_epsilon(SIZES[i], total_slots, 0.01, s) end RUBY + + # Check that we don't grow the heap in minor GC if we have alloctable pages. + env["RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO"] = "0.3" + env["RUBY_GC_HEAP_FREE_SLOTS_GOAL_RATIO"] = "0.99" + env["RUBY_GC_HEAP_FREE_SLOTS_MAX_RATIO"] = "1.0" + env["RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR"] = "100" # Large value to disable major GC + assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY) + SIZES = #{sizes} + + # Run a major GC to clear out dead objects. + GC.start + + # Disable GC so we can control when GC is ran. + GC.disable + + # Run minor GC enough times so that we don't grow the heap because we + # haven't yet ran RVALUE_OLD_AGE minor GC cycles. + GC::INTERNAL_CONSTANTS[:RVALUE_OLD_AGE].times { GC.start(full_mark: false) } + + # Fill size pool 0 to over 50% full so that the number of allocatable + # pages that will be created will be over the number in heap_allocatable_pages + # (calculated using RUBY_GC_HEAP_FREE_SLOTS_MIN_RATIO). + # 70% was chosen here to guarantee that. + ary = [] + while GC.stat_heap(0, :heap_allocatable_pages) > + (GC.stat_heap(0, :heap_allocatable_pages) + GC.stat_heap(0, :heap_eden_pages)) * 0.3 + ary << Object.new + end + + GC.start(full_mark: false) + + # Check that we still have the same number of slots as initially configured. + GC.stat_heap.each do |i, s| + # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. + slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page + assert_in_epsilon(SIZES[i], total_slots, 0.01, s) + end + RUBY end def test_profiler_enabled From c02f978fd5653b5ca1959b6547c26a834ee1f043 Mon Sep 17 00:00:00 2001 From: yui-knk Date: Mon, 28 Aug 2023 22:04:10 +0900 Subject: [PATCH 076/208] Lrama v0.5.5 --- tool/lrama/exe/lrama | 1 - tool/lrama/lib/lrama/command.rb | 9 ++------- tool/lrama/lib/lrama/context.rb | 2 -- tool/lrama/lib/lrama/counterexamples.rb | 4 +--- tool/lrama/lib/lrama/grammar.rb | 7 +++++-- tool/lrama/lib/lrama/grammar/code.rb | 1 - tool/lrama/lib/lrama/lexer.rb | 7 +++---- tool/lrama/lib/lrama/lexer/token.rb | 1 + tool/lrama/lib/lrama/output.rb | 2 +- tool/lrama/lib/lrama/parser.rb | 8 ++++++++ tool/lrama/lib/lrama/state.rb | 1 - tool/lrama/lib/lrama/states.rb | 9 +++++++-- tool/lrama/lib/lrama/states_reporter.rb | 20 ++++---------------- tool/lrama/lib/lrama/version.rb | 2 +- 14 files changed, 33 insertions(+), 41 deletions(-) diff --git a/tool/lrama/exe/lrama b/tool/lrama/exe/lrama index 7988afe50754bb..5e1ee582cf5681 100755 --- a/tool/lrama/exe/lrama +++ b/tool/lrama/exe/lrama @@ -1,6 +1,5 @@ #!/usr/bin/env ruby - $LOAD_PATH << File.join(__dir__, "../lib") require "lrama" diff --git a/tool/lrama/lib/lrama/command.rb b/tool/lrama/lib/lrama/command.rb index 9fceb55fe05568..45670ae405c0c6 100644 --- a/tool/lrama/lib/lrama/command.rb +++ b/tool/lrama/lib/lrama/command.rb @@ -5,7 +5,6 @@ class Command def initialize(argv) @argv = argv - @version = nil @skeleton = "bison/yacc.c" @header = false @header_file = nil @@ -23,15 +22,11 @@ def initialize(argv) def run parse_option - if @version - puts Lrama::VERSION - exit 0 - end - Report::Duration.enable if @trace_opts[:time] warning = Lrama::Warning.new grammar = Lrama::Parser.new(@y.read).parse + @y.close if @y != STDIN states = Lrama::States.new(grammar, warning, trace_state: (@trace_opts[:automaton] || @trace_opts[:closure])) states.compute context = Lrama::Context.new(states) @@ -112,7 +107,7 @@ def parse_option opt = OptionParser.new # opt.on('-h') {|v| p v } - opt.on('-V', '--version') {|v| @version = true } + opt.on('-V', '--version') {|v| puts "lrama #{Lrama::VERSION}"; exit 0 } # Tuning the Parser opt.on('-S', '--skeleton=FILE') {|v| @skeleton = v } diff --git a/tool/lrama/lib/lrama/context.rb b/tool/lrama/lib/lrama/context.rb index 861c4e1137d5eb..da70bf15422990 100644 --- a/tool/lrama/lib/lrama/context.rb +++ b/tool/lrama/lib/lrama/context.rb @@ -401,7 +401,6 @@ def debug_sorted_actions end print sprintf("]\n\n") - print sprintf("width [\n") vectors_count.times do |i| print sprintf("%d, ", ary[i] ? ary[i][3] : 0) @@ -409,7 +408,6 @@ def debug_sorted_actions end print sprintf("]\n\n") - print sprintf("tally [\n") vectors_count.times do |i| print sprintf("%d, ", ary[i] ? ary[i][2] : 0) diff --git a/tool/lrama/lib/lrama/counterexamples.rb b/tool/lrama/lib/lrama/counterexamples.rb index a5d62a0c7c3321..5019257dc3421a 100644 --- a/tool/lrama/lib/lrama/counterexamples.rb +++ b/tool/lrama/lib/lrama/counterexamples.rb @@ -205,7 +205,7 @@ def find_shift_conflict_shortest_state_items(reduce_path, conflict_state, confli end def build_paths_from_state_items(state_items) - paths = state_items.zip([nil] + state_items).map do |si, prev_si| + state_items.zip([nil] + state_items).map do |si, prev_si| case when prev_si.nil? StartPath.new(si) @@ -215,8 +215,6 @@ def build_paths_from_state_items(state_items) TransitionPath.new(prev_si, si) end end - - paths end def shortest_path(conflict_state, conflict_reduce_item, conflict_term) diff --git a/tool/lrama/lib/lrama/grammar.rb b/tool/lrama/lib/lrama/grammar.rb index 25f1a49170cd9c..81df3996820256 100644 --- a/tool/lrama/lib/lrama/grammar.rb +++ b/tool/lrama/lib/lrama/grammar.rb @@ -103,6 +103,10 @@ def add_right(sym, precedence) set_precedence(sym, Precedence.new(type: :right, precedence: precedence)) end + def add_precedence(sym, precedence) + set_precedence(sym, Precedence.new(type: :precedence, precedence: precedence)) + end + def set_precedence(sym, precedence) raise "" if sym.nterm? sym.precedence = precedence @@ -310,7 +314,6 @@ def find_nterm_by_id!(id) end || (raise "Nterm not found: #{id}") end - def append_special_symbols # YYEMPTY (token_id: -2, number: -2) is added when a template is evaluated # term = add_term(id: Token.new(Token::Ident, "YYEMPTY"), token_id: -2) @@ -512,7 +515,7 @@ def fill_symbol_number sym.token_id = 11 when "\"" sym.token_id = 34 - when "\'" + when "'" sym.token_id = 39 when "\\\\" sym.token_id = 92 diff --git a/tool/lrama/lib/lrama/grammar/code.rb b/tool/lrama/lib/lrama/grammar/code.rb index 0f61ebb58e4cf1..712cb1ad5a95cb 100644 --- a/tool/lrama/lib/lrama/grammar/code.rb +++ b/tool/lrama/lib/lrama/grammar/code.rb @@ -50,7 +50,6 @@ def translated_printer_code(tag) end alias :translated_error_token_code :translated_printer_code - private # * ($1) yyvsp[i] diff --git a/tool/lrama/lib/lrama/lexer.rb b/tool/lrama/lib/lrama/lexer.rb index c1b5c8fe4edd05..c591684a05e621 100644 --- a/tool/lrama/lib/lrama/lexer.rb +++ b/tool/lrama/lib/lrama/lexer.rb @@ -30,7 +30,6 @@ def initialize(text) @grammar_rules = [] @epilogue = [] - # @bison_declarations_tokens = [] @grammar_rules_tokens = [] @@ -155,6 +154,8 @@ def lex_common(lines, tokens) tokens << create_token(Token::P_left, ss[0], line, ss.pos - column) when ss.scan(/%right/) tokens << create_token(Token::P_right, ss[0], line, ss.pos - column) + when ss.scan(/%precedence/) + tokens << create_token(Token::P_precedence, ss[0], line, ss.pos - column) when ss.scan(/%prec/) tokens << create_token(Token::P_prec, ss[0], line, ss.pos - column) when ss.scan(/{/) @@ -223,7 +224,7 @@ def lex_user_code(ss, line, column, lines) references << [:dollar, ss[2], tag, str.length, str.length + ss[0].length - 1] when ss.scan(/@\$/) # @$ references << [:at, "$", nil, str.length, str.length + ss[0].length - 1] - when ss.scan(/@(\d)+/) # @1 + when ss.scan(/@(\d+)/) # @1 references << [:at, Integer(ss[1]), nil, str.length, str.length + ss[0].length - 1] when ss.scan(/{/) brace_count += 1 @@ -314,8 +315,6 @@ def lex_line_comment(ss, line, str) str << ss.getch next end - - str << ss[0] end line # Reach to end of input diff --git a/tool/lrama/lib/lrama/lexer/token.rb b/tool/lrama/lib/lrama/lexer/token.rb index aa49bfe47d7471..4eca6f60077bbb 100644 --- a/tool/lrama/lib/lrama/lexer/token.rb +++ b/tool/lrama/lib/lrama/lexer/token.rb @@ -61,6 +61,7 @@ def self.define_type(name) define_type(:P_nonassoc) # %nonassoc define_type(:P_left) # %left define_type(:P_right) # %right + define_type(:P_precedence) # %precedence define_type(:P_prec) # %prec define_type(:User_code) # { ... } define_type(:Tag) # diff --git a/tool/lrama/lib/lrama/output.rb b/tool/lrama/lib/lrama/output.rb index 757d16e25ae3ff..2dcdae7b422a33 100644 --- a/tool/lrama/lib/lrama/output.rb +++ b/tool/lrama/lib/lrama/output.rb @@ -252,7 +252,7 @@ def user_args end def extract_param_name(param) - /\A(\W*)([a-zA-Z0-9_]+)\z/.match(param.split.last)[2] + param[/\b([a-zA-Z0-9_]+)(?=\s*\z)/] end def parse_param_name diff --git a/tool/lrama/lib/lrama/parser.rb b/tool/lrama/lib/lrama/parser.rb index a12c8bbb34e33f..18e2ef033eb400 100644 --- a/tool/lrama/lib/lrama/parser.rb +++ b/tool/lrama/lib/lrama/parser.rb @@ -159,6 +159,14 @@ def parse_bison_declarations(ts, grammar) grammar.add_right(sym, precedence_number) end precedence_number += 1 + when T::P_precedence + # %precedence (ident|char|string)+ + ts.next + while (id = ts.consume(T::Ident, T::Char, T::String)) do + sym = grammar.add_term(id: id) + grammar.add_precedence(sym, precedence_number) + end + precedence_number += 1 when nil # end of input raise "Reach to end of input within declarations" diff --git a/tool/lrama/lib/lrama/state.rb b/tool/lrama/lib/lrama/state.rb index 6632bafeccb43e..1b406402157346 100644 --- a/tool/lrama/lib/lrama/state.rb +++ b/tool/lrama/lib/lrama/state.rb @@ -62,7 +62,6 @@ def set_items_to_state(items, next_state) @items_to_state[items] = next_state end - # def set_look_ahead(rule, look_ahead) reduce = reduces.find do |r| r.rule == rule diff --git a/tool/lrama/lib/lrama/states.rb b/tool/lrama/lib/lrama/states.rb index d27c5411ea9828..290e996b822b08 100644 --- a/tool/lrama/lib/lrama/states.rb +++ b/tool/lrama/lib/lrama/states.rb @@ -455,6 +455,11 @@ def compute_shift_reduce_conflicts # shift_prec == reduce_prec, then check associativity case sym.precedence.type + when :precedence + # %precedence only specifies precedence and not specify associativity + # then a conflict is unresolved if precedence is same. + state.conflicts << State::ShiftReduceConflict.new(symbols: [sym], shift: shift, reduce: reduce) + next when :right # Shift is selected state.resolved_conflicts << State::ResolvedConflict.new(symbol: sym, reduce: reduce, which: :shift, same_prec: true) @@ -515,9 +520,9 @@ def compute_default_reduction state.default_reduction_rule = state.reduces.map do |r| [r.rule, r.rule.id, (r.look_ahead || []).count] - end.sort_by do |rule, rule_id, count| + end.min_by do |rule, rule_id, count| [-count, rule_id] - end.first.first + end.first end end diff --git a/tool/lrama/lib/lrama/states_reporter.rb b/tool/lrama/lib/lrama/states_reporter.rb index 0d805829eb13f4..8be0f71e40dddf 100644 --- a/tool/lrama/lib/lrama/states_reporter.rb +++ b/tool/lrama/lib/lrama/states_reporter.rb @@ -110,7 +110,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report shifts tmp = state.term_transitions.select do |shift, _| !shift.not_selected @@ -123,7 +122,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if !tmp.empty? - # Report error caused by %nonassoc nl = false tmp = state.resolved_conflicts.select do |resolved| @@ -138,7 +136,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if !tmp.empty? - # Report reduces nl = false max_len = state.non_default_reduces.flat_map(&:look_ahead).compact.map(&:display_name).map(&:length).max || 0 @@ -171,7 +168,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if nl - # Report nonterminal transitions tmp = [] max_len = 0 @@ -189,7 +185,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" if !tmp.empty? - if solved # Report conflict resolutions state.resolved_conflicts.each do |resolved| @@ -202,13 +197,13 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) # Report counterexamples examples = cex.compute(state) examples.each do |example| - label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" + label0 = example.type == :shift_reduce ? "shift/reduce" : "reduce/reduce" label1 = example.type == :shift_reduce ? "Shift derivation" : "First Reduce derivation" label2 = example.type == :shift_reduce ? "Reduce derivation" : "Second Reduce derivation" io << " #{label0} conflict on token #{example.conflict_symbol.id.s_value}:\n" - io << " #{example.path1_item.to_s}\n" - io << " #{example.path2_item.to_s}\n" + io << " #{example.path1_item}\n" + io << " #{example.path2_item}\n" io << " #{label1}\n" example.derivations1.render_strings_for_report.each do |str| io << " #{str}\n" @@ -234,7 +229,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report reads_relation io << " [Reads Relation]\n" @states.nterms.each do |nterm| @@ -248,7 +242,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report read_sets io << " [Read sets]\n" read_sets = @states.read_sets @@ -263,7 +256,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report includes_relation io << " [Includes Relation]\n" @states.nterms.each do |nterm| @@ -277,7 +269,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report lookback_relation io << " [Lookback Relation]\n" @states.rules.each do |rule| @@ -286,12 +277,11 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) a.each do |state_id2, nterm_id2| n = @states.nterms.find {|n| n.token_id == nterm_id2 } - io << " (Rule: #{rule.to_s}) -> (State #{state_id2}, #{n.id.s_value})\n" + io << " (Rule: #{rule}) -> (State #{state_id2}, #{n.id.s_value})\n" end end io << "\n" - # Report follow_sets io << " [Follow sets]\n" follow_sets = @states.follow_sets @@ -306,7 +296,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) end io << "\n" - # Report LA io << " [Look-Ahead Sets]\n" tmp = [] @@ -326,7 +315,6 @@ def report_states(io, itemsets, lookaheads, solved, counterexamples, verbose) io << "\n" if !tmp.empty? end - # End of Report State io << "\n" end diff --git a/tool/lrama/lib/lrama/version.rb b/tool/lrama/lib/lrama/version.rb index 3f2eb3b9d4f5a3..41215c1aee8b18 100644 --- a/tool/lrama/lib/lrama/version.rb +++ b/tool/lrama/lib/lrama/version.rb @@ -1,3 +1,3 @@ module Lrama - VERSION = "0.5.4".freeze + VERSION = "0.5.5".freeze end From a6db6b150f2f37e05ea32fe6c13e4cf7a2f7838a Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 28 Aug 2023 17:47:30 +0900 Subject: [PATCH 077/208] sync_default_gems.rb: Split `sync_default_gems_with_commits` --- tool/sync_default_gems.rb | 337 ++++++++++++++++++++------------------ 1 file changed, 179 insertions(+), 158 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 9f1bde06a87c15..0bbaa234108d3d 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -494,22 +494,8 @@ def message_filter(repo, sha, input: ARGF) puts subject, "\n", log end - # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh - # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. - # @param ranges [Array] "before..after". Note that it will NOT sync "before" (but commits after that). - # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. - def sync_default_gems_with_commits(gem, ranges, edit: nil) - repo, default_branch = REPOSITORIES[gem] - puts "Sync #{repo} with commit history." - - # Fetch the repository to be synchronized - IO.popen(%W"git remote") do |f| - unless f.read.split.include?(gem) - `git remote add #{gem} https://github.com/#{repo}.git` - end - end - system(*%W"git fetch --no-tags #{gem}") - + # Returns commit list as array of [commit_hash, subject]. + def commits_in_ranges(gem, repo, default_branch, ranges) # If -a is given, discover all commits since the last picked commit if ranges == true pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)$" @@ -518,7 +504,7 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) end # Parse a given range with git log - commits = ranges.flat_map do |range| + ranges.flat_map do |range| unless range.include?("..") range = "#{range}~1..#{range}" end @@ -527,6 +513,179 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} end end + end + + #-- + # Following methods used by sync_default_gems_with_commits return + # true: success + # false: skipped + # nil: failed + #++ + + def resolve_conflicts(gem, sha) + # Forcibly remove any files that we don't want to copy to this repository. + # We also ignore them as new `toplevels` even when they don't conflict. + ignored_paths = [] + case gem + when "rubygems" + # We don't copy any vcr_cassettes to this repository. Because the directory does not + # exist, rename detection doesn't work. So it starts with the original path `bundler/`. + ignored_paths += %w[bundler/spec/support/artifice/vcr_cassettes] + when "yarp" + # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs + # since ruby/ruby/doc is not something owned by YARP. + ignored_paths += %w[docs/] + end + ignored_paths.each do |path| + if File.exist?(path) + puts "Removing: #{path}" + system("git", "reset", path) + rm_rf(path) + end + end + + # git has inexact rename detection, so they follow directory renames even for new files. + # However, new files are considered a `CONFLICT (file location)`, so you need to git-add them here. + # We hope that they are not other kinds of conflicts, assuming we don't modify them in this repository. + case gem + when "rubygems" + system(*%w[git add spec/bundler]) + when "yarp" + system(*%w[git add lib/yarp]) + system(*%w[git add test/yarp]) + system(*%w[git add yarp]) + end + + # Skip this commit if everything has been removed as `ignored_paths`. + changes = pipe_readlines(%W"git status --porcelain -z") + if changes.empty? + `git reset` && `git checkout .` && `git clean -fd` + puts "Skip empty commit #{sha}" + return false + end + + # We want to skip DD: deleted by both. + deleted = changes.grep(/^DD /) {$'} + system(*%W"git rm -f --", *deleted) unless deleted.empty? + + # Discover unmerged files + # AU: unmerged, added by us + # DU: unmerged, deleted by us + # UU: unmerged, both modified + # UA: unmerged, added by them + # AA: unmerged, both added + unmerged = changes.map {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} + unmerged.compact! + ignore, conflict = unmerged.partition {|name| ignore_file_pattern =~ name} + # Reset ignored files if they conflict + unless ignore.empty? + system(*%W"git reset HEAD --", *ignore) + File.unlink(*ignore) + ignore = pipe_readlines(%W"git status --porcelain -z" + ignore).map! {|line| line[/\A.. (.*)/, 1]} + system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? + end + # If -e option is given, open each conflicted file with an editor + unless conflict.empty? + if edit + case + when (editor = ENV["GIT_EDITOR"] and !editor.empty?) + when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) + end + if editor + system([editor, conflict].join(' ')) + end + end + end + + # Attempt to commit the cherry-pick + system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") || nil + end + + def remove_toplevel_addtions(gem, sha) + # Forcibly remove any new top-level entries, and any changes under + # /test/fixtures, /test/lib, or /tool. + changed = pipe_readlines(%W"git diff --name-only -z HEAD~..HEAD --") + toplevels = changed.map {|f| f[%r[\A(?!tool/)[^/]+/?]]}.compact + toplevels.delete_if do |top| + if system(*%w"git checkout -f HEAD~ --", top, err: File::NULL) + # previously existent path + system(*%w"git checkout -f HEAD --", top, out: File::NULL) + true + end + end + unless toplevels.empty? + puts "Remove files added to toplevel: #{toplevels.join(', ')}" + system(*%w"git rm -r --", *toplevels) + end + tools = changed.select {|f|f.start_with?("test/fixtures/", "test/lib/", "tool/")} + unless tools.empty? + system(*%W"git rm -r --", *tools) + system(*%W"git checkout HEAD~ --", *tools) + end + if toplevels.empty? and tools.empty? + return true + elsif system(*%W"git diff --quiet HEAD~") + `git reset HEAD~ --` && `git checkout .` && `git clean -fd` + puts "Skip commit #{sha} only for tools or toplevel" + return false + elsif !system(*%W"git commit --amend --no-edit --", *toplevels, *tools) + `git reset HEAD~ --` && `git checkout .` && `git clean -fd` + return nil + end + end + + def pickup_commit(gem, sha) + # Attempt to cherry-pick a commit + result = IO.popen(%W"git cherry-pick #{sha}", &:read) + if result =~ /nothing\ to\ commit/ + `git reset` + puts "Skip empty commit #{sha}" + return false + end + + # Skip empty commits + if result.empty? + return false + end + + # Skip the commit if it's empty or the cherry-pick attempt failed + if /^CONFLICT/ =~ result and !resolve_conflicts(gem, sha) + `git reset` && `git checkout .` && `git clean -fd` + return nil + end + + result = remove_toplevel_addtions(gem, sha) + return result unless result + + # Amend the commit if RDoc references need to be replaced + head = `git log --format=%H -1 HEAD`.chomp + system(*%w"git reset --quiet HEAD~ --") + amend = replace_rdoc_ref_all + system(*%W"git reset --quiet #{head} --") + if amend + `git commit --amend --no-edit --all` + end + + return true + end + + # NOTE: This method is also used by GitHub ruby/git.ruby-lang.org's bin/update-default-gem.sh + # @param gem [String] A gem name, also used as a git remote name. REPOSITORIES converts it to the appropriate GitHub repository. + # @param ranges [Array] "before..after". Note that it will NOT sync "before" (but commits after that). + # @param edit [TrueClass] Set true if you want to resolve conflicts. Obviously, update-default-gem.sh doesn't use this. + def sync_default_gems_with_commits(gem, ranges, edit: nil) + repo, default_branch = REPOSITORIES[gem] + puts "Sync #{repo} with commit history." + + # Fetch the repository to be synchronized + IO.popen(%W"git remote") do |f| + unless f.read.split.include?(gem) + `git remote add #{gem} https://github.com/#{repo}.git` + end + end + system(*%W"git fetch --no-tags #{gem}") + + commits = commits_in_ranges(gem, repo, default_branch, ranges) # Ignore Merge commits and already-merged commits. ignore_file_pattern = ignore_file_pattern_for(gem) @@ -554,152 +713,14 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) ] commits.each do |sha, subject| puts "Pick #{sha} from #{repo}." - - # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", &:read) - if result =~ /nothing\ to\ commit/ - `git reset` - puts "Skip empty commit #{sha}" + case pickup_commit(gem, sha) + when false next - end - - # Skip empty commits or deal with conflicts - skipped = false - if result.empty? - skipped = true - elsif /^CONFLICT/ =~ result - # Forcibly remove any files that we don't want to copy to this repository. - # We also ignore them as new `toplevels` even when they don't conflict. - ignored_paths = [] - case gem - when "rubygems" - # We don't copy any vcr_cassettes to this repository. Because the directory does not - # exist, rename detection doesn't work. So it starts with the original path `bundler/`. - ignored_paths += %w[bundler/spec/support/artifice/vcr_cassettes] - when "yarp" - # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs - # since ruby/ruby/doc is not something owned by YARP. - ignored_paths += %w[docs/] - end - ignored_paths.each do |path| - if File.exist?(path) - puts "Removing: #{path}" - system("git", "reset", path) - rm_rf(path) - end - end - - # git has inexact rename detection, so they follow directory renames even for new files. - # However, new files are considered a `CONFLICT (file location)`, so you need to git-add them here. - # We hope that they are not other kinds of conflicts, assuming we don't modify them in this repository. - case gem - when "rubygems" - system(*%w[git add spec/bundler]) - when "yarp" - system(*%w[git add lib/yarp]) - system(*%w[git add test/yarp]) - system(*%w[git add yarp]) - end - - # Skip this commit if everything has been removed as `ignored_paths`. - changes = pipe_readlines(%W"git status --porcelain -z") - if changes.empty? - `git reset` && `git checkout .` && `git clean -fd` - puts "Skip empty commit #{sha}" - next - end - - # For YARP, we want to skip DD: deleted by both. - if gem == "yarp" - deleted = changes.grep(/^DD /) - deleted.map! { |line| line.delete_prefix("DD ") } - system(*%W"git rm -f --", *deleted) unless deleted.empty? - end - - # Discover unmerged files - # AU: unmerged, added by us - # DU: unmerged, deleted by us - # UU: unmerged, both modified - # UA: unmerged, added by them - # AA: unmerged, both added - unmerged = changes.map {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} - unmerged.compact! - ignore, conflict = unmerged.partition {|name| ignore_file_pattern =~ name} - # Reset ignored files if they conflict - unless ignore.empty? - system(*%W"git reset HEAD --", *ignore) - File.unlink(*ignore) - ignore = pipe_readlines(%W"git status --porcelain -z" + ignore).map! {|line| line[/\A.. (.*)/, 1]} - system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? - end - # If -e option is given, open each conflicted file with an editor - unless conflict.empty? - if edit - case - when (editor = ENV["GIT_EDITOR"] and !editor.empty?) - when (editor = `git config core.editor` and (editor.chomp!; !editor.empty?)) - end - if editor - system([editor, conflict].join(' ')) - end - end - end - # Attempt to commit the cherry-pick - skipped = !system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") - end - - # Skip the commit if it's empty or the cherry-pick attempt failed - if skipped + when nil failed_commits << sha - `git reset` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" next end - # Forcibly remove any new top-level entries, and any changes under - # /test/fixtures, /test/lib, or /tool. - changed = pipe_readlines(%W"git diff --name-only -z HEAD~..HEAD --") - toplevels = changed.map {|f| f[%r[\A(?!tool/)[^/]+/?]]}.compact - toplevels.delete_if do |top| - if system(*%w"git checkout -f HEAD~ --", top, err: File::NULL) - # previously existent path - system(*%w"git checkout -f HEAD --", top, out: File::NULL) - true - end - end - unless toplevels.empty? - puts "Remove files added to toplevel: #{toplevels.join(', ')}" - system(*%w"git rm -r --", *toplevels) - end - tools = changed.select {|f|f.start_with?("test/fixtures/", "test/lib/", "tool/")} - unless tools.empty? - system(*%W"git rm -r --", *tools) - system(*%W"git checkout HEAD~ --", *tools) - end - unless toplevels.empty? and tools.empty? - clean = toplevels + tools - if system(*%W"git diff --quiet HEAD~") - `git reset HEAD~ --` && `git checkout .` && `git clean -fd` - puts "Skip commit #{sha} only for tools or toplevel" - next - end - unless system(*%W"git commit --amend --no-edit --", *clean) - failed_commits << sha - `git reset HEAD~ --` && `git checkout .` && `git clean -fd` - puts "Failed to pick #{sha}" - next - end - end - - # Amend the commit if RDoc references need to be replaced - head = `git log --format=%H -1 HEAD`.chomp - system(*%w"git reset --quiet HEAD~ --") - amend = replace_rdoc_ref_all - system(*%W"git reset --quiet #{head} --") - if amend - `git commit --amend --no-edit --all` - end - puts "Update commit message: #{sha}" # Run this script itself (tool/sync_default_gems.rb --message-filter) as a message filter From c4fc9477aa5f77a69a40b619b7144f2b118b772e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 29 Aug 2023 10:20:41 +0900 Subject: [PATCH 078/208] sync_default_gems.rb: Continue if files added to the toplevel removed --- tool/sync_default_gems.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 0bbaa234108d3d..1959b334aa3701 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -628,7 +628,9 @@ def remove_toplevel_addtions(gem, sha) `git reset HEAD~ --` && `git checkout .` && `git clean -fd` puts "Skip commit #{sha} only for tools or toplevel" return false - elsif !system(*%W"git commit --amend --no-edit --", *toplevels, *tools) + elsif system(*%W"git commit --amend --no-edit --", *toplevels, *tools) + return true + else `git reset HEAD~ --` && `git checkout .` && `git clean -fd` return nil end From 7e5c662a6f2e8435f8103bc16185bed6759cc557 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 29 Aug 2023 10:56:56 +0900 Subject: [PATCH 079/208] [Feature #18183] Add `chars:` option to `Random#alphanumeric` --- NEWS.md | 4 ++++ lib/random/formatter.rb | 13 ++++++++++--- test/ruby/test_random_formatter.rb | 14 ++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index 383e6ed04d2425..297ad8c1d309f0 100644 --- a/NEWS.md +++ b/NEWS.md @@ -64,6 +64,9 @@ Note: We're only listing outstanding class updates. ## Stdlib updates +* Random::Formatter#alphanumeric is extended to accept optional `chars` + keyword argument. [[Feature #18183]] + The following default gems are updated. * RubyGems 3.5.0.dev @@ -156,6 +159,7 @@ changelog for details of the default gems or bundled gems. * RJIT exists only for experimental purposes. * You should keep using YJIT in production. +[Feature #18183]: https://bugs.ruby-lang.org/issues/18183 [Feature #18498]: https://bugs.ruby-lang.org/issues/18498 [Feature #18885]: https://bugs.ruby-lang.org/issues/18885 [Bug #19150]: https://bugs.ruby-lang.org/issues/19150 diff --git a/lib/random/formatter.rb b/lib/random/formatter.rb index 4dea61c16c24af..18e6717b6ce193 100644 --- a/lib/random/formatter.rb +++ b/lib/random/formatter.rb @@ -226,11 +226,13 @@ def uuid # # The argument _n_ specifies the length, in characters, of the alphanumeric # string to be generated. + # The argument _chars_ specifies the character list which the result is + # consist of. # # If _n_ is not specified or is nil, 16 is assumed. # It may be larger in the future. # - # The result may contain A-Z, a-z and 0-9. + # The result may contain A-Z, a-z and 0-9, unless _chars_ is specified. # # require 'random/formatter' # @@ -238,8 +240,13 @@ def uuid # # or # prng = Random.new # prng.alphanumeric(10) #=> "i6K93NdqiH" - def alphanumeric(n=nil) + # + # Random.alphanumeric(4, chars: [*"0".."9"])' #=> "2952" + # # or + # prng = Random.new + # prng.alphanumeric(10, chars: [*"!".."/"]) #=> ",.,++%/''." + def alphanumeric(n = nil, chars: ALPHANUMERIC) n = 16 if n.nil? - choose(ALPHANUMERIC, n) + choose(chars, n) end end diff --git a/test/ruby/test_random_formatter.rb b/test/ruby/test_random_formatter.rb index a5072099e1bfb8..2c01d99b3dbe78 100644 --- a/test/ruby/test_random_formatter.rb +++ b/test/ruby/test_random_formatter.rb @@ -83,6 +83,20 @@ def test_alphanumeric end end + def test_alphanumeric_chars + [ + [[*"0".."9"], /\A\d*\z/], + [[*"a".."t"], /\A[a-t]*\z/], + ["一二三四五六七八九十".chars, /\A[一二三四五六七八九十]*\z/], + ].each do |chars, pattern| + 10.times do |n| + an = @it.alphanumeric(n, chars: chars) + assert_match(pattern, an) + assert_equal(n, an.length) + end + end + end + def assert_in_range(range, result, mesg = nil) assert(range.cover?(result), build_message(mesg, "Expected #{result} to be in #{range}")) end From c0e913ae88057fed121d56f2057357b1e4c3b346 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 29 Aug 2023 13:52:18 +0900 Subject: [PATCH 080/208] [DOC] Link method name references --- NEWS.md | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/NEWS.md b/NEWS.md index 297ad8c1d309f0..befe02ccdb865e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,7 +12,9 @@ Note that each entry is kept to a minimum, see links for details. * A new `performance` warning category was introduced. They are not displayed by default even in verbose mode. Turn them on with `-W:performance` or `Warning[:performance] = true`. [[Feature #19538]] -* The `RUBY_GC_HEAP_INIT_SLOTS` environment variable has been deprecated and removed. Environment variables `RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS` should be used instead. [[Feature #19785]] +* The `RUBY_GC_HEAP_INIT_SLOTS` environment variable has been deprecated and + removed. Environment variables `RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS` should be + used instead. [[Feature #19785]] ## Core classes updates @@ -20,25 +22,27 @@ Note: We're only listing outstanding class updates. * Array - * `Array#pack` now raises ArgumentError for unknown directives. [[Bug #19150]] + * Array#pack now raises ArgumentError for unknown directives. [[Bug #19150]] * Dir - * `Dir.for_fd` added for returning a Dir object for the directory specified + * Dir.for_fd added for returning a Dir object for the directory specified by the provided directory file descriptor. [[Feature #19347]] - * `Dir.fchdir` added for changing the directory to the directory specified + * Dir.fchdir added for changing the directory to the directory specified by the provided directory file descriptor. [[Feature #19347]] - * `Dir#chdir` added for changing the directory to the directory specified - by the provided `Dir` object. [[Feature #19347]] + * Dir#chdir added for changing the directory to the directory specified by + the provided `Dir` object. [[Feature #19347]] * MatchData - * MatchData#named_captures now accepts optional `symbolize_names` keyword. [[Feature #19591]] + * MatchData#named_captures now accepts optional `symbolize_names` + keyword. [[Feature #19591]] * String - * `String#unpack` now raises ArgumentError for unknown directives. [[Bug #19150]] - * `String#bytesplice` now accepts new arguments index/length or range of the source string to be copied. [[Feature #19314]] + * String#unpack now raises ArgumentError for unknown directives. [[Bug #19150]] + * String#bytesplice now accepts new arguments index/length or range of the + source string to be copied. [[Feature #19314]] * ObjectSpace::WeakKeyMap @@ -48,7 +52,8 @@ Note: We're only listing outstanding class updates. * Module - * `Module#set_temporary_name` added for setting a temporary name for a module. [[Feature #19521]] + * Module#set_temporary_name added for setting a temporary name for a + module. [[Feature #19521]] * Process.warmup @@ -60,7 +65,8 @@ Note: We're only listing outstanding class updates. * Refinement * Add Refinement#target as an alternative of Refinement#refined_class. - Refinement#refined_class is deprecated and will be removed in Ruby 3.4. [[Feature #19714]] + Refinement#refined_class is deprecated and will be removed in Ruby + 3.4. [[Feature #19714]] ## Stdlib updates @@ -135,13 +141,13 @@ changelog for details of the default gems or bundled gems. * Instance variables no longer exit to the interpreter with megamorphic Object Shapes. * Unsupported call types no longer exit to the interpreter. - * `Integer#!=`, `String#!=`, `Kernel#block_given?`, `Kernel#is_a?`, - `Kernel#instance_of?`, `Module#===` are specially optimized. + * Integer#!=, String#!=, Kernel#block_given?, Kernel#is_a?, + Kernel#instance_of?, Module#=== are specially optimized. * Now more than 3x faster than the interpreter on optcarrot! * Metadata for compiled code uses a lot less memory. * Generate more compact code on ARM64 * Option to start YJIT in paused mode and then later enable it manually - * `--yjit-pause` and `RubyVM::YJIT.resume` + * `--yjit-pause` and RubyVM::YJIT.resume * This can be used to enable YJIT only once your application is done booting * `ratio_in_yjit` stat produced by `--yjit-stats` is now avaiable in release builds, a special stats or dev build is no longer required. From 589cd0e511304be1419d4d7191eaf1063ef83fd5 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 29 Aug 2023 14:17:16 +0900 Subject: [PATCH 081/208] [DOC] Mention about https://bugs.ruby-lang.org/issues/19776 --- NEWS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/NEWS.md b/NEWS.md index befe02ccdb865e..e4bf72bae9f928 100644 --- a/NEWS.md +++ b/NEWS.md @@ -70,6 +70,9 @@ Note: We're only listing outstanding class updates. ## Stdlib updates +* RubyGems and Bundler warn if users require gem that is scheduled to become the bundled gems + in the future version of Ruby. [[Feature #19351]] [[Feature #19776]] [[Feature #19843]] + * Random::Formatter#alphanumeric is extended to accept optional `chars` keyword argument. [[Feature #18183]] @@ -171,8 +174,11 @@ changelog for details of the default gems or bundled gems. [Bug #19150]: https://bugs.ruby-lang.org/issues/19150 [Feature #19314]: https://bugs.ruby-lang.org/issues/19314 [Feature #19347]: https://bugs.ruby-lang.org/issues/19347 +[Feature #19351]: https://bugs.ruby-lang.org/issues/19351 [Feature #19521]: https://bugs.ruby-lang.org/issues/19521 [Feature #19538]: https://bugs.ruby-lang.org/issues/19538 [Feature #19591]: https://bugs.ruby-lang.org/issues/19591 [Feature #19714]: https://bugs.ruby-lang.org/issues/19714 +[Feature #19776]: https://bugs.ruby-lang.org/issues/19776 [Feature #19785]: https://bugs.ruby-lang.org/issues/19785 +[Feature #19843]: https://bugs.ruby-lang.org/issues/19843 From 141102b0b08c4feb682210033cac8d0f042c4beb Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 29 Aug 2023 13:44:03 +0900 Subject: [PATCH 082/208] Expose Test.filter_backtrace for the default gems. --- tool/lib/core_assertions.rb | 37 +++++++++++++++++++++++++++++++++++++ tool/lib/test/unit.rb | 36 ------------------------------------ 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 4887d944c5cc38..23de1f7e50ea67 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -1,6 +1,43 @@ # frozen_string_literal: true module Test + + class << self + ## + # Filter object for backtraces. + + attr_accessor :backtrace_filter + end + + class BacktraceFilter # :nodoc: + def filter bt + return ["No backtrace"] unless bt + + new_bt = [] + pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] + + unless $DEBUG then + bt.each do |line| + break if pattern.match?(line) + new_bt << line + end + + new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? + new_bt = bt.dup if new_bt.empty? + else + new_bt = bt.dup + end + + new_bt + end + end + + self.backtrace_filter = BacktraceFilter.new + + def self.filter_backtrace bt # :nodoc: + backtrace_filter.filter bt + end + module Unit module Assertions def assert_raises(*exp, &b) diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 68d2ab471e1cb9..23dfbd8f0f027d 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -24,42 +24,6 @@ def warn(message, category: nil, **kwargs) # See Test::Unit module Test - class << self - ## - # Filter object for backtraces. - - attr_accessor :backtrace_filter - end - - class BacktraceFilter # :nodoc: - def filter bt - return ["No backtrace"] unless bt - - new_bt = [] - pattern = %r[/(?:lib\/test/|core_assertions\.rb:)] - - unless $DEBUG then - bt.each do |line| - break if pattern.match?(line) - new_bt << line - end - - new_bt = bt.reject { |line| pattern.match?(line) } if new_bt.empty? - new_bt = bt.dup if new_bt.empty? - else - new_bt = bt.dup - end - - new_bt - end - end - - self.backtrace_filter = BacktraceFilter.new - - def self.filter_backtrace bt # :nodoc: - backtrace_filter.filter bt - end - ## # Test::Unit is an implementation of the xUnit testing framework for Ruby. module Unit From 901b6d9c5025a30b3d7a5ed0a2c00baf9cfb061d Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 29 Aug 2023 20:04:14 +1200 Subject: [PATCH 083/208] Validate the typed data before dereferencing the internal struct. (#8315) --- process.c | 5 +++-- test/fiber/test_process.rb | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/process.c b/process.c index 0de0b1f0a3ee19..5468cf08466f1b 100644 --- a/process.c +++ b/process.c @@ -1222,7 +1222,7 @@ rb_waitpid(rb_pid_t pid, int *st, int flags) VALUE status = rb_process_status_wait(pid, flags); if (NIL_P(status)) return 0; - struct rb_process_status *data = RTYPEDDATA_DATA(status); + struct rb_process_status *data = rb_check_typeddata(status, &rb_process_status_type); pid = data->pid; if (st) *st = data->status; @@ -4748,7 +4748,8 @@ rb_f_system(int argc, VALUE *argv, VALUE _) if (pid > 0) { VALUE status = rb_process_status_wait(pid, 0); - struct rb_process_status *data = RTYPEDDATA_DATA(status); + + struct rb_process_status *data = rb_check_typeddata(status, &rb_process_status_type); // Set the last status: rb_obj_freeze(status); diff --git a/test/fiber/test_process.rb b/test/fiber/test_process.rb index a5990be2044a80..cc1694576eb9cd 100644 --- a/test/fiber/test_process.rb +++ b/test/fiber/test_process.rb @@ -34,6 +34,27 @@ def test_system end.join end + def test_system_faulty_process_wait + Thread.new do + scheduler = Scheduler.new + + def scheduler.process_wait(pid, flags) + Fiber.blocking{Process.wait(pid, flags)} + + # Don't return `Process::Status` instance. + return false + end + + Fiber.set_scheduler scheduler + + Fiber.schedule do + assert_raise TypeError do + system("true") + end + end + end.join + end + def test_fork omit 'fork not supported' unless Process.respond_to?(:fork) Thread.new do From 5e81127c1bb810734a410b3f5905e01ab6b37b3c Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 29 Aug 2023 17:41:26 +0900 Subject: [PATCH 084/208] Fixed broken -a option behavior --- tool/sync_default_gems.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 1959b334aa3701..5bdede34a94c0b 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -522,7 +522,7 @@ def commits_in_ranges(gem, repo, default_branch, ranges) # nil: failed #++ - def resolve_conflicts(gem, sha) + def resolve_conflicts(gem, sha, edit) # Forcibly remove any files that we don't want to copy to this repository. # We also ignore them as new `toplevels` even when they don't conflict. ignored_paths = [] @@ -576,6 +576,7 @@ def resolve_conflicts(gem, sha) # AA: unmerged, both added unmerged = changes.map {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} unmerged.compact! + ignore_file_pattern = ignore_file_pattern_for(gem) ignore, conflict = unmerged.partition {|name| ignore_file_pattern =~ name} # Reset ignored files if they conflict unless ignore.empty? @@ -636,7 +637,7 @@ def remove_toplevel_addtions(gem, sha) end end - def pickup_commit(gem, sha) + def pickup_commit(gem, sha, edit) # Attempt to cherry-pick a commit result = IO.popen(%W"git cherry-pick #{sha}", &:read) if result =~ /nothing\ to\ commit/ @@ -651,7 +652,7 @@ def pickup_commit(gem, sha) end # Skip the commit if it's empty or the cherry-pick attempt failed - if /^CONFLICT/ =~ result and !resolve_conflicts(gem, sha) + if /^CONFLICT/ =~ result and !resolve_conflicts(gem, sha, edit) `git reset` && `git checkout .` && `git clean -fd` return nil end @@ -715,7 +716,7 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) ] commits.each do |sha, subject| puts "Pick #{sha} from #{repo}." - case pickup_commit(gem, sha) + case pickup_commit(gem, sha, edit) when false next when nil From f16c50772c97c9cbc2f9f1eb0087224a92c3c99b Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 28 Aug 2023 14:58:33 +0900 Subject: [PATCH 085/208] [rubygems/rubygems] rubocop -a https://github.com/rubygems/rubygems/commit/f240bfad2a --- lib/bundler/rubygems_integration.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index 060be66c8c3e15..d6f179cc96f6dc 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -253,21 +253,21 @@ def replace_require(specs) rescue GemfileNotFound "inline Gemfile" end - be = RUBY_VERSION < ::Gem::BUNDLED_GEMS::SINCE[name] ? "will be" : "is" + be = ::Gem::BUNDLED_GEMS::SINCE[name] > RUBY_VERSION ? "will be" : "is" message = "#{name} #{be} not part of the default gems since Ruby #{::Gem::BUNDLED_GEMS::SINCE[name]}." \ " Add #{name} to your #{target_file}." location = caller_locations(1,1)[0]&.path if File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) caller_gem = nil Gem.path.each do |path| - if location =~ /#{path}\/gems\/([\w\-\.]+)/ + if location =~ %r{#{path}/gems/([\w\-\.]+)} caller_gem = $1 break end end message += " Also contact author of #{caller_gem} to add #{name} into its gemspec." end - warn message, uplevel: 1 + warn message, :uplevel => 1 end end kernel_class.send(:no_warning_require, file) From a28c5151f567cada0d2f5c0c3ec4df7f97b80784 Mon Sep 17 00:00:00 2001 From: Kouhei Yanagita Date: Tue, 29 Aug 2023 14:49:57 +0900 Subject: [PATCH 086/208] Fix Array#bsearch when block returns a non-integer numeric value --- array.c | 4 ++-- test/ruby/test_array.rb | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/array.c b/array.c index b4d4cd5bff1410..376efd36b973ad 100644 --- a/array.c +++ b/array.c @@ -3522,8 +3522,8 @@ rb_ary_bsearch_index(VALUE ary) const VALUE zero = INT2FIX(0); switch (rb_cmpint(rb_funcallv(v, id_cmp, 1, &zero), v, zero)) { case 0: return INT2FIX(mid); - case 1: smaller = 1; break; - case -1: smaller = 0; + case 1: smaller = 0; break; + case -1: smaller = 1; } } else { diff --git a/test/ruby/test_array.rb b/test/ruby/test_array.rb index f58f8a27789eba..6c0db0832b93f1 100644 --- a/test/ruby/test_array.rb +++ b/test/ruby/test_array.rb @@ -3336,6 +3336,8 @@ def test_bsearch_in_find_any_mode assert_equal(nil, a.bsearch {|x| 1 * (2**100) }) assert_equal(nil, a.bsearch {|x| (-1) * (2**100) }) + assert_equal(4, a.bsearch {|x| (4 - x).to_r }) + assert_include([4, 7], a.bsearch {|x| (2**100).coerce((1 - x / 4) * (2**100)).first }) end @@ -3371,6 +3373,8 @@ def test_bsearch_index_in_find_any_mode assert_equal(nil, a.bsearch_index {|x| 1 * (2**100) }) assert_equal(nil, a.bsearch_index {|x| (-1) * (2**100) }) + assert_equal(1, a.bsearch_index {|x| (4 - x).to_r }) + assert_include([1, 2], a.bsearch_index {|x| (2**100).coerce((1 - x / 4) * (2**100)).first }) end From 9126dd4b0e5f7410b32a93c753b06d970c641e04 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 29 Aug 2023 17:27:40 +0900 Subject: [PATCH 087/208] Enable jobserver mode in submake [ci skip] Filter out `-j` option not to reset jobserver mode which is enabled by the environment variable. --- .github/actions/setup/directories/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup/directories/action.yml b/.github/actions/setup/directories/action.yml index 3890c4d9dc1b26..ea5f2720ef343e 100644 --- a/.github/actions/setup/directories/action.yml +++ b/.github/actions/setup/directories/action.yml @@ -94,7 +94,8 @@ runs: run: | touch config.status touch .rbconfig.time - sed -f tool/prereq.status template/Makefile.in common.mk > Makefile + sed -f tool/prereq.status template/Makefile.in > Makefile + sed -f tool/prereq.status template/GNUmakefile.in > GNUmakefile make up # Cleanup, runs even on failure From 0cd92819c933ac3aea8eb173986b6478fcd1fbfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Summer=20=E2=98=80=EF=B8=8F?= <4400771+smmr0@users.noreply.github.com> Date: Sun, 20 Aug 2023 04:09:50 -0600 Subject: [PATCH 088/208] [ruby/irb] Remove unused `PROMPT_N` (https://github.com/ruby/irb/pull/685) https://github.com/ruby/irb/commit/66e69fa0dc --- doc/irb/irb.rd.ja | 4 +--- lib/irb.rb | 9 --------- lib/irb/context.rb | 3 --- lib/irb/init.rb | 6 ------ test/irb/test_cmd.rb | 18 ++++++------------ test/irb/test_context.rb | 1 - test/irb/yamatanooroti/test_rendering.rb | 2 -- 7 files changed, 7 insertions(+), 36 deletions(-) diff --git a/doc/irb/irb.rd.ja b/doc/irb/irb.rd.ja index 633c08cbd4c846..c51e0bd60d566b 100644 --- a/doc/irb/irb.rd.ja +++ b/doc/irb/irb.rd.ja @@ -125,7 +125,6 @@ irb起動時に``~/.irbrc''を読み込みます. もし存在しない場合は IRB.conf[:PROMPT][:MY_PROMPT] = { # プロンプトモードの名前 :PROMPT_I => nil, # 通常のプロンプト - :PROMPT_N => nil, # 継続行のプロンプト :PROMPT_S => nil, # 文字列などの継続行のプロンプト :PROMPT_C => nil, # 式が継続している時のプロンプト :RETURN => " ==>%s\n" # リターン時のプロンプト @@ -140,7 +139,7 @@ OKです. IRB.conf[:PROMPT_MODE] = :MY_PROMPT -PROMPT_I, PROMPT_N, PROMPT_S, PROMPT_Cは, フォーマットを指定します. +PROMPT_I, PROMPT_S, PROMPT_Cは, フォーマットを指定します. %N 起動しているコマンド名が出力される. %m mainオブジェクト(self)がto_sで出力される. @@ -155,7 +154,6 @@ PROMPT_I, PROMPT_N, PROMPT_S, PROMPT_Cは, フォーマットを指定します. IRB.conf[:PROMPT][:DEFAULT] = { :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => "%N(%m):%03n:%i> ", :PROMPT_S => "%N(%m):%03n:%i%l ", :PROMPT_C => "%N(%m):%03n:%i* ", :RETURN => "=> %s\n" diff --git a/lib/irb.rb b/lib/irb.rb index aa412583056b3c..db0aed055e962c 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -197,7 +197,6 @@ # # IRB.conf[:PROMPT_MODE][:DEFAULT] = { # :PROMPT_I => "%N(%m):%03n> ", -# :PROMPT_N => "%N(%m):%03n> ", # :PROMPT_S => "%N(%m):%03n%l ", # :PROMPT_C => "%N(%m):%03n* ", # :RETURN => "%s\n" # used to printf @@ -207,35 +206,30 @@ # # # :NULL: # # :PROMPT_I: -# # :PROMPT_N: # # :PROMPT_S: # # :PROMPT_C: # # :RETURN: | # # %s # # :DEFAULT: # # :PROMPT_I: ! '%N(%m):%03n> ' -# # :PROMPT_N: ! '%N(%m):%03n> ' # # :PROMPT_S: ! '%N(%m):%03n%l ' # # :PROMPT_C: ! '%N(%m):%03n* ' # # :RETURN: | # # => %s # # :CLASSIC: # # :PROMPT_I: ! '%N(%m):%03n:%i> ' -# # :PROMPT_N: ! '%N(%m):%03n:%i> ' # # :PROMPT_S: ! '%N(%m):%03n:%i%l ' # # :PROMPT_C: ! '%N(%m):%03n:%i* ' # # :RETURN: | # # %s # # :SIMPLE: # # :PROMPT_I: ! '>> ' -# # :PROMPT_N: ! '>> ' # # :PROMPT_S: # # :PROMPT_C: ! '?> ' # # :RETURN: | # # => %s # # :INF_RUBY: # # :PROMPT_I: ! '%N(%m):%03n> ' -# # :PROMPT_N: # # :PROMPT_S: # # :PROMPT_C: # # :RETURN: | @@ -243,7 +237,6 @@ # # :AUTO_INDENT: true # # :XMP: # # :PROMPT_I: -# # :PROMPT_N: # # :PROMPT_S: # # :PROMPT_C: # # :RETURN: |2 @@ -528,8 +521,6 @@ def eval_input f = @context.prompt_s elsif continue f = @context.prompt_c - elsif indent > 0 - f = @context.prompt_n else f = @context.prompt_i end diff --git a/lib/irb/context.rb b/lib/irb/context.rb index 43d9b53435df98..6d6261e60b43ac 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -229,8 +229,6 @@ def main # # See IRB@Customizing+the+IRB+Prompt for more information. attr_accessor :prompt_c - # See IRB@Customizing+the+IRB+Prompt for more information. - attr_accessor :prompt_n # Can be either the default IRB.conf[:AUTO_INDENT], or the # mode set by #prompt_mode= # @@ -414,7 +412,6 @@ def prompt_mode=(mode) @prompt_i = pconf[:PROMPT_I] @prompt_s = pconf[:PROMPT_S] @prompt_c = pconf[:PROMPT_C] - @prompt_n = pconf[:PROMPT_N] @return_format = pconf[:RETURN] @return_format = "%s\n" if @return_format == nil if ai = pconf.include?(:AUTO_INDENT) diff --git a/lib/irb/init.rb b/lib/irb/init.rb index ef07a5f1e6b3f9..d1097b573854ef 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -58,35 +58,30 @@ def IRB.init_config(ap_path) @CONF[:PROMPT] = { :NULL => { :PROMPT_I => nil, - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => "%s\n" }, :DEFAULT => { :PROMPT_I => "%N(%m):%03n> ", - :PROMPT_N => "%N(%m):%03n> ", :PROMPT_S => "%N(%m):%03n%l ", :PROMPT_C => "%N(%m):%03n* ", :RETURN => "=> %s\n" }, :CLASSIC => { :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => "%N(%m):%03n:%i> ", :PROMPT_S => "%N(%m):%03n:%i%l ", :PROMPT_C => "%N(%m):%03n:%i* ", :RETURN => "%s\n" }, :SIMPLE => { :PROMPT_I => ">> ", - :PROMPT_N => ">> ", :PROMPT_S => "%l> ", :PROMPT_C => "?> ", :RETURN => "=> %s\n" }, :INF_RUBY => { :PROMPT_I => "%N(%m):%03n> ", - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => "%s\n", @@ -94,7 +89,6 @@ def IRB.init_config(ap_path) }, :XMP => { :PROMPT_I => nil, - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => " ==>%s\n" diff --git a/test/irb/test_cmd.rb b/test/irb/test_cmd.rb index 670078679f1152..e8c959ec39956f 100644 --- a/test/irb/test_cmd.rb +++ b/test/irb/test_cmd.rb @@ -211,8 +211,7 @@ def test_measure DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -241,8 +240,7 @@ def test_measure_keeps_previous_value DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -269,8 +267,7 @@ def test_measure_enabled_by_rc DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -300,8 +297,7 @@ def test_measure_enabled_by_rc_with_custom DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -331,8 +327,7 @@ def test_measure_with_custom DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, @@ -358,8 +353,7 @@ def test_measure_with_proc DEFAULT: { PROMPT_I: '> ', PROMPT_S: '> ', - PROMPT_C: '> ', - PROMPT_N: '> ' + PROMPT_C: '> ' } }, PROMPT_MODE: :DEFAULT, diff --git a/test/irb/test_context.rb b/test/irb/test_context.rb index 5847df172e9e07..29c67392f39869 100644 --- a/test/irb/test_context.rb +++ b/test/irb/test_context.rb @@ -454,7 +454,6 @@ def main.inspect def test_default_return_format IRB.conf[:PROMPT][:MY_PROMPT] = { :PROMPT_I => "%03n> ", - :PROMPT_N => "%03n> ", :PROMPT_S => "%03n> ", :PROMPT_C => "%03n> " # without :RETURN diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index 3da3935c195c07..80833d05222f89 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -179,7 +179,6 @@ def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_right write_irbrc <<~'LINES' IRB.conf[:PROMPT][:MY_PROMPT] = { :PROMPT_I => "%03n> ", - :PROMPT_N => "%03n> ", :PROMPT_S => "%03n> ", :PROMPT_C => "%03n> " } @@ -214,7 +213,6 @@ def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_left write_irbrc <<~'LINES' IRB.conf[:PROMPT][:MY_PROMPT] = { :PROMPT_I => "%03n> ", - :PROMPT_N => "%03n> ", :PROMPT_S => "%03n> ", :PROMPT_C => "%03n> " } From 221c2d0e19171f059461f8a8c3489cd9578ed43e Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Wed, 23 Aug 2023 16:37:16 +0100 Subject: [PATCH 089/208] [ruby/irb] Print deprecation message for prompt_n methods (https://github.com/ruby/irb/pull/691) They were removed in #685, but we should still keep them to avoid breaking changes to tools like Chef. https://github.com/chef/chef/blob/533ff089479763f29045e4e6ddf388b73fc99338/lib/chef/shell.rb#L138 https://github.com/ruby/irb/commit/b585e0c835 --- lib/irb/context.rb | 13 +++++++++++++ test/irb/test_context.rb | 12 ++++++++++++ 2 files changed, 25 insertions(+) diff --git a/lib/irb/context.rb b/lib/irb/context.rb index 6d6261e60b43ac..a20510d73c3ee8 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -229,6 +229,19 @@ def main # # See IRB@Customizing+the+IRB+Prompt for more information. attr_accessor :prompt_c + + # TODO: Remove this when developing v2.0 + def prompt_n + warn "IRB::Context#prompt_n is deprecated and will be removed in the next major release." + "" + end + + # TODO: Remove this when developing v2.0 + def prompt_n=(_) + warn "IRB::Context#prompt_n= is deprecated and will be removed in the next major release." + "" + end + # Can be either the default IRB.conf[:AUTO_INDENT], or the # mode set by #prompt_mode= # diff --git a/test/irb/test_context.rb b/test/irb/test_context.rb index 29c67392f39869..dae0f8f6e4c21b 100644 --- a/test/irb/test_context.rb +++ b/test/irb/test_context.rb @@ -90,6 +90,18 @@ def test_eval_input_raise2x ], out) end + def test_prompt_n_deprecation + irb = IRB::Irb.new(IRB::WorkSpace.new(Object.new)) + + out, err = capture_output do + irb.context.prompt_n = "foo" + irb.context.prompt_n + end + + assert_include err, "IRB::Context#prompt_n is deprecated" + assert_include err, "IRB::Context#prompt_n= is deprecated" + end + def test_output_to_pipe require 'stringio' input = TestInputMethod.new(["n=1"]) From 13ed1d7b6012e006369a9074bdc070f1b6d57fc6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Sat, 26 Aug 2023 14:40:25 -0400 Subject: [PATCH 090/208] [ruby/yarp] Remove unnecessary NUL byte in string https://github.com/ruby/yarp/commit/af867e35b1 --- yarp/templates/src/node.c.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarp/templates/src/node.c.erb b/yarp/templates/src/node.c.erb index f837ca0324a6f7..c173f677601612 100644 --- a/yarp/templates/src/node.c.erb +++ b/yarp/templates/src/node.c.erb @@ -173,5 +173,5 @@ yp_node_type_to_str(yp_node_type_t node_type) return "<%= node.type %>"; <%- end -%> } - return "\0"; + return ""; } From 0a219ef44af0cfb126799a7109c2e658172d7819 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 10 Aug 2023 15:43:11 +0900 Subject: [PATCH 091/208] jruby: Add StringIO::VERSION (#59) Fixes GH-57 --- test/stringio/test_stringio.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb index 246f107d053899..d2d96c5719a43c 100644 --- a/test/stringio/test_stringio.rb +++ b/test/stringio/test_stringio.rb @@ -14,6 +14,10 @@ def open_file(content) include TestEOF::Seek + def test_version + assert_kind_of(String, StringIO::VERSION) + end + def test_initialize assert_kind_of StringIO, StringIO.new assert_kind_of StringIO, StringIO.new('str') From 5ed42c980067d1e9970cfe2b5bc0527033e92a12 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Thu, 10 Aug 2023 15:48:54 +0900 Subject: [PATCH 092/208] Development of 3.0.9 started. --- ext/stringio/stringio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index f5213f2fc0fc92..e89a1d780cf996 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -12,7 +12,7 @@ **********************************************************************/ -#define STRINGIO_VERSION "3.0.8" +#define STRINGIO_VERSION "3.0.9" #include "ruby.h" #include "ruby/io.h" From 0744da1b3bdaad43bf98f2d8b2425841f787b2dc Mon Sep 17 00:00:00 2001 From: git Date: Tue, 29 Aug 2023 09:16:36 +0000 Subject: [PATCH 093/208] Update default gems list at 5ed42c980067d1e9970cfe2b5bc052 [ci skip] --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e4bf72bae9f928..9f6012394efe38 100644 --- a/NEWS.md +++ b/NEWS.md @@ -90,7 +90,7 @@ The following default gems are updated. * optparse 0.4.0.pre.1 * psych 5.1.0 * reline 0.3.8 -* stringio 3.0.8 +* stringio 3.0.9 * strscan 3.0.7 * syntax_suggest 1.1.0 * time 0.2.2 From e46e48d690cf543193418ef08725cdbe31688975 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 29 Aug 2023 22:24:55 +1200 Subject: [PATCH 094/208] Expose `rb_process_status_wait` and hide `rb_process_status_waitv`. (#8316) --- include/ruby/internal/intern/process.h | 9 +++++++++ process.c | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/ruby/internal/intern/process.h b/include/ruby/internal/intern/process.h index 7a7b24ed4b0ae0..5dcf85e80c04e1 100644 --- a/include/ruby/internal/intern/process.h +++ b/include/ruby/internal/intern/process.h @@ -30,6 +30,15 @@ RBIMPL_SYMBOL_EXPORT_BEGIN() /* process.c */ +/** + * Wait for the specified process to terminate, reap it, and return its status. + * + * @param[in] pid The process ID to wait for. + * @param[in] flags The flags to pass to waitpid(2). + * @return VALUE An instance of Process::Status. + */ +VALUE rb_process_status_wait(rb_pid_t pid, int flags); + /** * Sets the "last status", or the `$?`. * diff --git a/process.c b/process.c index 5468cf08466f1b..377da3829a814c 100644 --- a/process.c +++ b/process.c @@ -1197,7 +1197,7 @@ rb_process_status_wait(rb_pid_t pid, int flags) * This is an EXPERIMENTAL FEATURE. */ -VALUE +static VALUE rb_process_status_waitv(int argc, VALUE *argv, VALUE _) { rb_check_arity(argc, 0, 2); From 247fa3ca7666a5873a575ccfdb78c87753c2ad80 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 29 Aug 2023 19:40:52 +0900 Subject: [PATCH 095/208] [DOC] Remove typo --- array.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/array.c b/array.c index 376efd36b973ad..65041c9365f359 100644 --- a/array.c +++ b/array.c @@ -4310,7 +4310,7 @@ rb_ary_reject(VALUE ary) * a = [:foo, 'bar', 2] * a.delete_if # => # * -3 */ + */ static VALUE rb_ary_delete_if(VALUE ary) From cfae3ed42224b3e59cd6a02bf25a1d820e3b2b36 Mon Sep 17 00:00:00 2001 From: Petrik Date: Tue, 29 Aug 2023 12:37:46 +0200 Subject: [PATCH 096/208] Fix code example doc for Random.alphanumeric --- lib/random/formatter.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/random/formatter.rb b/lib/random/formatter.rb index 18e6717b6ce193..45b53a7b6e620e 100644 --- a/lib/random/formatter.rb +++ b/lib/random/formatter.rb @@ -241,7 +241,7 @@ def uuid # prng = Random.new # prng.alphanumeric(10) #=> "i6K93NdqiH" # - # Random.alphanumeric(4, chars: [*"0".."9"])' #=> "2952" + # Random.alphanumeric(4, chars: [*"0".."9"]) #=> "2952" # # or # prng = Random.new # prng.alphanumeric(10, chars: [*"!".."/"]) #=> ",.,++%/''." From f0b43597ffbef0a43ca3459d8807bdc00ba4ec59 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Tue, 29 Aug 2023 23:53:28 +1200 Subject: [PATCH 097/208] [DOC] Improved documentation. (#8319) --- io_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io_buffer.c b/io_buffer.c index 8d09e09a894f74..d987b8fa38e3cf 100644 --- a/io_buffer.c +++ b/io_buffer.c @@ -2181,8 +2181,8 @@ rb_io_buffer_initialize_copy(VALUE self, VALUE source) * call-seq: * copy(source, [offset, [length, [source_offset]]]) -> size * - * Efficiently copy buffer from a source IO::Buffer into the buffer, - * at +offset+ using +memcpy+. For copying String instances, see #set_string. + * Efficiently copy from a source IO::Buffer into the buffer, at +offset+ + * using +memcpy+. For copying String instances, see #set_string. * * buffer = IO::Buffer.new(32) * # => From 0cfb7796928f16f967fcf7b4d0a49fab6774001f Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Tue, 29 Aug 2023 13:48:29 +0100 Subject: [PATCH 098/208] [ruby/irb] Fix test warnings (https://github.com/ruby/irb/pull/698) * Encoding should be saved before creating Irb objects * Fix unused local warning https://github.com/ruby/irb/commit/036ec31034 --- test/irb/test_context.rb | 2 +- test/irb/test_irb.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/irb/test_context.rb b/test/irb/test_context.rb index dae0f8f6e4c21b..f59a23a99df9c4 100644 --- a/test/irb/test_context.rb +++ b/test/irb/test_context.rb @@ -93,7 +93,7 @@ def test_eval_input_raise2x def test_prompt_n_deprecation irb = IRB::Irb.new(IRB::WorkSpace.new(Object.new)) - out, err = capture_output do + _, err = capture_output do irb.context.prompt_n = "foo" irb.context.prompt_n end diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index c685912093dffb..08fe41f5e7854e 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -94,8 +94,8 @@ def dynamic_prompt(&block) end def setup - @irb = build_irb save_encodings + @irb = build_irb end def teardown From 6ed1a504d49af09aac543ef39facaa11639add7e Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Tue, 29 Aug 2023 13:54:21 +0100 Subject: [PATCH 099/208] [ruby/irb] irb:rdbg cleanups (https://github.com/ruby/irb/pull/697) * Remove unused method and constant from IRB::Debug * Update comments https://github.com/ruby/irb/commit/98914a963c --- lib/irb.rb | 7 +++---- lib/irb/debug.rb | 12 ------------ 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/lib/irb.rb b/lib/irb.rb index db0aed055e962c..44477b487f7bba 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -439,9 +439,9 @@ def initialize(workspace = nil, input_method = nil) @scanner = RubyLex.new(@context) end - # A hook point for `debug` command's TracePoint after :IRB_EXIT as well as its clean-up + # A hook point for `debug` command's breakpoint after :IRB_EXIT as well as its clean-up def debug_break - # it means the debug command is executed + # it means the debug integration has been activated if defined?(DEBUGGER__) && DEBUGGER__.respond_to?(:capture_frames_without_irb) # after leaving this initial breakpoint, revert the capture_frames patch DEBUGGER__.singleton_class.send(:alias_method, :capture_frames, :capture_frames_without_irb) @@ -547,14 +547,13 @@ def eval_input each_top_level_statement do |statement, line_no| signal_status(:IN_EVAL) do begin - # If the integration with debugger is activated, we need to handle certain input differently + # If the integration with debugger is activated, we return certain input if it should be dealt with by debugger if @context.with_debugger && statement.should_be_handled_by_debugger? return statement.code end @context.evaluate(statement.evaluable_code, line_no) - # Don't echo if the line ends with a semicolon if @context.echo? && !statement.suppresses_echo? if statement.is_assignment? if @context.echo_on_assignment? diff --git a/lib/irb/debug.rb b/lib/irb/debug.rb index f72282299322c0..f819f850b10abf 100644 --- a/lib/irb/debug.rb +++ b/lib/irb/debug.rb @@ -2,10 +2,6 @@ module IRB module Debug - BINDING_IRB_FRAME_REGEXPS = [ - '', - binding.method(:irb).source_location.first, - ].map { |file| /\A#{Regexp.escape(file)}:\d+:in `irb'\z/ } IRB_DIR = File.expand_path('..', __dir__) class << self @@ -76,14 +72,6 @@ def configure_irb_for_debugger(irb) irb.context.irb_name += ":rdbg" end - def binding_irb? - caller.any? do |frame| - BINDING_IRB_FRAME_REGEXPS.any? do |regexp| - frame.match?(regexp) - end - end - end - module SkipPathHelperForIRB def skip_internal_path?(path) # The latter can be removed once https://github.com/ruby/debug/issues/866 is resolved From 51e7fb533176abb53027729bcd52e77245757f10 Mon Sep 17 00:00:00 2001 From: Chad Schroeder Date: Tue, 29 Aug 2023 08:10:31 -0500 Subject: [PATCH 100/208] [ruby/irb] fixes https://github.com/ruby/irb/pull/524 (https://github.com/ruby/irb/pull/696) https://github.com/ruby/irb/commit/59bcc07def --- lib/irb.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/irb.rb b/lib/irb.rb index 44477b487f7bba..57ec911f344b3e 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -535,7 +535,6 @@ def eval_input prompt_i = @context.prompt_i.nil? ? "" : @context.prompt_i ind = prompt(prompt_i, ltype, indent, line_no)[/.*\z/].size + indent * 2 - p.size - ind += 2 if continue @context.io.prompt = p + " " * ind if ind > 0 end end From 062eec75582dc0c813c945c2ce900e6ce0487610 Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Wed, 30 Aug 2023 00:34:45 +0900 Subject: [PATCH 101/208] [ruby/irb] Add --nomultiline indent and prompt test (https://github.com/ruby/irb/pull/699) https://github.com/ruby/irb/commit/9b4aea753b --- test/irb/yamatanooroti/test_rendering.rb | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index 80833d05222f89..279eff6515a144 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -47,6 +47,35 @@ def test_launch EOC end + def test_nomultiline + write_irbrc <<~'LINES' + puts 'start IRB' + LINES + start_terminal(25, 80, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb --nomultiline}, startup_message: 'start IRB') + write(<<~EOC) + if true + if false + a = "hello + world" + puts a + end + end + EOC + close + assert_screen(<<~EOC) + start IRB + irb(main):001> if true + irb(main):002* if false + irb(main):003* a = "hello + irb(main):004" world" + irb(main):005* puts a + irb(main):006* end + irb(main):007* end + => nil + irb(main):008> + EOC + end + def test_multiline_paste write_irbrc <<~'LINES' puts 'start IRB' From 36e210718cc0242d3d1a043bd32b4618db592024 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Tue, 29 Aug 2023 09:48:10 -0400 Subject: [PATCH 102/208] [ruby/yarp] simplify `context_pop` https://github.com/ruby/yarp/commit/fe85b595b6 --- yarp/yarp.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/yarp/yarp.c b/yarp/yarp.c index ad433efc4b47d5..8cd2c2007b5029 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -4902,14 +4902,9 @@ context_push(yp_parser_t *parser, yp_context_t context) { static void context_pop(yp_parser_t *parser) { - if (parser->current_context->prev == NULL) { - free(parser->current_context); - parser->current_context = NULL; - } else { - yp_context_node_t *prev = parser->current_context->prev; - free(parser->current_context); - parser->current_context = prev; - } + yp_context_node_t *prev = parser->current_context->prev; + free(parser->current_context); + parser->current_context = prev; } static bool From 535045ab3b27dd42030e837bcd83b74ff075b84b Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Tue, 29 Aug 2023 12:27:00 -0400 Subject: [PATCH 103/208] [YARP] Compile basic types (#8311) * Add a compile_context arg to yp_compile_node The compile_context will allow us to pass around the parser, and the constants and lookup table (to be used in future commits). * Compile yp_program_node_t and yp_statements_node_t Add the compilation for program and statements node so that we can successfully compile an empty program with YARP. * Helper functions for parsing numbers, strings, and symbols * Compile basic numeric / boolean node types in YARP * Compile StringNode and SymbolNodes in YARP * Compile several basic node types in YARP * Added error return for missing node --- compile.c | 33 +++- iseq.c | 7 +- yarp/yarp_compiler.c | 454 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 481 insertions(+), 13 deletions(-) diff --git a/compile.c b/compile.c index d7f345d0af1dab..2673df5dbb5fbb 100644 --- a/compile.c +++ b/compile.c @@ -45,8 +45,6 @@ #include "insns_info.inc" #include "yarp/yarp.h" -VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer); - #undef RUBY_UNTYPED_DATA_WARNING #define RUBY_UNTYPED_DATA_WARNING 0 @@ -858,15 +856,40 @@ rb_iseq_compile_node(rb_iseq_t *iseq, const NODE *node) return iseq_setup(iseq, ret); } -static VALUE rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret); +typedef struct yp_compile_context { + yp_parser_t *parser; + struct yp_compile_context *previous; + ID *constants; + st_table *index_lookup_table; +} yp_compile_context_t; + +static VALUE rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, yp_compile_context_t *compile_context); VALUE -rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer) +rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t *yarp_pointer, yp_parser_t *parser) { DECL_ANCHOR(ret); INIT_ANCHOR(ret); - CHECK(rb_translate_yarp(iseq, yarp_pointer, ret)); + ID *constants = calloc(parser->constant_pool.size, sizeof(ID)); + rb_encoding *encoding = rb_enc_find(parser->encoding.name); + + for (size_t index = 0; index < parser->constant_pool.capacity; index++) { + yp_constant_t constant = parser->constant_pool.constants[index]; + + if (constant.id != 0) { + constants[constant.id - 1] = rb_intern3(constant.start, constant.length, encoding); + } + } + + yp_compile_context_t compile_context = { + .parser = parser, + .previous = NULL, + .constants = constants + }; + + CHECK(rb_translate_yarp(iseq, yarp_pointer, ret, &compile_context)); + free(constants); CHECK(iseq_setup_insn(iseq, ret)); return iseq_setup(iseq, ret); diff --git a/iseq.c b/iseq.c index 0b7ea138fa7e10..cb54d267f4950a 100644 --- a/iseq.c +++ b/iseq.c @@ -45,8 +45,6 @@ #include "insns_info.inc" #include "yarp/yarp.h" -VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer); - VALUE rb_cISeq; static VALUE iseqw_new(const rb_iseq_t *iseq); static const rb_iseq_t *iseqw_check(VALUE iseqw); @@ -944,6 +942,8 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea return iseq_translate(iseq); } +VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer, yp_parser_t *parser); + rb_iseq_t * rb_iseq_new_with_callback( const struct rb_iseq_new_with_callback_callback_func * ifunc, @@ -1404,8 +1404,9 @@ iseqw_s_compile_yarp(int argc, VALUE *argv, VALUE self) prepare_iseq_build(iseq, name, file, path, first_lineno, &node_location, node_id, parent, 0, (enum rb_iseq_type)iseq_type, Qnil, &option); - rb_iseq_compile_yarp_node(iseq, node); + rb_iseq_compile_yarp_node(iseq, node, &parser); + finish_iseq_build(iseq); yp_node_destroy(&parser, node); yp_parser_free(&parser); diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index ae8575e33ac3fe..81c5a36d58c671 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -1,17 +1,461 @@ #include "yarp.h" +static VALUE +parse_number(const yp_node_t *node) { + const char *start = node->location.start; + const char *end = node->location.end; + size_t length = end - start; + + char *buffer = malloc(length + 1); + memcpy(buffer, start, length); + + buffer[length] = '\0'; + VALUE number = rb_cstr_to_inum(buffer, -10, Qfalse); + + free(buffer); + return number; +} + +static inline VALUE +parse_string(yp_string_t *string) { + return rb_str_new(yp_string_source(string), yp_string_length(string)); +} + +static inline ID +parse_symbol(const char *start, const char *end) { + return rb_intern2(start, end - start); +} + +static inline ID +parse_node_symbol(yp_node_t *node) { + return parse_symbol(node->location.start, node->location.end); +} + +static inline ID +parse_string_symbol(yp_string_t *string) { + const char *start = yp_string_source(string); + return parse_symbol(start, start + yp_string_length(string)); +} + +static inline ID +parse_location_symbol(yp_location_t *location) { + return parse_symbol(location->start, location->end); +} + +/* + * Compiles a YARP node into instruction sequences + * + * iseq - The current instruction sequence object (used for locals) + * node - The yarp node to compile + * ret - The linked list of instruction sequences to append instructions onto + * popped - True if compiling something with no side effects, so instructions don't + * need to be added + * compile_context - Stores parser and local information + */ static void -yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped) { - return; +yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) { + yp_parser_t *parser = compile_context->parser; + yp_newline_list_t newline_list = parser->newline_list; + int lineno = (int)yp_newline_list_line_column(&newline_list, node->location.start).line; + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + + switch (YP_NODE_TYPE(node)) { + case YP_NODE_ALIAS_NODE: { + yp_alias_node_t *alias_node = (yp_alias_node_t *) node; + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_VMCORE); + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_CBASE); + + yp_compile_node(iseq, alias_node->new_name, ret, src, popped, compile_context); + yp_compile_node(iseq, alias_node->old_name, ret, src, popped, compile_context); + + ADD_SEND(ret, &dummy_line_node, id_core_set_method_alias, INT2FIX(3)); + return; + } + case YP_NODE_AND_NODE: { + yp_and_node_t *and_node = (yp_and_node_t *) node; + + LABEL *end_label = NEW_LABEL(lineno); + yp_compile_node(iseq, and_node->left, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ADD_INSNL(ret, &dummy_line_node, branchunless, end_label); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + yp_compile_node(iseq, and_node->right, ret, src, popped, compile_context); + ADD_LABEL(ret, end_label); + return; + } + case YP_NODE_ARGUMENTS_NODE: { + yp_arguments_node_t *arguments_node = (yp_arguments_node_t *) node; + yp_node_list_t node_list = arguments_node->arguments; + for (size_t index = 0; index < node_list.size; index++) { + yp_compile_node(iseq, node_list.nodes[index], ret, src, popped, compile_context); + } + return; + } + case YP_NODE_ASSOC_NODE: { + yp_assoc_node_t *assoc_node = (yp_assoc_node_t *) node; + yp_compile_node(iseq, assoc_node->key, ret, src, popped, compile_context); + yp_compile_node(iseq, assoc_node->value, ret, src, popped, compile_context); + return; + } + case YP_NODE_BEGIN_NODE: { + yp_begin_node_t *begin_node = (yp_begin_node_t *) node; + if (begin_node->statements) { + yp_compile_node(iseq, (yp_node_t *)begin_node->statements, ret, src, popped, compile_context); + } + return; + } + case YP_NODE_CLASS_VARIABLE_READ_NODE: + if (!popped) { + ID cvar_name = parse_node_symbol((yp_node_t *)node); + ADD_INSN2( + ret, + &dummy_line_node, + getclassvariable, + ID2SYM(cvar_name), + get_cvar_ic_value(iseq, cvar_name) + ); + } + return; + case YP_NODE_CLASS_VARIABLE_WRITE_NODE: { + yp_class_variable_write_node_t *write_node = (yp_class_variable_write_node_t *) node; + yp_compile_node(iseq, write_node->value, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ID cvar_name = parse_location_symbol(&write_node->name_loc); + ADD_INSN2(ret, &dummy_line_node, setclassvariable, ID2SYM(cvar_name), get_cvar_ic_value(iseq, cvar_name)); + return; + } + case YP_NODE_CONSTANT_PATH_NODE: { + yp_constant_path_node_t *constant_path_node = (yp_constant_path_node_t*) node; + if (constant_path_node->parent) { + yp_compile_node(iseq, constant_path_node->parent, ret, src, popped, compile_context); + } + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + ADD_INSN1(ret, &dummy_line_node, getconstant, ID2SYM(parse_node_symbol((yp_node_t *)constant_path_node->child))); + return; + } + case YP_NODE_CONSTANT_PATH_WRITE_NODE: { + yp_constant_path_write_node_t *constant_path_write_node = (yp_constant_path_write_node_t*) node; + yp_compile_node(iseq, constant_path_write_node->value, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ID constant_var_name = parse_location_symbol(&constant_path_write_node->target->base.location); + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + ADD_INSN1(ret, &dummy_line_node, setconstant, ID2SYM(constant_var_name)); + return; + } + + case YP_NODE_CONSTANT_READ_NODE: { + yp_constant_read_node_t *constant_read_node = (yp_constant_read_node_t *) node; + ADD_INSN(ret, &dummy_line_node, putnil); + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + ADD_INSN1(ret, &dummy_line_node, getconstant, ID2SYM(parse_node_symbol((yp_node_t *)constant_read_node))); + return; + } + case YP_NODE_CONSTANT_WRITE_NODE: { + yp_constant_write_node_t *constant_write_node = (yp_constant_write_node_t *) node; + yp_compile_node(iseq, constant_write_node->value, ret, src, popped, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + ID constant_name = parse_location_symbol(&constant_write_node->name_loc); + ADD_INSN1(ret, &dummy_line_node, setconstant, ID2SYM(constant_name)); + return; + } + case YP_NODE_EMBEDDED_VARIABLE_NODE: { + yp_embedded_variable_node_t *embedded_node = (yp_embedded_variable_node_t *)node; + yp_compile_node(iseq, embedded_node->variable, ret, src, popped, compile_context); + return; + } + case YP_NODE_FALSE_NODE: + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + } + return; + case YP_NODE_FLOAT_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); + } + return; + } + case YP_NODE_GLOBAL_VARIABLE_READ_NODE: + if (!popped) { + ID gvar_name = parse_node_symbol((yp_node_t *)node); + ADD_INSN1(ret, &dummy_line_node, getglobal, ID2SYM(gvar_name)); + } + return; + case YP_NODE_GLOBAL_VARIABLE_WRITE_NODE: { + yp_global_variable_write_node_t *write_node = (yp_global_variable_write_node_t *) node; + yp_compile_node(iseq, write_node->value, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ID ivar_name = parse_location_symbol(&write_node->name_loc); + ADD_INSN1(ret, &dummy_line_node, setglobal, ID2SYM(ivar_name)); + return; + } + case YP_NODE_IMAGINARY_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); + } + return; + } + case YP_NODE_INSTANCE_VARIABLE_READ_NODE: { + if (!popped) { + ID ivar_name = parse_node_symbol((yp_node_t *)node); + ADD_INSN2(ret, &dummy_line_node, getinstancevariable, + ID2SYM(ivar_name), + get_ivar_ic_value(iseq, ivar_name)); + } + return; + } + case YP_NODE_INSTANCE_VARIABLE_WRITE_NODE: { + yp_instance_variable_write_node_t *write_node = (yp_instance_variable_write_node_t *) node; + yp_compile_node(iseq, write_node->value, ret, src, popped, compile_context); + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ID ivar_name = parse_location_symbol(&write_node->name_loc); + ADD_INSN2(ret, &dummy_line_node, setinstancevariable, + ID2SYM(ivar_name), + get_ivar_ic_value(iseq, ivar_name)); + return; + } + case YP_NODE_INTEGER_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); + } + return; + } + case YP_NODE_INTERPOLATED_STRING_NODE: { + yp_interpolated_string_node_t *interp_string_node= (yp_interpolated_string_node_t *) node; + + for (size_t index = 0; index < interp_string_node->parts.size; index++) { + yp_node_t *part = interp_string_node->parts.nodes[index]; + + switch (part->type) { + case YP_NODE_STRING_NODE: { + yp_string_node_t *string_node = (yp_string_node_t *) part; + ADD_INSN1(ret, &dummy_line_node, putobject,parse_string(&string_node->unescaped)); + break; + } + default: + yp_compile_node(iseq, part, ret, src, popped, compile_context); + ADD_INSN(ret, &dummy_line_node, dup); + ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); + ADD_INSN(ret, &dummy_line_node, anytostring); + break; + } + } + + if (interp_string_node->parts.size > 1) { + ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX((int)(interp_string_node->parts.size))); + } + return; + } + case YP_NODE_INTERPOLATED_SYMBOL_NODE: { + yp_interpolated_symbol_node_t *interp_symbol_node= (yp_interpolated_symbol_node_t *) node; + + for (size_t index = 0; index < interp_symbol_node->parts.size; index++) { + yp_node_t *part = interp_symbol_node->parts.nodes[index]; + + switch (part->type) { + case YP_NODE_STRING_NODE: { + yp_string_node_t *string_node = (yp_string_node_t *) part; + ADD_INSN1(ret, &dummy_line_node, putobject, parse_string(&string_node->unescaped)); + break; + } + default: + yp_compile_node(iseq, part, ret, src, popped, compile_context); + ADD_INSN(ret, &dummy_line_node, dup); + ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE, NULL, FALSE)); + ADD_INSN(ret, &dummy_line_node, anytostring); + break; + } + } + + if (interp_symbol_node->parts.size > 1) { + ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX((int)(interp_symbol_node->parts.size))); + } + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, intern); + } + else { + ADD_INSN(ret, &dummy_line_node, pop); + } + + return; + } + case YP_NODE_KEYWORD_HASH_NODE: { + yp_keyword_hash_node_t *keyword_hash_node = (yp_keyword_hash_node_t *) node; + yp_node_list_t elements = keyword_hash_node->elements; + + for (size_t index = 0; index < elements.size; index++) { + yp_compile_node(iseq, elements.nodes[index], ret, src, popped, compile_context); + } + + ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(elements.size * 2)); + return; + } + case YP_NODE_MISSING_NODE: { + rb_bug("A yp_missing_node_t should not exist in YARP's AST."); + return; + } + case YP_NODE_NIL_NODE: + if (!popped) { + ADD_INSN(ret, &dummy_line_node, putnil); + } + return; + case YP_NODE_OR_NODE: { + yp_or_node_t *or_node = (yp_or_node_t *) node; + + LABEL *end_label = NEW_LABEL(lineno); + yp_compile_node(iseq, or_node->left, ret, src, popped, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + ADD_INSNL(ret, &dummy_line_node, branchif, end_label); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + yp_compile_node(iseq, or_node->right, ret, src, popped, compile_context); + ADD_LABEL(ret, end_label); + + return; + } + case YP_NODE_PARENTHESES_NODE: { + yp_parentheses_node_t *parentheses_node = (yp_parentheses_node_t *) node; + + if (parentheses_node->body == NULL) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, parentheses_node->body, ret, src, popped, compile_context); + } + + return; + } + case YP_NODE_PROGRAM_NODE: { + yp_program_node_t *program_node = (yp_program_node_t *) node; + + if (program_node->statements->body.size == 0) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, (yp_node_t *) program_node->statements, ret, src, popped, compile_context); + } + + ADD_INSN(ret, &dummy_line_node, leave); + return; + } + case YP_NODE_RETURN_NODE: { + yp_arguments_node_t *arguments = ((yp_return_node_t *)node)->arguments; + + if (arguments) { + yp_compile_node(iseq, (yp_node_t *)arguments, ret, src, popped, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_TRACE(ret, RUBY_EVENT_RETURN); + ADD_INSN(ret, &dummy_line_node, leave); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, putnil); + } + return; + } + case YP_NODE_SELF_NODE: + ADD_INSN(ret, &dummy_line_node, putself); + return; + case YP_NODE_STATEMENTS_NODE: { + yp_statements_node_t *statements_node = (yp_statements_node_t *) node; + yp_node_list_t node_list = statements_node->body; + for (size_t index = 0; index < node_list.size; index++) { + // We only want to have popped == false for the last instruction + if (!popped && (index != node_list.size - 1)) { + popped = true; + } + yp_compile_node(iseq, node_list.nodes[index], ret, src, popped, compile_context); + } + return; + } + case YP_NODE_STRING_CONCAT_NODE: { + yp_string_concat_node_t *str_concat_node = (yp_string_concat_node_t *)node; + yp_compile_node(iseq, str_concat_node->left, ret, src, popped, compile_context); + yp_compile_node(iseq, str_concat_node->right, ret, src, popped, compile_context); + return; + } + case YP_NODE_STRING_NODE: { + if (!popped) { + yp_string_node_t *string_node = (yp_string_node_t *) node; + ADD_INSN1(ret, &dummy_line_node, putstring, parse_string(&string_node->unescaped)); + } + return; + } + case YP_NODE_SYMBOL_NODE: { + yp_symbol_node_t *symbol_node = (yp_symbol_node_t *) node; + ADD_INSN1(ret, &dummy_line_node, putobject, ID2SYM(parse_string_symbol(&symbol_node->unescaped))); + return; + } + case YP_NODE_TRUE_NODE: + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + } + return; + case YP_NODE_UNDEF_NODE: { + yp_undef_node_t *undef_node = (yp_undef_node_t *) node; + + for (size_t index = 0; index < undef_node->names.size; index++) { + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_VMCORE); + ADD_INSN1(ret, &dummy_line_node, putspecialobject, VM_SPECIAL_OBJECT_CBASE); + + yp_compile_node(iseq, undef_node->names.nodes[index], ret, src, popped, compile_context); + + ADD_SEND(ret, &dummy_line_node, rb_intern("core#undef_method"), INT2NUM(2)); + + if (index < undef_node->names.size - 1) + ADD_INSN(ret, &dummy_line_node, pop); + } + + return; + } + case YP_NODE_X_STRING_NODE: { + yp_x_string_node_t *xstring_node = (yp_x_string_node_t *) node; + ADD_INSN(ret, &dummy_line_node, putself); + ADD_INSN1(ret, &dummy_line_node, putobject, parse_string(&xstring_node->unescaped)); + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, rb_intern("`"), INT2NUM(1), INT2FIX(VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE)); + return; + } + default: + rb_raise(rb_eNotImpError, "node type %s not implemented", yp_node_type_to_str(node->type)); + return; + } } static VALUE -rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret) +rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, yp_compile_context_t *compile_context) { RUBY_ASSERT(ISEQ_COMPILE_DATA(iseq)); - RUBY_ASSERT(node->type == YP_NODE_PROGRAM_NODE); + RUBY_ASSERT(node->type == YP_NODE_PROGRAM_NODE || node->type == YP_NODE_SCOPE_NODE); - yp_compile_node(iseq, node, ret, node->location.start, false); + yp_compile_node(iseq, node, ret, node->location.start, false, compile_context); iseq_set_sequence(iseq, ret); return Qnil; From 95efdef3b29bcf903bc3e11bed942bedf3ef95bb Mon Sep 17 00:00:00 2001 From: elfham <38372058+elfham@users.noreply.github.com> Date: Wed, 30 Aug 2023 02:17:03 +0900 Subject: [PATCH 104/208] [ruby/reline] Set EastAsianWidth::UNICODE_VERSION (https://github.com/ruby/reline/pull/586) * Set EastAsianWidth::UNICODE_VERSION * Commented out UNICODE_VERSION in Reline::Unicode::EastAsianWidth * Commented out UNICODE_VERSION in Reline::Unicode::EastAsianWidth https://github.com/ruby/reline/commit/6d94f2a26a --- lib/reline/unicode/east_asian_width.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/reline/unicode/east_asian_width.rb b/lib/reline/unicode/east_asian_width.rb index 97dfec4c52f8f8..ce7f7f26458077 100644 --- a/lib/reline/unicode/east_asian_width.rb +++ b/lib/reline/unicode/east_asian_width.rb @@ -1,6 +1,7 @@ class Reline::Unicode::EastAsianWidth # This is based on EastAsianWidth.txt # EastAsianWidth.txt + # UNICODE_VERSION = '15.0.0' # Fullwidth TYPE_F = /^[#{ %W( From c58561b5e3f597867a6e62f451a77ea27a6b9150 Mon Sep 17 00:00:00 2001 From: ima1zumi <52617472+ima1zumi@users.noreply.github.com> Date: Wed, 30 Aug 2023 02:50:06 +0900 Subject: [PATCH 105/208] [ruby/reline] Remove `ARGV.first` in east_asian_width.rb (https://github.com/ruby/reline/pull/587) `ARGV.first` is the name of the EastAsianWidth file and is not needed for east_asian_width.rb https://github.com/ruby/reline/commit/6649bda31c --- lib/reline/unicode/east_asian_width.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/reline/unicode/east_asian_width.rb b/lib/reline/unicode/east_asian_width.rb index ce7f7f26458077..c1c941f7993cc9 100644 --- a/lib/reline/unicode/east_asian_width.rb +++ b/lib/reline/unicode/east_asian_width.rb @@ -1,6 +1,5 @@ class Reline::Unicode::EastAsianWidth # This is based on EastAsianWidth.txt - # EastAsianWidth.txt # UNICODE_VERSION = '15.0.0' # Fullwidth From f37f357e808a7435f56946cd603a333feba7691b Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Tue, 29 Aug 2023 19:08:15 +0100 Subject: [PATCH 106/208] [ruby/irb] Improve help/show_cmds message during debugger integration (https://github.com/ruby/irb/pull/693) * `help` should display debugger's help during irb:rdbg session * Update `show_cmds`'s output when in irb:rdbg session https://github.com/ruby/irb/commit/4029c2e564 --- lib/irb/cmd/show_cmds.rb | 15 +++++++++++++++ lib/irb/statement.rb | 4 +++- test/irb/test_debug_cmd.rb | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/lib/irb/cmd/show_cmds.rb b/lib/irb/cmd/show_cmds.rb index cef6254ca039de..7d6b3ec2668b6f 100644 --- a/lib/irb/cmd/show_cmds.rb +++ b/lib/irb/cmd/show_cmds.rb @@ -15,6 +15,16 @@ class ShowCmds < Nop def execute(*args) commands_info = IRB::ExtendCommandBundle.all_commands_info commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } + + if irb_context.with_debugger + # Remove the original "Debugging" category + commands_grouped_by_categories.delete("Debugging") + # Remove the `help` command as it's delegated to the debugger + commands_grouped_by_categories["Context"].delete_if { |cmd| cmd[:display_name] == :help } + # Add an empty "Debugging (from debug.gem)" category at the end + commands_grouped_by_categories["Debugging (from debug.gem)"] = [] + end + longest_cmd_name_length = commands_info.map { |c| c[:display_name].length }.max output = StringIO.new @@ -29,6 +39,11 @@ def execute(*args) output.puts end + # Append the debugger help at the end + if irb_context.with_debugger + output.puts DEBUGGER__.help + end + Pager.page_content(output.string) end end diff --git a/lib/irb/statement.rb b/lib/irb/statement.rb index 9493c3ffb1277a..b12110600cc728 100644 --- a/lib/irb/statement.rb +++ b/lib/irb/statement.rb @@ -60,7 +60,9 @@ def suppresses_echo? end def should_be_handled_by_debugger? - IRB::ExtendCommand::DebugCommand > @command_class + require_relative 'cmd/help' + require_relative 'cmd/debug' + IRB::ExtendCommand::DebugCommand > @command_class || IRB::ExtendCommand::Help == @command_class end def evaluable_code diff --git a/test/irb/test_debug_cmd.rb b/test/irb/test_debug_cmd.rb index a99f7a943f62cd..d669c174e6ef6a 100644 --- a/test/irb/test_debug_cmd.rb +++ b/test/irb/test_debug_cmd.rb @@ -331,6 +331,39 @@ def bar assert_include(output, "InputMethod: RelineInputMethod") end + def test_help_command_is_delegated_to_the_debugger + write_ruby <<~'ruby' + binding.irb + ruby + + output = run_ruby_file do + type "debug" + type "help" + type "continue" + end + + assert_include(output, "### Frame control") + end + + def test_show_cmds_display_different_content_when_debugger_is_enabled + write_ruby <<~'ruby' + # disable pager + STDIN.singleton_class.define_method(:tty?) { false } + binding.irb + ruby + + output = run_ruby_file do + type "debug" + type "show_cmds" + type "continue" + end + + # IRB's commands should still be listed + assert_match(/show_cmds\s+List all available commands and their description\./, output) + # debug gem's commands should be appended at the end + assert_match(/Debugging \(from debug\.gem\)\s+### Control flow/, output) + end + def test_input_is_evaluated_in_the_context_of_the_current_thread write_ruby <<~'ruby' current_thread = Thread.current From 7a5df9d0ed7cbd0660803074b34623a1f1dc0768 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Mon, 28 Aug 2023 18:31:52 -0700 Subject: [PATCH 107/208] [rubygems/rubygems] Fix bundle update --redownload It now does the redownloading/installing just like bundle install --redownload https://github.com/rubygems/rubygems/commit/3b058e5eca --- lib/bundler/cli/update.rb | 1 + spec/bundler/update/redownload_spec.rb | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/lib/bundler/cli/update.rb b/lib/bundler/cli/update.rb index b49182655bf9d7..22dd1a78dd1448 100644 --- a/lib/bundler/cli/update.rb +++ b/lib/bundler/cli/update.rb @@ -63,6 +63,7 @@ def run opts = options.dup opts["update"] = true opts["local"] = options[:local] + opts["force"] = options[:redownload] Bundler.settings.set_command_option_if_given :jobs, opts["jobs"] diff --git a/spec/bundler/update/redownload_spec.rb b/spec/bundler/update/redownload_spec.rb index 147be823f57406..b55593d077642f 100644 --- a/spec/bundler/update/redownload_spec.rb +++ b/spec/bundler/update/redownload_spec.rb @@ -30,5 +30,15 @@ bundle "update rack --no-color --redownload" expect(err).not_to include "[DEPRECATED] The `--force` option has been renamed to `--redownload`" end + + it "re-installs installed gems" do + rack_lib = default_bundle_path("gems/rack-1.0.0/lib/rack.rb") + rack_lib.open("w") {|f| f.write("blah blah blah") } + bundle :update, :redownload => true + + expect(out).to include "Installing rack 1.0.0" + expect(rack_lib.open(&:read)).to eq("RACK = '1.0.0'\n") + expect(the_bundle).to include_gems "rack 1.0.0" + end end end From 5161c6c4cdf989ee63dbbe0baa81317f8e8ae491 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 28 Aug 2023 09:06:19 -0400 Subject: [PATCH 108/208] [ruby/yarp] Statements inside ensure blocks can accept blocks https://github.com/ruby/yarp/commit/be84ea5343 --- test/yarp/fixtures/begin_ensure.txt | 7 ++ test/yarp/snapshots/begin_ensure.txt | 97 +++++++++++++++++++++++++++- yarp/yarp.c | 14 ++++ 3 files changed, 116 insertions(+), 2 deletions(-) diff --git a/test/yarp/fixtures/begin_ensure.txt b/test/yarp/fixtures/begin_ensure.txt index 1c6f9dac359290..6cfb6274539100 100644 --- a/test/yarp/fixtures/begin_ensure.txt +++ b/test/yarp/fixtures/begin_ensure.txt @@ -12,3 +12,10 @@ begin a begin a; ensure b; end +begin begin:s.l begin ensure Module.new do + begin + break + ensure Module.new do + end + end +end end end end diff --git a/test/yarp/snapshots/begin_ensure.txt b/test/yarp/snapshots/begin_ensure.txt index e04a1236df3ee0..8e9873122bb589 100644 --- a/test/yarp/snapshots/begin_ensure.txt +++ b/test/yarp/snapshots/begin_ensure.txt @@ -1,6 +1,6 @@ -ProgramNode(0...94)( +ProgramNode(0...211)( [], - StatementsNode(0...94)( + StatementsNode(0...211)( [BeginNode(0...20)( (0...5), StatementsNode(6...7)( @@ -64,6 +64,99 @@ ProgramNode(0...94)( (91...94) ), (91...94) + ), + BeginNode(96...211)( + (96...101), + StatementsNode(102...207)( + [BeginNode(102...207)( + (102...107), + StatementsNode(107...203)( + [CallNode(107...203)( + SymbolNode(107...109)((107...108), (108...109), nil, "s"), + (109...110), + (110...111), + nil, + ArgumentsNode(112...203)( + [BeginNode(112...203)( + (112...117), + nil, + nil, + nil, + EnsureNode(118...203)( + (118...124), + StatementsNode(125...199)( + [CallNode(125...199)( + ConstantReadNode(125...131)(), + (131...132), + (132...135), + nil, + nil, + nil, + BlockNode(136...199)( + [], + nil, + StatementsNode(141...195)( + [BeginNode(141...195)( + (141...146), + StatementsNode(151...156)( + [BreakNode(151...156)(nil, (151...156))] + ), + nil, + nil, + EnsureNode(161...195)( + (161...167), + StatementsNode(168...189)( + [CallNode(168...189)( + ConstantReadNode(168...174)(), + (174...175), + (175...178), + nil, + nil, + nil, + BlockNode(179...189)( + [], + nil, + nil, + (179...181), + (186...189) + ), + 0, + "new" + )] + ), + (192...195) + ), + (192...195) + )] + ), + (136...138), + (196...199) + ), + 0, + "new" + )] + ), + (200...203) + ), + (200...203) + )] + ), + nil, + nil, + 0, + "l" + )] + ), + nil, + nil, + nil, + (204...207) + )] + ), + nil, + nil, + nil, + (208...211) )] ) ) diff --git a/yarp/yarp.c b/yarp/yarp.c index 8cd2c2007b5029..78c0a03cfe1688 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -9049,10 +9049,12 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { } if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_ELSE, YP_TOKEN_KEYWORD_ENSURE, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); yp_statements_node_t *statements = parse_statements(parser, YP_CONTEXT_RESCUE); if (statements) { yp_rescue_node_statements_set(rescue, statements); } + yp_accepts_block_stack_pop(parser); accept_any(parser, 2, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON); } @@ -9083,7 +9085,9 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_statements_node_t *else_statements = NULL; if (!match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_END, YP_TOKEN_KEYWORD_ENSURE)) { + yp_accepts_block_stack_push(parser, true); else_statements = parse_statements(parser, YP_CONTEXT_RESCUE_ELSE); + yp_accepts_block_stack_pop(parser); accept_any(parser, 2, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON); } @@ -9097,7 +9101,9 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { yp_statements_node_t *ensure_statements = NULL; if (!match_type_p(parser, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); ensure_statements = parse_statements(parser, YP_CONTEXT_ENSURE); + yp_accepts_block_stack_pop(parser); accept_any(parser, 2, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON); } @@ -9207,7 +9213,9 @@ parse_block(yp_parser_t *parser) { } else { if (!match_type_p(parser, YP_TOKEN_KEYWORD_END)) { if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ELSE, YP_TOKEN_KEYWORD_ENSURE)) { + yp_accepts_block_stack_push(parser, true); statements = (yp_node_t *) parse_statements(parser, YP_CONTEXT_BLOCK_KEYWORDS); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -11368,7 +11376,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_node_t *statements = NULL; if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); statements = (yp_node_t *) parse_statements(parser, YP_CONTEXT_SCLASS); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -11649,7 +11659,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_do_loop_stack_push(parser, false); if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE, YP_TOKEN_KEYWORD_END)) { + yp_accepts_block_stack_push(parser, true); statements = (yp_node_t *) parse_statements(parser, YP_CONTEXT_DEF); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { @@ -12506,7 +12518,9 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { opening = parser->previous; if (!match_any_type_p(parser, 3, YP_TOKEN_KEYWORD_END, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { + yp_accepts_block_stack_push(parser, true); body = (yp_node_t *) parse_statements(parser, YP_CONTEXT_LAMBDA_DO_END); + yp_accepts_block_stack_pop(parser); } if (match_any_type_p(parser, 2, YP_TOKEN_KEYWORD_RESCUE, YP_TOKEN_KEYWORD_ENSURE)) { From 455153705c81fc1561317279da54daa5e5b479b3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Aug 2023 14:34:06 -0400 Subject: [PATCH 109/208] Tests for the YARP compiler --- test/yarp/compiler_test.rb | 166 +++++++++++++++++++++++++++++++++++++ tool/sync_default_gems.rb | 6 +- 2 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 test/yarp/compiler_test.rb diff --git a/test/yarp/compiler_test.rb b/test/yarp/compiler_test.rb new file mode 100644 index 00000000000000..88d0bddec7f8df --- /dev/null +++ b/test/yarp/compiler_test.rb @@ -0,0 +1,166 @@ +# frozen_string_literal: true + +module YARP + class CompilerTest < Test::Unit::TestCase + ############################################################################ + # Literals # + ############################################################################ + + def test_FalseNode + assert_equal false, compile("false") + end + + def test_FloatNode + assert_equal 1.0, compile("1.0") + assert_equal 1.0e0, compile("1.0e0") + assert_equal +1.0e+0, compile("+1.0e+0") + assert_equal -1.0e-0, compile("-1.0e-0") + end + + def test_ImaginaryNode + # assert_equal 1i, compile("1i") + # assert_equal +1.0i, compile("+1.0i") + # assert_equal 1ri, compile("1ri") + end + + def test_IntegerNode + assert_equal 1, compile("1") + assert_equal +1, compile("+1") + assert_equal -1, compile("-1") + # assert_equal 0x10, compile("0x10") + # assert_equal 0b10, compile("0b10") + # assert_equal 0o10, compile("0o10") + # assert_equal 010, compile("010") + end + + def test_NilNode + assert_nil compile("nil") + end + + def test_SelfNode + assert_equal TOPLEVEL_BINDING.eval("self"), compile("self") + end + + def test_TrueNode + assert_equal true, compile("true") + end + + ############################################################################ + # Reads # + ############################################################################ + + def test_ClassVariableReadNode + # assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; end; @@yct") + end + + def test_ConstantPathNode + assert_equal YARP::CompilerTest, compile("YARP::CompilerTest") + end + + def test_ConstantReadNode + assert_equal YARP, compile("YARP") + end + + def test_GlobalVariableReadNode + # assert_equal 1, compile("$yct = 1; $yct") + end + + def test_InstanceVariableReadNode + # assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; @yct; end") + end + + ############################################################################ + # Writes # + ############################################################################ + + def test_ClassVariableWriteNode + # assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; end") + end + + def test_ConstantWriteNode + assert_equal 1, compile("YCT = 1") + end + + def test_ConstantPathWriteNode + assert_equal 1, compile("YARP::YCT = 1") + end + + def test_GlobalVariableWriteNode + assert_equal 1, compile("$yct = 1") + end + + def test_InstanceVariableWriteNode + # assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; end") + end + + ############################################################################ + # String-likes # + ############################################################################ + + def test_EmbeddedVariableNode + # assert_equal "1", compile('class YARP::CompilerTest; @yct = 1; "#@yct"; end') + # assert_equal "1", compile('class YARP::CompilerTest; @@yct = 1; "#@@yct"; end') + # assert_equal "1", compile('$yct = 1; "#$yct"') + end + + def test_InterpolatedStringNode + # assert_equal "1 1 1", compile('$yct = 1; "1 #$yct 1"') + # assert_equal "1 3 1", compile('"1 #{1 + 2} 1"') + end + + def test_InterpolatedSymbolNode + # assert_equal :"1 1 1", compile('$yct = 1; :"1 #$yct 1"') + # assert_equal :"1 3 1", compile(':"1 #{1 + 2} 1"') + end + + def test_StringConcatNode + # assert_equal "YARP::CompilerTest", compile('"YARP" "::" "CompilerTest"') + end + + def test_StringNode + assert_equal "yct", compile('"yct"') + end + + def test_SymbolNode + assert_equal :yct, compile(":yct") + end + + def test_XStringNode + # assert_equal "yctyct", compile(<<~RUBY) + # class YARP::CompilerTest + # def self.`(command) = command * 2 + # `yct` + # end + # RUBY + end + + ############################################################################ + # Jumps # + ############################################################################ + + def test_AndNode + assert_equal 1, compile("true && 1") + assert_equal false, compile("false && 1") + end + + def test_OrNode + assert_equal true, compile("true || 1") + assert_equal 1, compile("false || 1") + end + + ############################################################################ + # Scopes/statements # + ############################################################################ + + def test_ParenthesesNode + assert_equal (), compile("()") + assert_equal (1), compile("(1)") + end + + private + + def compile(source) + RubyVM::InstructionSequence.compile_yarp(source).eval + end + end +end diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 5bdede34a94c0b..2ac66f919500ca 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -399,8 +399,9 @@ def sync_default_gems(gem) when "yarp" # We don't want to remove yarp_init.c, so we temporarily move it # out of the yarp dir, wipe the yarp dir, and then put it back - mv("yarp/yarp_init.c", ".") if File.exist? "yarp/yarp_init.c" - mv("yarp/yarp_compiler.c", ".") if File.exist? "yarp/yarp_compiler.c" + mv("yarp/yarp_init.c", ".") + mv("yarp/yarp_compiler.c", ".") + mv("test/yarp/compiler_test.rb", ".") rm_rf(%w[test/yarp yarp]) # Run the YARP templating scripts @@ -419,6 +420,7 @@ def sync_default_gems(gem) rm("yarp/extconf.rb") mv("yarp_init.c", "yarp/") mv("yarp_compiler.c", "yarp/") + mv("compiler_test.rb", "test/yarp/") else sync_lib gem, upstream end From b435161404ed960e02fadfd1c3d983d1fbfb91a0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Aug 2023 09:17:34 -0400 Subject: [PATCH 110/208] [ruby/yarp] Add instance variable names to the constant pool https://github.com/ruby/yarp/commit/f049932c44 --- lib/yarp/desugar_visitor.rb | 10 ++--- test/yarp/snapshots/methods.txt | 2 +- test/yarp/snapshots/patterns.txt | 2 +- test/yarp/snapshots/seattlerb/case_in.txt | 2 +- ...erpolation_and_carriage_return_escapes.txt | 2 +- ...on_and_carriage_return_escapes_windows.txt | 2 +- .../snapshots/seattlerb/lasgn_ivar_env.txt | 7 +++- ...ine_call_ivar_arg_no_parens_line_break.txt | 2 +- .../parse_line_call_ivar_line_break_paren.txt | 2 +- test/yarp/snapshots/strings.txt | 2 +- .../unparser/corpus/literal/assignment.txt | 7 +++- .../unparser/corpus/literal/defined.txt | 2 +- .../unparser/corpus/literal/dstr.txt | 2 +- .../unparser/corpus/literal/literal.txt | 22 ++++++---- .../unparser/corpus/literal/variables.txt | 2 +- .../unparser/corpus/semantic/dstr.txt | 6 ++- .../unparser/corpus/semantic/literal.txt | 2 +- test/yarp/snapshots/variables.txt | 8 ++-- .../whitequark/array_words_interp.txt | 2 +- test/yarp/snapshots/whitequark/defined.txt | 2 +- test/yarp/snapshots/whitequark/ivar.txt | 2 +- test/yarp/snapshots/whitequark/ivasgn.txt | 7 +++- .../yarp/snapshots/whitequark/masgn_splat.txt | 2 +- .../snapshots/whitequark/parser_bug_272.txt | 2 +- .../yarp/snapshots/whitequark/resbody_var.txt | 2 +- .../snapshots/whitequark/string_concat.txt | 2 +- .../yarp/snapshots/whitequark/string_dvar.txt | 2 +- .../yarp/snapshots/whitequark/var_op_asgn.txt | 1 + yarp/config.yml | 14 +++++++ yarp/yarp.c | 40 +++++++++++-------- 30 files changed, 103 insertions(+), 59 deletions(-) diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb index 3f901630b0c0cc..83a9bb5336a300 100644 --- a/lib/yarp/desugar_visitor.rb +++ b/lib/yarp/desugar_visitor.rb @@ -173,8 +173,8 @@ def visit_global_variable_operator_write_node(node) # @foo && @foo = bar def visit_instance_variable_and_write_node(node) AndNode.new( - InstanceVariableReadNode.new(node.name_loc), - InstanceVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + InstanceVariableReadNode.new(node.name, node.name_loc), + InstanceVariableWriteNode.new(node.name, node.name_loc, node.value, node.operator_loc, node.location), node.operator_loc, node.location ) @@ -187,8 +187,8 @@ def visit_instance_variable_and_write_node(node) # @foo || @foo = bar def visit_instance_variable_or_write_node(node) OrNode.new( - InstanceVariableReadNode.new(node.name_loc), - InstanceVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), + InstanceVariableReadNode.new(node.name, node.name_loc), + InstanceVariableWriteNode.new(node.name, node.name_loc, node.value, node.operator_loc, node.location), node.operator_loc, node.location ) @@ -200,7 +200,7 @@ def visit_instance_variable_or_write_node(node) # # @foo = @foo + bar def visit_instance_variable_operator_write_node(node) - desugar_operator_write_node(node, InstanceVariableWriteNode, InstanceVariableReadNode) + desugar_operator_write_node(node, InstanceVariableWriteNode, InstanceVariableReadNode, arguments: [node.name]) end # foo &&= bar diff --git a/test/yarp/snapshots/methods.txt b/test/yarp/snapshots/methods.txt index a2cfa8e06d0626..32a7a4e7cd903b 100644 --- a/test/yarp/snapshots/methods.txt +++ b/test/yarp/snapshots/methods.txt @@ -199,7 +199,7 @@ ProgramNode(0...1194)( ), DefNode(190...204)( (199...200), - InstanceVariableReadNode(194...198)(), + InstanceVariableReadNode(194...198)(:var), nil, nil, [], diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index de5946a7cf0da2..f888d1498fcd09 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -923,7 +923,7 @@ ProgramNode(0...3743)( "foo" ), PinnedVariableNode(961...966)( - InstanceVariableReadNode(962...966)(), + InstanceVariableReadNode(962...966)(:bar), (961...962) ), (958...960) diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index 8a5fafdf74b6a8..5ea3d11b78e10f 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -494,7 +494,7 @@ ProgramNode(0...747)( ArrayPatternNode(627...643)( nil, [PinnedVariableNode(628...631)( - InstanceVariableReadNode(629...631)(), + InstanceVariableReadNode(629...631)(:a), (628...629) ), PinnedVariableNode(633...636)( diff --git a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt index cfb1f9644370c7..5aab2888ef81ad 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt @@ -6,7 +6,7 @@ ProgramNode(0...5)( [StringNode(6...11)(nil, (6...11), nil, "foo\r"), EmbeddedVariableNode(11...16)( (11...12), - InstanceVariableReadNode(12...16)() + InstanceVariableReadNode(12...16)(:bar) ), StringNode(16...17)(nil, (16...17), nil, "\n")], (17...21) diff --git a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt index ee013f51378747..04727011a2a369 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt @@ -6,7 +6,7 @@ ProgramNode(0...5)( [StringNode(7...12)(nil, (7...12), nil, "foo\r"), EmbeddedVariableNode(12...17)( (12...13), - InstanceVariableReadNode(13...17)() + InstanceVariableReadNode(13...17)(:bar) ), StringNode(17...19)(nil, (17...19), nil, "\r\n")], (19...24) diff --git a/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt b/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt index b71eeaf2a7916d..f065032f3a2b2c 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt @@ -1,6 +1,11 @@ ProgramNode(0...7)( [], StatementsNode(0...7)( - [InstanceVariableWriteNode(0...7)((0...2), IntegerNode(5...7)(), (3...4))] + [InstanceVariableWriteNode(0...7)( + :a, + (0...2), + IntegerNode(5...7)(), + (3...4) + )] ) ) diff --git a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt index b7cca1732b5225..efbc413fe1485c 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt @@ -6,7 +6,7 @@ ProgramNode(0...4)( nil, (0...1), nil, - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)()]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:b)]), nil, nil, 0, diff --git a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt index ae67416e67a765..6466d9dd45b8d4 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt @@ -6,7 +6,7 @@ ProgramNode(0...6)( nil, (0...1), (1...2), - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)()]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:b)]), (5...6), nil, 0, diff --git a/test/yarp/snapshots/strings.txt b/test/yarp/snapshots/strings.txt index 38033453af3da3..d6f624624c0d40 100644 --- a/test/yarp/snapshots/strings.txt +++ b/test/yarp/snapshots/strings.txt @@ -185,7 +185,7 @@ ProgramNode(0...498)( (414...415), [EmbeddedVariableNode(415...420)( (415...416), - InstanceVariableReadNode(416...420)() + InstanceVariableReadNode(416...420)(:foo) )], (420...421) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index bcf853886f0d8c..6166e0cd26814e 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -66,8 +66,8 @@ ProgramNode(0...704)( (74...75) ), MultiWriteNode(85...102)( - [InstanceVariableTargetNode(86...88)(), - InstanceVariableTargetNode(90...92)()], + [InstanceVariableTargetNode(86...88)(:a), + InstanceVariableTargetNode(90...92)(:b)], (94...95), ArrayNode(96...102)( [IntegerNode(97...98)(), IntegerNode(100...101)()], @@ -296,6 +296,7 @@ ProgramNode(0...704)( (306...307) ), InstanceVariableWriteNode(310...316)( + :a, (310...312), IntegerNode(315...316)(), (313...314) @@ -605,6 +606,7 @@ ProgramNode(0...704)( (543...546) ), InstanceVariableOrWriteNode(551...561)( + :a, (551...553), (554...557), StringNode(558...561)((558...560), (560...560), (560...561), "") @@ -698,6 +700,7 @@ ProgramNode(0...704)( (665...668) ), InstanceVariableOrWriteNode(687...704)( + :a, (687...689), (690...693), InterpolatedStringNode(694...704)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/defined.txt b/test/yarp/snapshots/unparser/corpus/literal/defined.txt index c2d6a71bb1b132..1d39c45c836728 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/defined.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/defined.txt @@ -3,7 +3,7 @@ ProgramNode(0...56)( StatementsNode(0...56)( [DefinedNode(0...14)( (8...9), - InstanceVariableReadNode(9...13)(), + InstanceVariableReadNode(9...13)(:foo), (13...14), (0...8) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt index 846614936967e2..49d08824bfce0f 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt @@ -75,7 +75,7 @@ ProgramNode(0...299)( [StringNode(160...161)(nil, (160...161), nil, "a"), EmbeddedVariableNode(161...164)( (161...162), - InstanceVariableReadNode(162...164)() + InstanceVariableReadNode(162...164)(:a) )], (164...165) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/literal.txt b/test/yarp/snapshots/unparser/corpus/literal/literal.txt index bc9f86842dce37..a38a7f4f888cd1 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/literal.txt @@ -153,7 +153,7 @@ ProgramNode(0...916)( (206...207), [EmbeddedVariableNode(207...210)( (207...208), - InstanceVariableReadNode(208...210)() + InstanceVariableReadNode(208...210)(:a) ), StringNode(210...211)(nil, (210...211), nil, " "), EmbeddedVariableNode(211...215)( @@ -267,7 +267,9 @@ ProgramNode(0...916)( [StringNode(419...422)(nil, (419...422), nil, "foo"), EmbeddedStatementsNode(422...429)( (422...424), - StatementsNode(424...428)([InstanceVariableReadNode(424...428)()]), + StatementsNode(424...428)( + [InstanceVariableReadNode(424...428)(:bar)] + ), (428...429) )], (429...430) @@ -300,7 +302,9 @@ ProgramNode(0...916)( [StringNode(516...519)(nil, (516...519), nil, "foo"), EmbeddedStatementsNode(519...526)( (519...521), - StatementsNode(521...525)([InstanceVariableReadNode(521...525)()]), + StatementsNode(521...525)( + [InstanceVariableReadNode(521...525)(:bar)] + ), (525...526) )], (526...527), @@ -311,7 +315,9 @@ ProgramNode(0...916)( [StringNode(529...532)(nil, (529...532), nil, "foo"), EmbeddedStatementsNode(532...539)( (532...534), - StatementsNode(534...538)([InstanceVariableReadNode(534...538)()]), + StatementsNode(534...538)( + [InstanceVariableReadNode(534...538)(:bar)] + ), (538...539) )], (539...543), @@ -501,7 +507,7 @@ ProgramNode(0...916)( [IntegerNode(693...694)(), SplatNode(696...701)( (696...697), - InstanceVariableReadNode(697...701)() + InstanceVariableReadNode(697...701)(:foo) )], (692...693), (701...702) @@ -509,7 +515,7 @@ ProgramNode(0...916)( ArrayNode(703...713)( [SplatNode(704...709)( (704...705), - InstanceVariableReadNode(705...709)() + InstanceVariableReadNode(705...709)(:foo) ), IntegerNode(711...712)()], (703...704), @@ -518,11 +524,11 @@ ProgramNode(0...916)( ArrayNode(714...728)( [SplatNode(715...720)( (715...716), - InstanceVariableReadNode(716...720)() + InstanceVariableReadNode(716...720)(:foo) ), SplatNode(722...727)( (722...723), - InstanceVariableReadNode(723...727)() + InstanceVariableReadNode(723...727)(:baz) )], (714...715), (727...728) diff --git a/test/yarp/snapshots/unparser/corpus/literal/variables.txt b/test/yarp/snapshots/unparser/corpus/literal/variables.txt index b7ab14e36c35f7..29857ec53c5591 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/variables.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/variables.txt @@ -2,7 +2,7 @@ ProgramNode(0...66)( [], StatementsNode(0...66)( [CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), - InstanceVariableReadNode(2...4)(), + InstanceVariableReadNode(2...4)(:a), ClassVariableReadNode(5...8)(), GlobalVariableReadNode(9...11)(), NumberedReferenceReadNode(12...14)(), diff --git a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt index 14843a83a65114..490b13e0bc874a 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt @@ -207,7 +207,9 @@ ProgramNode(0...608)( [StringNode(563...564)(nil, (563...564), nil, "a"), EmbeddedStatementsNode(564...569)( (564...566), - StatementsNode(566...568)([InstanceVariableReadNode(566...568)()]), + StatementsNode(566...568)( + [InstanceVariableReadNode(566...568)(:a)] + ), (568...569) )], (569...570) @@ -220,7 +222,7 @@ ProgramNode(0...608)( [StringNode(576...577)(nil, (576...577), nil, "a"), EmbeddedVariableNode(577...580)( (577...578), - InstanceVariableReadNode(578...580)() + InstanceVariableReadNode(578...580)(:a) )], (580...581) ), diff --git a/test/yarp/snapshots/unparser/corpus/semantic/literal.txt b/test/yarp/snapshots/unparser/corpus/semantic/literal.txt index d8578058df0b35..b9df7fcbd9a182 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/literal.txt @@ -15,7 +15,7 @@ ProgramNode(0...131)( (71...74), [EmbeddedStatementsNode(74...81)( (74...76), - StatementsNode(76...80)([InstanceVariableReadNode(76...80)()]), + StatementsNode(76...80)([InstanceVariableReadNode(76...80)(:bar)]), (80...81) ), StringNode(81...84)(nil, (81...84), nil, "baz")], diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index 35b90dbae3b484..a6b6a444f9ecd3 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -30,8 +30,9 @@ ProgramNode(0...293)( IntegerNode(57...58)() ), GlobalVariableReadNode(60...64)(), - InstanceVariableReadNode(66...70)(), + InstanceVariableReadNode(66...70)(:abc), InstanceVariableWriteNode(72...80)( + :abc, (72...76), IntegerNode(79...80)(), (77...78) @@ -62,14 +63,15 @@ ProgramNode(0...293)( ) ), MultiWriteNode(123...137)( - [InstanceVariableTargetNode(123...127)(), - InstanceVariableTargetNode(129...133)()], + [InstanceVariableTargetNode(123...127)(:foo), + InstanceVariableTargetNode(129...133)(:bar)], (134...135), IntegerNode(136...137)(), nil, nil ), InstanceVariableWriteNode(139...150)( + :foo, (139...143), ArrayNode(146...150)( [IntegerNode(146...147)(), IntegerNode(149...150)()], diff --git a/test/yarp/snapshots/whitequark/array_words_interp.txt b/test/yarp/snapshots/whitequark/array_words_interp.txt index 3cc21323f7f910..963a83dd480833 100644 --- a/test/yarp/snapshots/whitequark/array_words_interp.txt +++ b/test/yarp/snapshots/whitequark/array_words_interp.txt @@ -51,7 +51,7 @@ ProgramNode(0...38)( StringNode(29...32)(nil, (29...32), nil, "foo"), EmbeddedVariableNode(32...37)( (32...33), - InstanceVariableReadNode(33...37)() + InstanceVariableReadNode(33...37)(:baz) )], nil )], diff --git a/test/yarp/snapshots/whitequark/defined.txt b/test/yarp/snapshots/whitequark/defined.txt index 12a30055939bca..9d8d997e2bab16 100644 --- a/test/yarp/snapshots/whitequark/defined.txt +++ b/test/yarp/snapshots/whitequark/defined.txt @@ -3,7 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [DefinedNode(0...13)( nil, - InstanceVariableReadNode(9...13)(), + InstanceVariableReadNode(9...13)(:foo), nil, (0...8) ), diff --git a/test/yarp/snapshots/whitequark/ivar.txt b/test/yarp/snapshots/whitequark/ivar.txt index 88bb2ace357f5f..cf233251f2835d 100644 --- a/test/yarp/snapshots/whitequark/ivar.txt +++ b/test/yarp/snapshots/whitequark/ivar.txt @@ -1,4 +1,4 @@ ProgramNode(0...4)( [], - StatementsNode(0...4)([InstanceVariableReadNode(0...4)()]) + StatementsNode(0...4)([InstanceVariableReadNode(0...4)(:foo)]) ) diff --git a/test/yarp/snapshots/whitequark/ivasgn.txt b/test/yarp/snapshots/whitequark/ivasgn.txt index 4f6f90bbd71dde..587dcfaa9df007 100644 --- a/test/yarp/snapshots/whitequark/ivasgn.txt +++ b/test/yarp/snapshots/whitequark/ivasgn.txt @@ -1,6 +1,11 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [InstanceVariableWriteNode(0...9)((0...4), IntegerNode(7...9)(), (5...6))] + [InstanceVariableWriteNode(0...9)( + :var, + (0...4), + IntegerNode(7...9)(), + (5...6) + )] ) ) diff --git a/test/yarp/snapshots/whitequark/masgn_splat.txt b/test/yarp/snapshots/whitequark/masgn_splat.txt index e10e6d22d16dec..c6265c652e02f8 100644 --- a/test/yarp/snapshots/whitequark/masgn_splat.txt +++ b/test/yarp/snapshots/whitequark/masgn_splat.txt @@ -51,7 +51,7 @@ ProgramNode(0...139)( nil ), MultiWriteNode(47...65)( - [InstanceVariableTargetNode(47...51)(), + [InstanceVariableTargetNode(47...51)(:foo), ClassVariableTargetNode(53...58)()], (59...60), ArrayNode(61...65)( diff --git a/test/yarp/snapshots/whitequark/parser_bug_272.txt b/test/yarp/snapshots/whitequark/parser_bug_272.txt index eeb981d694fbb3..50d753980713e9 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_272.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_272.txt @@ -6,7 +6,7 @@ ProgramNode(0...15)( nil, (0...1), nil, - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)()]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:b)]), nil, BlockNode(5...15)( [:c], diff --git a/test/yarp/snapshots/whitequark/resbody_var.txt b/test/yarp/snapshots/whitequark/resbody_var.txt index 861a91f79f04c1..4530e6cdbd442a 100644 --- a/test/yarp/snapshots/whitequark/resbody_var.txt +++ b/test/yarp/snapshots/whitequark/resbody_var.txt @@ -10,7 +10,7 @@ ProgramNode(0...73)( (13...19), [], (20...22), - InstanceVariableTargetNode(23...26)(), + InstanceVariableTargetNode(23...26)(:ex), StatementsNode(28...31)( [CallNode(28...31)( nil, diff --git a/test/yarp/snapshots/whitequark/string_concat.txt b/test/yarp/snapshots/whitequark/string_concat.txt index c20f4ac35e7b2b..fde498b39fecef 100644 --- a/test/yarp/snapshots/whitequark/string_concat.txt +++ b/test/yarp/snapshots/whitequark/string_concat.txt @@ -7,7 +7,7 @@ ProgramNode(0...14)( [StringNode(1...4)(nil, (1...4), nil, "foo"), EmbeddedVariableNode(4...7)( (4...5), - InstanceVariableReadNode(5...7)() + InstanceVariableReadNode(5...7)(:a) )], (7...8) ), diff --git a/test/yarp/snapshots/whitequark/string_dvar.txt b/test/yarp/snapshots/whitequark/string_dvar.txt index c89e3595653eb7..2378e8fdcbc3c3 100644 --- a/test/yarp/snapshots/whitequark/string_dvar.txt +++ b/test/yarp/snapshots/whitequark/string_dvar.txt @@ -5,7 +5,7 @@ ProgramNode(0...14)( (0...1), [EmbeddedVariableNode(1...4)( (1...2), - InstanceVariableReadNode(2...4)() + InstanceVariableReadNode(2...4)(:a) ), StringNode(4...5)(nil, (4...5), nil, " "), EmbeddedVariableNode(5...9)((5...6), ClassVariableReadNode(6...9)()), diff --git a/test/yarp/snapshots/whitequark/var_op_asgn.txt b/test/yarp/snapshots/whitequark/var_op_asgn.txt index 4591e68c96fbc8..3aff7a6a60267a 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn.txt @@ -8,6 +8,7 @@ ProgramNode(0...53)( :| ), InstanceVariableOperatorWriteNode(13...20)( + :a, (13...15), (16...18), IntegerNode(19...20)(), diff --git a/yarp/config.yml b/yarp/config.yml index d35436088abdda..b4e8b479137581 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -1298,6 +1298,8 @@ nodes: ^^^^^^^^^^^ - name: InstanceVariableAndWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: operator_loc @@ -1311,6 +1313,8 @@ nodes: ^^^^^^^^^^^^^^^^^ - name: InstanceVariableOperatorWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: operator_loc @@ -1326,6 +1330,8 @@ nodes: ^^^^^^^^^^^^^^^^ - name: InstanceVariableOrWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: operator_loc @@ -1338,12 +1344,18 @@ nodes: @target ||= value ^^^^^^^^^^^^^^^^^ - name: InstanceVariableReadNode + child_nodes: + - name: name + type: constant comment: | Represents referencing an instance variable. @foo ^^^^ - name: InstanceVariableTargetNode + child_nodes: + - name: name + type: constant comment: | Represents writing to an instance variable in a context that doesn't have an explicit value. @@ -1351,6 +1363,8 @@ nodes: ^^^^ ^^^^ - name: InstanceVariableWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: value diff --git a/yarp/yarp.c b/yarp/yarp.c index 78c0a03cfe1688..089de42b9a83f3 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2621,8 +2621,7 @@ yp_in_node_create(yp_parser_t *parser, yp_node_t *pattern, yp_statements_node_t // Allocate and initialize a new InstanceVariableAndWriteNode node. static yp_instance_variable_and_write_node_t * -yp_instance_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_INSTANCE_VARIABLE_READ_NODE)); +yp_instance_variable_and_write_node_create(yp_parser_t *parser, yp_instance_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); yp_instance_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_and_write_node_t); @@ -2630,11 +2629,12 @@ yp_instance_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *targe { .type = YP_NODE_INSTANCE_VARIABLE_AND_WRITE_NODE, .location = { - .start = target->location.start, + .start = target->base.location.start, .end = value->location.end } }, - .name_loc = target->location, + .name = target->name, + .name_loc = target->base.location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -2644,18 +2644,19 @@ yp_instance_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *targe // Allocate and initialize a new InstanceVariableOperatorWriteNode node. static yp_instance_variable_operator_write_node_t * -yp_instance_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { +yp_instance_variable_operator_write_node_create(yp_parser_t *parser, yp_instance_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { yp_instance_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_operator_write_node_t); *node = (yp_instance_variable_operator_write_node_t) { { .type = YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE, .location = { - .start = target->location.start, + .start = target->base.location.start, .end = value->location.end } }, - .name_loc = target->location, + .name = target->name, + .name_loc = target->base.location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) @@ -2666,8 +2667,7 @@ yp_instance_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t * // Allocate and initialize a new InstanceVariableOrWriteNode node. static yp_instance_variable_or_write_node_t * -yp_instance_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_INSTANCE_VARIABLE_READ_NODE)); +yp_instance_variable_or_write_node_create(yp_parser_t *parser, yp_instance_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); yp_instance_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_or_write_node_t); @@ -2675,11 +2675,12 @@ yp_instance_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target { .type = YP_NODE_INSTANCE_VARIABLE_OR_WRITE_NODE, .location = { - .start = target->location.start, + .start = target->base.location.start, .end = value->location.end } }, - .name_loc = target->location, + .name = target->name, + .name_loc = target->base.location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -2693,9 +2694,13 @@ yp_instance_variable_read_node_create(yp_parser_t *parser, const yp_token_t *tok assert(token->type == YP_TOKEN_INSTANCE_VARIABLE); yp_instance_variable_read_node_t *node = YP_ALLOC_NODE(parser, yp_instance_variable_read_node_t); - *node = (yp_instance_variable_read_node_t) {{ - .type = YP_NODE_INSTANCE_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) - }}; + *node = (yp_instance_variable_read_node_t) { + { + .type = YP_NODE_INSTANCE_VARIABLE_READ_NODE, + .location = YP_LOCATION_TOKEN_VALUE(token) + }, + .name = yp_parser_constant_id_location(parser, token->start + 1, token->end) + }; return node; } @@ -2712,6 +2717,7 @@ yp_instance_variable_write_node_create(yp_parser_t *parser, yp_instance_variable .end = value->location.end } }, + .name = read_node->name, .name_loc = YP_LOCATION_NODE_BASE_VALUE(read_node), .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value @@ -12821,7 +12827,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_instance_variable_and_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_instance_variable_and_write_node_create(parser, (yp_instance_variable_read_node_t *) node, &token, value); yp_node_destroy(parser, node); return result; @@ -12922,7 +12928,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_instance_variable_or_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_instance_variable_or_write_node_create(parser, (yp_instance_variable_read_node_t *) node, &token, value); yp_node_destroy(parser, node); return result; @@ -13033,7 +13039,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_instance_variable_operator_write_node_create(parser, (yp_instance_variable_read_node_t *) node, &token, value); yp_node_destroy(parser, node); return result; From 80dc570a451ed493797fa68b59a8dd60c1fa51a6 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Tue, 29 Aug 2023 16:13:15 -0400 Subject: [PATCH 111/208] Compile more YARP node types (#8322) * Add several more node simple types to YARP's compiler: Nodes include: DefinedNode, EmbeddedStatementsNode, LocalVariableReadNode, LocalVariableWriteNode, MultiWriteNode, OptionalParameterNode, SplatNode, YieldNode * Add AssocSplatNode, RangeNode * Add RangeNode, other helpers for future nodes * Add ArrayNode, HashNode, static literal helpers * Add branch conditionals * Add IfNode, UnlessNode * Add ScopeNode * NEW_ISEQ and NEW_CHILD_ISEQ implemented for YARP * Add nodes that depend on ScopeNode * Addressed PR comments --- iseq.c | 46 +++ yarp/yarp_compiler.c | 900 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 945 insertions(+), 1 deletion(-) diff --git a/iseq.c b/iseq.c index cb54d267f4950a..9082cd4861783c 100644 --- a/iseq.c +++ b/iseq.c @@ -944,6 +944,52 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea VALUE rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t * yarp_pointer, yp_parser_t *parser); +rb_iseq_t * +yp_iseq_new_with_opt(yp_node_t *node, yp_parser_t *parser, VALUE name, VALUE path, VALUE realpath, + int first_lineno, const rb_iseq_t *parent, int isolated_depth, + enum rb_iseq_type type, const rb_compile_option_t *option) +{ + rb_iseq_t *iseq = iseq_alloc(); + rb_compile_option_t new_opt; + + if (option) { + new_opt = *option; + } + else { + new_opt = COMPILE_OPTION_DEFAULT; + } + + VALUE script_lines = Qnil; + + rb_code_location_t code_loc; + + if (node) { + yp_line_column_t start_line_col = yp_newline_list_line_column(&(parser->newline_list), node->location.start); + yp_line_column_t end_line_col= yp_newline_list_line_column(&(parser->newline_list), node->location.end); + code_loc = (rb_code_location_t) { + .beg_pos = { + .lineno = (int)start_line_col.line, + .column = (int)start_line_col.column + }, + .end_pos = { + .lineno = (int)end_line_col.line, + .column = (int)end_line_col.column + }, + }; + } + + // TODO: node_id + int node_id = -1; + prepare_iseq_build(iseq, name, path, realpath, first_lineno, &code_loc, node_id, + parent, isolated_depth, type, script_lines, &new_opt); + + rb_iseq_compile_yarp_node(iseq, node, parser); + + finish_iseq_build(iseq); + + return iseq_translate(iseq); +} + rb_iseq_t * rb_iseq_new_with_callback( const struct rb_iseq_new_with_callback_callback_func * ifunc, diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index 81c5a36d58c671..7157c2b9faf89e 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -1,5 +1,22 @@ #include "yarp.h" +#define OLD_ISEQ NEW_ISEQ +#undef NEW_ISEQ + +#define NEW_ISEQ(node, name, type, line_no) \ + yp_new_child_iseq(iseq, (node), parser, rb_fstring(name), 0, (type), (line_no)) + +#define OLD_CHILD_ISEQ NEW_CHILD_ISEQ +#undef NEW_CHILD_ISEQ + +#define NEW_CHILD_ISEQ(node, name, type, line_no) \ + yp_new_child_iseq(iseq, (node), parser, rb_fstring(name), iseq, (type), (line_no)) + +rb_iseq_t * +yp_iseq_new_with_opt(yp_scope_node_t *node, yp_parser_t *parser, VALUE name, VALUE path, VALUE realpath, + int first_lineno, const rb_iseq_t *parent, int isolated_depth, + enum rb_iseq_type type, const rb_compile_option_t *option); + static VALUE parse_number(const yp_node_t *node) { const char *start = node->location.start; @@ -42,6 +59,265 @@ parse_location_symbol(yp_location_t *location) { return parse_symbol(location->start, location->end); } +static int +yp_optimizable_range_item_p(yp_node_t *node) +{ + return (!node || node->type == YP_NODE_INTEGER_NODE || node->type == YP_NODE_NIL_NODE); +} + +static bool +yp_static_node_literal_p(yp_node_t *node) +{ + switch(node->type) { + case YP_NODE_FALSE_NODE: + case YP_NODE_FLOAT_NODE: + case YP_NODE_IMAGINARY_NODE: + case YP_NODE_INTEGER_NODE: + case YP_NODE_NIL_NODE: + case YP_NODE_RATIONAL_NODE: + case YP_NODE_SELF_NODE: + case YP_NODE_STRING_NODE: + case YP_NODE_SOURCE_ENCODING_NODE: + case YP_NODE_SOURCE_FILE_NODE: + case YP_NODE_SOURCE_LINE_NODE: + case YP_NODE_SYMBOL_NODE: + case YP_NODE_TRUE_NODE: + return true; + default: + return false; + } +} + +static inline VALUE +yp_static_literal_value(yp_node_t *node) +{ + switch(node->type) { + case YP_NODE_NIL_NODE: + return Qnil; + case YP_NODE_TRUE_NODE: + return Qtrue; + case YP_NODE_FALSE_NODE: + return Qfalse; + // TODO: Implement this method for the other literal nodes described above + default: + rb_bug("This node type doesn't have a literal value"); + } +} + +static void +yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_node_t *cond, + LABEL *then_label, LABEL *else_label, const char *src, bool popped, yp_compile_context_t *compile_context); + +static void +yp_compile_logical(rb_iseq_t *iseq, LINK_ANCHOR *const ret, yp_node_t *cond, + LABEL *then_label, LABEL *else_label, const char *src, bool popped, yp_compile_context_t *compile_context) +{ + yp_parser_t *parser = compile_context->parser; + yp_newline_list_t newline_list = parser->newline_list; + int lineno = (int)yp_newline_list_line_column(&newline_list, cond->location.start).line; + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + + DECL_ANCHOR(seq); + INIT_ANCHOR(seq); + LABEL *label = NEW_LABEL(lineno); + if (!then_label) then_label = label; + else if (!else_label) else_label = label; + + yp_compile_branch_condition(iseq, seq, cond, then_label, else_label, src, popped, compile_context); + + if (LIST_INSN_SIZE_ONE(seq)) { + INSN *insn = (INSN *)ELEM_FIRST_INSN(FIRST_ELEMENT(seq)); + if (insn->insn_id == BIN(jump) && (LABEL *)(insn->operands[0]) == label) + return; + } + if (!label->refcnt) { + ADD_INSN(seq, &dummy_line_node, putnil); + } + else { + ADD_LABEL(seq, label); + } + ADD_SEQ(ret, seq); + return; +} + +static void yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *context); + +static void +yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_node_t *cond, + LABEL *then_label, LABEL *else_label, const char *src, bool popped, yp_compile_context_t *compile_context) +{ + yp_parser_t *parser = compile_context->parser; + yp_newline_list_t newline_list = parser->newline_list; + int lineno = (int) yp_newline_list_line_column(&newline_list, cond->location.start).line; + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + +again: + switch (YP_NODE_TYPE(cond)) { + case YP_NODE_AND_NODE: { + yp_and_node_t *and_node = (yp_and_node_t *)cond; + yp_compile_logical(iseq, ret, and_node->left, NULL, else_label, src, popped, compile_context); + cond = and_node->right; + goto again; + } + case YP_NODE_OR_NODE: { + yp_or_node_t *or_node = (yp_or_node_t *)cond; + yp_compile_logical(iseq, ret, or_node->left, then_label, NULL, src, popped, compile_context); + cond = or_node->right; + goto again; + } + case YP_NODE_FALSE_NODE: + case YP_NODE_NIL_NODE: + ADD_INSNL(ret, &dummy_line_node, jump, else_label); + return; + case YP_NODE_FLOAT_NODE: + case YP_NODE_IMAGINARY_NODE: + case YP_NODE_INTEGER_NODE: + case YP_NODE_LAMBDA_NODE: + case YP_NODE_RATIONAL_NODE: + case YP_NODE_REGULAR_EXPRESSION_NODE: + case YP_NODE_STRING_NODE: + case YP_NODE_SYMBOL_NODE: + case YP_NODE_TRUE_NODE: + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + return; + // TODO: Several more nodes in this case statement + default: + { + DECL_ANCHOR(cond_seq); + INIT_ANCHOR(cond_seq); + + yp_compile_node(iseq, cond, cond_seq, src, Qfalse, compile_context); + ADD_SEQ(ret, cond_seq); + } + break; + } + ADD_INSNL(ret, &dummy_line_node, branchunless, else_label); + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + return; +} + +static void +yp_compile_if(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) { + yp_parser_t *parser = compile_context->parser; + yp_statements_node_t *node_body; + yp_node_t *node_else; + yp_node_t *predicate; + + if (node->type == YP_NODE_IF_NODE) { + yp_if_node_t *if_node = (yp_if_node_t *)node; + node_body = if_node->statements; + node_else = if_node->consequent; + predicate = if_node->predicate; + } + else { + yp_unless_node_t *unless_node = (yp_unless_node_t *)node; + node_body = unless_node->statements; + node_else = (yp_node_t *)(unless_node->consequent); + predicate = unless_node->predicate; + } + + const int line = (int)yp_newline_list_line_column(&(parser->newline_list), node->location.start).line; + NODE line_node = generate_dummy_line_node(line, line); + + DECL_ANCHOR(cond_seq); + + LABEL *then_label, *else_label, *end_label; + + INIT_ANCHOR(cond_seq); + then_label = NEW_LABEL(line); + else_label = NEW_LABEL(line); + end_label = 0; + + yp_compile_branch_condition(iseq, cond_seq, predicate, then_label, else_label, src, popped, compile_context); + ADD_SEQ(ret, cond_seq); + + if (then_label->refcnt) { + ADD_LABEL(ret, then_label); + + DECL_ANCHOR(then_seq); + INIT_ANCHOR(then_seq); + if (node_body) { + yp_compile_node(iseq, (yp_node_t *)node_body, then_seq, src, popped, compile_context); + } + else { + if (!popped) { + ADD_INSN(ret, &line_node, putnil); + } + } + + if (else_label->refcnt) { + end_label = NEW_LABEL(line); + ADD_INSNL(then_seq, &line_node, jump, end_label); + if (!popped) { + ADD_INSN(then_seq, &line_node, pop); + } + } + ADD_SEQ(ret, then_seq); + } + + if (else_label->refcnt) { + ADD_LABEL(ret, else_label); + + DECL_ANCHOR(else_seq); + INIT_ANCHOR(else_seq); + if (node_else) { + yp_compile_node(iseq, (yp_node_t *)(((yp_else_node_t *)node_else)->statements), else_seq, src, popped, compile_context); + } + else { + if (!popped) { + ADD_INSN(ret, &line_node, putnil); + } + } + + ADD_SEQ(ret, else_seq); + } + + if (end_label) { + ADD_LABEL(ret, end_label); + } + + return; +} + +static rb_iseq_t * +yp_new_child_iseq(rb_iseq_t *iseq, yp_scope_node_t * node, yp_parser_t *parser, + VALUE name, const rb_iseq_t *parent, enum rb_iseq_type type, int line_no) +{ + debugs("[new_child_iseq]> ---------------------------------------\n"); + int isolated_depth = ISEQ_COMPILE_DATA(iseq)->isolated_depth; + rb_iseq_t * ret_iseq = yp_iseq_new_with_opt(node, parser, name, + rb_iseq_path(iseq), rb_iseq_realpath(iseq), + line_no, parent, + isolated_depth ? isolated_depth + 1 : 0, + type, ISEQ_COMPILE_DATA(iseq)->option); + debugs("[new_child_iseq]< ---------------------------------------\n"); + return ret_iseq; +} + + +static int +yp_compile_class_path(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const yp_node_t *constant_path_node, const NODE *line_node) +{ + if (constant_path_node->type == YP_NODE_CONSTANT_PATH_NODE) { + if (((yp_constant_path_node_t *)constant_path_node)->parent) { + /* Bar::Foo */ + // TODO: yp_compile_node(ret, "nd_else->nd_head", cpath->nd_head)); + return VM_DEFINECLASS_FLAG_SCOPED; + } + else { + /* toplevel class ::Foo */ + ADD_INSN1(ret, line_node, putobject, rb_cObject); + return VM_DEFINECLASS_FLAG_SCOPED; + } + } + else { + /* class at cbase Foo */ + ADD_INSN1(ret, line_node, putspecialobject, + INT2FIX(VM_SPECIAL_OBJECT_CONST_BASE)); + return 0; + } +} + /* * Compiles a YARP node into instruction sequences * @@ -97,12 +373,46 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } return; } + case YP_NODE_ARRAY_NODE: { + yp_array_node_t *array_node = (yp_array_node_t *) node; + yp_node_list_t elements = array_node->elements; + if (elements.size == 1 && yp_static_node_literal_p(elements.nodes[0])) { + VALUE ary = rb_ary_hidden_new(1); + rb_ary_push(ary, yp_static_literal_value(elements.nodes[0])); + OBJ_FREEZE(ary); + + ADD_INSN1(ret, &dummy_line_node, duparray, ary); + } + else { + for (size_t index = 0; index < elements.size; index++) { + yp_compile_node(iseq, elements.nodes[index], ret, src, popped, compile_context); + } + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(elements.size)); + } + } + + return; + } case YP_NODE_ASSOC_NODE: { yp_assoc_node_t *assoc_node = (yp_assoc_node_t *) node; yp_compile_node(iseq, assoc_node->key, ret, src, popped, compile_context); yp_compile_node(iseq, assoc_node->value, ret, src, popped, compile_context); return; } + case YP_NODE_ASSOC_SPLAT_NODE: { + yp_assoc_splat_node_t *assoc_splat_node = (yp_assoc_splat_node_t *)node; + yp_compile_node(iseq, assoc_splat_node->value, ret, src, popped, compile_context); + + // TODO: Not sure this is accurate, look at FLUSH_CHUNK in the compiler + ADD_INSN1(ret, &dummy_line_node, newarraykwsplat, INT2FIX(0)); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } case YP_NODE_BEGIN_NODE: { yp_begin_node_t *begin_node = (yp_begin_node_t *) node; if (begin_node->statements) { @@ -110,6 +420,87 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } return; } + case YP_NODE_CALL_NODE: { + yp_call_node_t *call_node = (yp_call_node_t *) node; + + ID mid = parse_string_symbol(&call_node->name); + int flags = 0; + int orig_argc = 0; + + if (call_node->receiver == NULL) { + ADD_INSN(ret, &dummy_line_node, putself); + } else { + yp_compile_node(iseq, call_node->receiver, ret, src, popped, compile_context); + } + + if (call_node->arguments == NULL) { + if (flags & VM_CALL_FCALL) { + flags |= VM_CALL_VCALL; + } + } else { + yp_arguments_node_t *arguments = call_node->arguments; + yp_compile_node(iseq, (yp_node_t *) arguments, ret, src, popped, compile_context); + orig_argc = (int)arguments->arguments.size; + } + + VALUE block_iseq = Qnil; + if (call_node->block != NULL) { + // Scope associated with the block + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)call_node->block, &scope_node); + + const rb_iseq_t *block_iseq = NEW_CHILD_ISEQ(&scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); + ISEQ_COMPILE_DATA(iseq)->current_block = block_iseq; + ADD_SEND_WITH_BLOCK(ret, &dummy_line_node, mid, INT2FIX(orig_argc), block_iseq); + } + else { + if (block_iseq == Qnil && flags == 0) { + flags |= VM_CALL_ARGS_SIMPLE; + } + + if (call_node->receiver == NULL) { + flags |= VM_CALL_FCALL; + + if (block_iseq == Qnil && call_node->arguments == NULL) { + flags |= VM_CALL_VCALL; + } + } + + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, mid, INT2NUM(orig_argc), INT2FIX(flags)); + } + return; + } + case YP_NODE_CLASS_NODE: { + yp_class_node_t *class_node = (yp_class_node_t *)node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)class_node, &scope_node); + + ID class_id = parse_string_symbol(&class_node->name); + + VALUE class_name = rb_str_freeze(rb_sprintf("", rb_id2str(class_id))); + + const rb_iseq_t *class_iseq = NEW_CHILD_ISEQ(&scope_node, class_name, ISEQ_TYPE_CLASS, lineno); + + // TODO: Once we merge constant path nodes correctly, fix this flag + const int flags = VM_DEFINECLASS_TYPE_CLASS | + (class_node->superclass ? VM_DEFINECLASS_FLAG_HAS_SUPERCLASS : 0) | + yp_compile_class_path(ret, iseq, class_node->constant_path, &dummy_line_node); + + if (class_node->superclass) { + yp_compile_node(iseq, class_node->superclass, ret, src, popped, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_INSN3(ret, &dummy_line_node, defineclass, ID2SYM(class_id), class_iseq, INT2FIX(flags)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)class_iseq); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } case YP_NODE_CLASS_VARIABLE_READ_NODE: if (!popped) { ID cvar_name = parse_node_symbol((yp_node_t *)node); @@ -176,6 +567,41 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN1(ret, &dummy_line_node, setconstant, ID2SYM(constant_name)); return; } + case YP_NODE_DEF_NODE: { + yp_def_node_t *def_node = (yp_def_node_t *) node; + ID method_name = parse_location_symbol(&def_node->name_loc); + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)def_node, &scope_node); + rb_iseq_t *method_iseq = NEW_ISEQ(&scope_node, rb_id2str(method_name), ISEQ_TYPE_METHOD, lineno); + + ADD_INSN2(ret, &dummy_line_node, definemethod, ID2SYM(method_name), method_iseq); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)method_iseq); + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, ID2SYM(method_name)); + } + return; + } + case YP_NODE_DEFINED_NODE: { + ADD_INSN(ret, &dummy_line_node, putself); + yp_defined_node_t *defined_node = (yp_defined_node_t *)node; + // TODO: Correct defined_type + enum defined_type dtype = DEFINED_CONST; + VALUE sym; + + sym = parse_number(defined_node->value); + + ADD_INSN3(ret, &dummy_line_node, defined, INT2FIX(dtype), sym, rb_iseq_defined_string(dtype)); + return; + } + case YP_NODE_EMBEDDED_STATEMENTS_NODE: { + yp_embedded_statements_node_t *embedded_statements_node = (yp_embedded_statements_node_t *)node; + + if (embedded_statements_node->statements) + yp_compile_node(iseq, (yp_node_t *) (embedded_statements_node->statements), ret, src, popped, compile_context); + // TODO: Concatenate the strings that exist here + return; + } case YP_NODE_EMBEDDED_VARIABLE_NODE: { yp_embedded_variable_node_t *embedded_node = (yp_embedded_variable_node_t *)node; yp_compile_node(iseq, embedded_node->variable, ret, src, popped, compile_context); @@ -186,6 +612,47 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); } return; + case YP_NODE_FLIP_FLOP_NODE: { + // TODO: The labels here are wrong, figure out why..... + yp_flip_flop_node_t *flip_flop_node = (yp_flip_flop_node_t *)node; + + LABEL *lend = NEW_LABEL(lineno); + LABEL *then_label = NEW_LABEL(lineno); + LABEL *else_label = NEW_LABEL(lineno); + //TODO: int again = type == NODE_FLIP2; + int again = 0; + + rb_num_t cnt = ISEQ_FLIP_CNT_INCREMENT(ISEQ_BODY(iseq)->local_iseq) + + VM_SVAR_FLIPFLOP_START; + VALUE key = INT2FIX(cnt); + + ADD_INSN2(ret, &dummy_line_node, getspecial, key, INT2FIX(0)); + ADD_INSNL(ret, &dummy_line_node, branchif, lend); + + yp_compile_node(iseq, flip_flop_node->left, ret, src, popped, compile_context); + /* *flip == 0 */ + ADD_INSNL(ret, &dummy_line_node, branchunless, else_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + ADD_INSN1(ret, &dummy_line_node, setspecial, key); + if (!again) { + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + } + + /* *flip == 1 */ + ADD_LABEL(ret, lend); + yp_compile_node(iseq, flip_flop_node->right, ret, src, popped, compile_context); + ADD_INSNL(ret, &dummy_line_node, branchunless, then_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + ADD_INSN1(ret, &dummy_line_node, setspecial, key); + ADD_INSNL(ret, &dummy_line_node, jump, then_label); + ADD_LABEL(ret, then_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); + ADD_INSNL(ret, &dummy_line_node, jump, lend); + ADD_LABEL(ret, else_label); + ADD_INSN1(ret, &dummy_line_node, putobject, Qfalse); + ADD_LABEL(ret, lend); + return; + } case YP_NODE_FLOAT_NODE: { if (!popped) { ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); @@ -208,6 +675,37 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN1(ret, &dummy_line_node, setglobal, ID2SYM(ivar_name)); return; } + case YP_NODE_HASH_NODE: { + yp_hash_node_t *hash_node = (yp_hash_node_t *) node; + yp_node_list_t elements = hash_node->elements; + + if (elements.size == 1) { + assert(elements.nodes[0]->type == YP_NODE_ASSOC_NODE); + yp_assoc_node_t *assoc_node = (yp_assoc_node_t *) elements.nodes[0]; + + if (yp_static_node_literal_p(assoc_node->key) && + yp_static_node_literal_p(assoc_node->value)) { + VALUE hash = rb_hash_new_with_size(1); + hash = rb_obj_hide(hash); + OBJ_FREEZE(hash); + ADD_INSN1(ret, &dummy_line_node, duphash, hash); + return; + } + } + + for (size_t index = 0; index < elements.size; index++) { + yp_compile_node(iseq, elements.nodes[index], ret, src, popped, compile_context); + } + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(elements.size * 2)); + } + return; + } + case YP_NODE_IF_NODE: { + yp_compile_if(iseq, node, ret, src, popped, compile_context); + return; + } case YP_NODE_IMAGINARY_NODE: { if (!popped) { ADD_INSN1(ret, &dummy_line_node, putobject, parse_number(node)); @@ -312,10 +810,110 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(elements.size * 2)); return; } + case YP_NODE_LAMBDA_NODE: { + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)node, &scope_node); + + const rb_iseq_t *block = NEW_CHILD_ISEQ(&scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); + VALUE argc = INT2FIX(0); + + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + ADD_CALL_WITH_BLOCK(ret, &dummy_line_node, idLambda, argc, block); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)block); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_LOCAL_VARIABLE_READ_NODE: { + yp_local_variable_read_node_t *local_read_node = (yp_local_variable_read_node_t *) node; + + yp_constant_id_t constant_id = local_read_node->name; + st_data_t local_index; + + for(uint32_t i = 0; i < local_read_node->depth; i++) { + compile_context = compile_context->previous; + iseq = (rb_iseq_t *)ISEQ_BODY(iseq)->parent_iseq; + } + + int num_params = ISEQ_BODY(iseq)->param.size; + + if (!st_lookup(compile_context->index_lookup_table, constant_id, &local_index)) { + rb_bug("This local does not exist"); + } + + int index = num_params - (int)local_index; + + if (!popped) { + ADD_GETLOCAL(ret, &dummy_line_node, index, local_read_node->depth); + } + return; + } + case YP_NODE_LOCAL_VARIABLE_WRITE_NODE: { + yp_local_variable_write_node_t *local_write_node = (yp_local_variable_write_node_t *) node; + yp_compile_node(iseq, local_write_node->value, ret, src, false, compile_context); + + if (!popped) { + ADD_INSN(ret, &dummy_line_node, dup); + } + + yp_constant_id_t constant_id = local_write_node->name; + size_t stack_index; + + if (!st_lookup(compile_context->index_lookup_table, constant_id, &stack_index)) { + rb_bug("This local doesn't exist"); + } + + unsigned int num_params = ISEQ_BODY(iseq)->param.size; + size_t index = num_params - stack_index; + + ADD_SETLOCAL(ret, &dummy_line_node, (int)index, local_write_node->depth); + return; + } case YP_NODE_MISSING_NODE: { rb_bug("A yp_missing_node_t should not exist in YARP's AST."); return; } + case YP_NODE_MODULE_NODE: { + yp_module_node_t *module_node = (yp_module_node_t *)node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)module_node, &scope_node); + + ID module_id = parse_string_symbol(&module_node->name); + VALUE module_name = rb_str_freeze(rb_sprintf("", rb_id2str(module_id))); + + const rb_iseq_t *module_iseq = NEW_CHILD_ISEQ(&scope_node, module_name, ISEQ_TYPE_CLASS, lineno); + + const int flags = VM_DEFINECLASS_TYPE_MODULE | + yp_compile_class_path(ret, iseq, module_node->constant_path, &dummy_line_node); + + ADD_INSN (ret, &dummy_line_node, putnil); + ADD_INSN3(ret, &dummy_line_node, defineclass, ID2SYM(module_id), module_iseq, INT2FIX(flags)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)module_iseq); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } + case YP_NODE_MULTI_WRITE_NODE: { + yp_multi_write_node_t *multi_write_node = (yp_multi_write_node_t *)node; + yp_compile_node(iseq, multi_write_node->value, ret, src, popped, compile_context); + + // TODO: int flag = 0x02 | (NODE_NAMED_REST_P(restn) ? 0x01 : 0x00); + int flag = 0x00; + + ADD_INSN(ret, &dummy_line_node, dup); + ADD_INSN2(ret, &dummy_line_node, expandarray, INT2FIX(multi_write_node->targets.size), INT2FIX(flag)); + yp_node_list_t node_list = multi_write_node->targets; + + for (size_t index = 0; index < node_list.size; index++) { + yp_compile_node(iseq, node_list.nodes[index], ret, src, popped, compile_context); + } + + return; + } case YP_NODE_NIL_NODE: if (!popped) { ADD_INSN(ret, &dummy_line_node, putnil); @@ -340,6 +938,24 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; } + case YP_NODE_OPTIONAL_PARAMETER_NODE: { + yp_optional_parameter_node_t *optional_parameter_node = (yp_optional_parameter_node_t *)node; + yp_compile_node(iseq, optional_parameter_node->value, ret, src, false, compile_context); + + yp_constant_id_t constant_id = optional_parameter_node->name; + + size_t param_number; + if (!st_lookup(compile_context->index_lookup_table, constant_id, ¶m_number)) { + rb_bug("This local doesn't exist"); + } + + unsigned int num_params = ISEQ_BODY(iseq)->param.size; + int index = (int) (num_params - param_number); + + ADD_SETLOCAL(ret, &dummy_line_node, index, 0); + + return; + } case YP_NODE_PARENTHESES_NODE: { yp_parentheses_node_t *parentheses_node = (yp_parentheses_node_t *) node; @@ -363,6 +979,42 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN(ret, &dummy_line_node, leave); return; } + case YP_NODE_RANGE_NODE: { + yp_range_node_t *range_node = (yp_range_node_t *) node; + bool exclusive = (range_node->operator_loc.end - range_node->operator_loc.start) == 3; + + if (yp_optimizable_range_item_p(range_node->left) && yp_optimizable_range_item_p(range_node->right)) { + if (!popped) { + yp_node_t *left = range_node->left; + yp_node_t *right = range_node->right; + VALUE val = rb_range_new( + left && left->type == YP_NODE_INTEGER_NODE ? parse_number(left) : Qnil, + right && right->type == YP_NODE_INTEGER_NODE ? parse_number(right) : Qnil, + exclusive + ); + ADD_INSN1(ret, &dummy_line_node, putobject, val); + RB_OBJ_WRITTEN(iseq, Qundef, val); + } + } + else { + if (range_node->left == NULL) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, range_node->left, ret, src, popped, compile_context); + } + + if (range_node->right == NULL) { + ADD_INSN(ret, &dummy_line_node, putnil); + } else { + yp_compile_node(iseq, range_node->right, ret, src, popped, compile_context); + } + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newrange, INT2FIX(exclusive)); + } + } + return; + } case YP_NODE_RETURN_NODE: { yp_arguments_node_t *arguments = ((yp_return_node_t *)node)->arguments; @@ -381,9 +1033,154 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } return; } + case YP_NODE_SCOPE_NODE: { + yp_scope_node_t *scope_node = (yp_scope_node_t *)node; + yp_constant_id_list_t locals = scope_node->locals; + + yp_parameters_node_t *parameters_node = (yp_parameters_node_t *)scope_node->parameters; + yp_node_list_t requireds_list = YP_EMPTY_NODE_LIST; + yp_node_list_t optionals_list = YP_EMPTY_NODE_LIST; + + + if (parameters_node) { + requireds_list = parameters_node->requireds; + optionals_list = parameters_node->optionals; + } + + size_t size = locals.size; + + // Index lookup table buffer size is only the number of the locals + st_table *index_lookup_table = st_init_numtable(); + + VALUE idtmp = 0; + rb_ast_id_table_t *tbl = ALLOCV(idtmp, sizeof(rb_ast_id_table_t) + size * sizeof(ID)); + tbl->size = (int)size; + + // First param gets 0, second param 1, param n... + // Calculate the local index for all locals + for (size_t i = 0; i < size; i++) { + yp_constant_id_t constant_id = locals.ids[i]; + ID local = compile_context->constants[constant_id - 1]; + tbl->ids[i] = local; + st_insert(index_lookup_table, constant_id, i); + } + + yp_compile_context_t scope_compile_context = { + .parser = parser, + .previous = compile_context, + .constants = compile_context->constants, + .index_lookup_table = index_lookup_table + }; + + ISEQ_BODY(iseq)->param.lead_num = (int)requireds_list.size; + ISEQ_BODY(iseq)->param.opt_num = (int)optionals_list.size; + // TODO: Set all the other nums (good comment by lead_num illustrating what they are) + ISEQ_BODY(iseq)->param.size = (unsigned int)size; + + if (optionals_list.size) { + LABEL **opt_table = (LABEL **)ALLOC_N(VALUE, optionals_list.size + 1); + LABEL *label; + + // TODO: Should we make an api for NEW_LABEL where you can pass + // a pointer to the label it should fill out? We already + // have a list of labels allocated above so it seems wasteful + // to do the copies. + for (size_t i = 0; i < optionals_list.size; i++) { + label = NEW_LABEL(lineno); + opt_table[i] = label; + ADD_LABEL(ret, label); + yp_node_t *optional_node = optionals_list.nodes[i]; + yp_compile_node(iseq, optional_node, ret, src, false, &scope_compile_context); + } + + // Set the last label + label = NEW_LABEL(lineno); + opt_table[optionals_list.size] = label; + ADD_LABEL(ret, label); + + ISEQ_BODY(iseq)->param.flags.has_opt = TRUE; + ISEQ_BODY(iseq)->param.opt_table = (const VALUE *)opt_table; + } + + iseq_set_local_table(iseq, tbl); + + switch (ISEQ_BODY(iseq)->type) { + case ISEQ_TYPE_BLOCK: + { + LABEL *start = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(0); + LABEL *end = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(0); + + start->rescued = LABEL_RESCUE_BEG; + end->rescued = LABEL_RESCUE_END; + + ADD_TRACE(ret, RUBY_EVENT_B_CALL); + NODE dummy_line_node = generate_dummy_line_node(ISEQ_BODY(iseq)->location.first_lineno, -1); + ADD_INSN (ret, &dummy_line_node, nop); + ADD_LABEL(ret, start); + + yp_compile_node(iseq, (yp_node_t *)(scope_node->body), ret, src, popped, &scope_compile_context); + + ADD_LABEL(ret, end); + ADD_TRACE(ret, RUBY_EVENT_B_RETURN); + ISEQ_COMPILE_DATA(iseq)->last_line = ISEQ_BODY(iseq)->location.code_location.end_pos.lineno; + + /* wide range catch handler must put at last */ + ADD_CATCH_ENTRY(CATCH_TYPE_REDO, start, end, NULL, start); + ADD_CATCH_ENTRY(CATCH_TYPE_NEXT, start, end, NULL, end); + break; + } + default: + if (scope_node->body) { + yp_compile_node(iseq, (yp_node_t *)(scope_node->body), ret, src, popped, &scope_compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + } + + free(index_lookup_table); + + ADD_INSN(ret, &dummy_line_node, leave); + return; + } case YP_NODE_SELF_NODE: ADD_INSN(ret, &dummy_line_node, putself); return; + case YP_NODE_SINGLETON_CLASS_NODE: { + yp_singleton_class_node_t *singleton_class_node = (yp_singleton_class_node_t *)node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)singleton_class_node, &scope_node); + + const rb_iseq_t *singleton_class = NEW_ISEQ(&scope_node, rb_fstring_lit("singleton class"), + ISEQ_TYPE_CLASS, lineno); + + yp_compile_node(iseq, singleton_class_node->expression, ret, src, popped, compile_context); + ADD_INSN(ret, &dummy_line_node, putnil); + ID singletonclass; + CONST_ID(singletonclass, "singletonclass"); + + ADD_INSN3(ret, &dummy_line_node, defineclass, + ID2SYM(singletonclass), singleton_class, + INT2FIX(VM_DEFINECLASS_TYPE_SINGLETON_CLASS)); + RB_OBJ_WRITTEN(iseq, Qundef, (VALUE)singleton_class); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + return; + } + case YP_NODE_SPLAT_NODE: { + yp_splat_node_t *splat_node = (yp_splat_node_t *)node; + yp_compile_node(iseq, splat_node->expression, ret, src, popped, compile_context); + + ADD_INSN1(ret, &dummy_line_node, splatarray, Qtrue); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + return; + } case YP_NODE_STATEMENTS_NODE: { yp_statements_node_t *statements_node = (yp_statements_node_t *) node; yp_node_list_t node_list = statements_node->body; @@ -436,6 +1233,80 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; } + case YP_NODE_UNLESS_NODE: { + yp_compile_if(iseq, node, ret, src, popped, compile_context); + return; + } + case YP_NODE_WHILE_NODE: { + yp_while_node_t *while_node = (yp_while_node_t *)node; + + LABEL *prev_start_label = ISEQ_COMPILE_DATA(iseq)->start_label; + LABEL *prev_end_label = ISEQ_COMPILE_DATA(iseq)->end_label; + LABEL *prev_redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label; + int prev_loopval_popped = ISEQ_COMPILE_DATA(iseq)->loopval_popped; + + // TODO: Deal with ensures in here + LABEL *next_label = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(lineno); /* next */ + LABEL *redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label = NEW_LABEL(lineno); /* redo */ + LABEL *break_label = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(lineno); /* break */ + LABEL *end_label = NEW_LABEL(lineno); + LABEL *adjust_label = NEW_LABEL(lineno); + + LABEL *next_catch_label = NEW_LABEL(lineno); + LABEL *tmp_label = NULL; + + ISEQ_COMPILE_DATA(iseq)->loopval_popped = 0; + + // begin; end while true + if (while_node->base.flags & YP_LOOP_FLAGS_BEGIN_MODIFIER) { + ADD_INSNL(ret, &dummy_line_node, jump, next_label); + } + else { + // while true; end + tmp_label = NEW_LABEL(lineno); + ADD_INSNL(ret, &dummy_line_node, jump, tmp_label); + } + + ADD_LABEL(ret, adjust_label); + ADD_INSN(ret, &dummy_line_node, putnil); + ADD_LABEL(ret, next_catch_label); + ADD_INSN(ret, &dummy_line_node, pop); + ADD_INSNL(ret, &dummy_line_node, jump, next_label); + if (tmp_label) ADD_LABEL(ret, tmp_label); + + ADD_LABEL(ret, redo_label); + if (while_node->statements) { + yp_compile_node(iseq, (yp_node_t *)while_node->statements, ret, src, Qfalse, compile_context); + } + + ADD_LABEL(ret, next_label); + + yp_compile_branch_condition(iseq, ret, while_node->predicate, redo_label, end_label, src, popped, compile_context); + + ADD_LABEL(ret, end_label); + ADD_ADJUST_RESTORE(ret, adjust_label); + + ADD_INSN(ret, &dummy_line_node, putnil); + + ADD_LABEL(ret, break_label); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + ADD_CATCH_ENTRY(CATCH_TYPE_BREAK, redo_label, break_label, NULL, + break_label); + ADD_CATCH_ENTRY(CATCH_TYPE_NEXT, redo_label, break_label, NULL, + next_catch_label); + ADD_CATCH_ENTRY(CATCH_TYPE_REDO, redo_label, break_label, NULL, + ISEQ_COMPILE_DATA(iseq)->redo_label); + + ISEQ_COMPILE_DATA(iseq)->start_label = prev_start_label; + ISEQ_COMPILE_DATA(iseq)->end_label = prev_end_label; + ISEQ_COMPILE_DATA(iseq)->redo_label = prev_redo_label; + ISEQ_COMPILE_DATA(iseq)->loopval_popped = prev_loopval_popped; + return; + } case YP_NODE_X_STRING_NODE: { yp_x_string_node_t *xstring_node = (yp_x_string_node_t *) node; ADD_INSN(ret, &dummy_line_node, putself); @@ -443,6 +1314,28 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_SEND_WITH_FLAG(ret, &dummy_line_node, rb_intern("`"), INT2NUM(1), INT2FIX(VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE)); return; } + case YP_NODE_YIELD_NODE: { + unsigned int flag = 0; + struct rb_callinfo_kwarg *keywords = NULL; + + VALUE argc = INT2FIX(0); + + ADD_INSN1(ret, &dummy_line_node, invokeblock, new_callinfo(iseq, 0, FIX2INT(argc), flag, keywords, FALSE)); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + int level = 0; + const rb_iseq_t *tmp_iseq = iseq; + for (; tmp_iseq != ISEQ_BODY(iseq)->local_iseq; level++ ) { + tmp_iseq = ISEQ_BODY(tmp_iseq)->parent_iseq; + } + + if (level > 0) access_outer_variables(iseq, level, rb_intern("yield"), true); + + return; + } default: rb_raise(rb_eNotImpError, "node type %s not implemented", yp_node_type_to_str(node->type)); return; @@ -457,6 +1350,11 @@ rb_translate_yarp(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret yp_compile_node(iseq, node, ret, node->location.start, false, compile_context); iseq_set_sequence(iseq, ret); - return Qnil; } + +#undef NEW_ISEQ +#define NEW_ISEQ OLD_ISEQ + +#undef NEW_CHILD_ISEQ +#define NEW_CHILD_ISEQ OLD_CHILD_ISEQ From 3151d7876fac408ad7060b317ae7798263870daa Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Tue, 29 Aug 2023 17:17:08 -0400 Subject: [PATCH 112/208] [YARP] Until Node, minor cleanup (#8325) * Remove conditional from yp_compile_if * Extracted yp_compile_while, compile UntilNode * Small checks for body / value that could be empty --- yarp/yarp_compiler.c | 204 ++++++++++++++++++++++++------------------- 1 file changed, 114 insertions(+), 90 deletions(-) diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index 7157c2b9faf89e..fa71c12226decb 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -197,26 +197,7 @@ yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_no } static void -yp_compile_if(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) { - yp_parser_t *parser = compile_context->parser; - yp_statements_node_t *node_body; - yp_node_t *node_else; - yp_node_t *predicate; - - if (node->type == YP_NODE_IF_NODE) { - yp_if_node_t *if_node = (yp_if_node_t *)node; - node_body = if_node->statements; - node_else = if_node->consequent; - predicate = if_node->predicate; - } - else { - yp_unless_node_t *unless_node = (yp_unless_node_t *)node; - node_body = unless_node->statements; - node_else = (yp_node_t *)(unless_node->consequent); - predicate = unless_node->predicate; - } - - const int line = (int)yp_newline_list_line_column(&(parser->newline_list), node->location.start).line; +yp_compile_if(rb_iseq_t *iseq, const int line, yp_statements_node_t *node_body, yp_node_t *node_else, yp_node_t *predicate, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) { NODE line_node = generate_dummy_line_node(line, line); DECL_ANCHOR(cond_seq); @@ -279,6 +260,84 @@ yp_compile_if(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, co return; } +static void +yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_node_type type, yp_statements_node_t *statements, yp_node_t *predicate, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) +{ + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + + LABEL *prev_start_label = ISEQ_COMPILE_DATA(iseq)->start_label; + LABEL *prev_end_label = ISEQ_COMPILE_DATA(iseq)->end_label; + LABEL *prev_redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label; + int prev_loopval_popped = ISEQ_COMPILE_DATA(iseq)->loopval_popped; + + // TODO: Deal with ensures in here + LABEL *next_label = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(lineno); /* next */ + LABEL *redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label = NEW_LABEL(lineno); /* redo */ + LABEL *break_label = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(lineno); /* break */ + LABEL *end_label = NEW_LABEL(lineno); + LABEL *adjust_label = NEW_LABEL(lineno); + + LABEL *next_catch_label = NEW_LABEL(lineno); + LABEL *tmp_label = NULL; + + ISEQ_COMPILE_DATA(iseq)->loopval_popped = 0; + + // begin; end while true + if (flags & YP_LOOP_FLAGS_BEGIN_MODIFIER) { + ADD_INSNL(ret, &dummy_line_node, jump, next_label); + } + else { + // while true; end + tmp_label = NEW_LABEL(lineno); + ADD_INSNL(ret, &dummy_line_node, jump, tmp_label); + } + + ADD_LABEL(ret, adjust_label); + ADD_INSN(ret, &dummy_line_node, putnil); + ADD_LABEL(ret, next_catch_label); + ADD_INSN(ret, &dummy_line_node, pop); + ADD_INSNL(ret, &dummy_line_node, jump, next_label); + if (tmp_label) ADD_LABEL(ret, tmp_label); + + ADD_LABEL(ret, redo_label); + if (statements) { + yp_compile_node(iseq, (yp_node_t *)statements, ret, src, Qtrue, compile_context); + } + + ADD_LABEL(ret, next_label); + + if (type == YP_NODE_WHILE_NODE) { + yp_compile_branch_condition(iseq, ret, predicate, redo_label, end_label, src, popped, compile_context); + } + else if (type == YP_NODE_UNTIL_NODE) { + yp_compile_branch_condition(iseq, ret, predicate, end_label, redo_label, src, popped, compile_context); + } + + ADD_LABEL(ret, end_label); + ADD_ADJUST_RESTORE(ret, adjust_label); + + ADD_INSN(ret, &dummy_line_node, putnil); + + ADD_LABEL(ret, break_label); + + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } + + ADD_CATCH_ENTRY(CATCH_TYPE_BREAK, redo_label, break_label, NULL, + break_label); + ADD_CATCH_ENTRY(CATCH_TYPE_NEXT, redo_label, break_label, NULL, + next_catch_label); + ADD_CATCH_ENTRY(CATCH_TYPE_REDO, redo_label, break_label, NULL, + ISEQ_COMPILE_DATA(iseq)->redo_label); + + ISEQ_COMPILE_DATA(iseq)->start_label = prev_start_label; + ISEQ_COMPILE_DATA(iseq)->end_label = prev_end_label; + ISEQ_COMPILE_DATA(iseq)->redo_label = prev_redo_label; + ISEQ_COMPILE_DATA(iseq)->loopval_popped = prev_loopval_popped; + return; +} + static rb_iseq_t * yp_new_child_iseq(rb_iseq_t *iseq, yp_scope_node_t * node, yp_parser_t *parser, VALUE name, const rb_iseq_t *parent, enum rb_iseq_type type, int line_no) @@ -329,7 +388,8 @@ yp_compile_class_path(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const yp_node_t * * compile_context - Stores parser and local information */ static void -yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) { +yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) +{ yp_parser_t *parser = compile_context->parser; yp_newline_list_t newline_list = parser->newline_list; int lineno = (int)yp_newline_list_line_column(&newline_list, node->location.start).line; @@ -398,7 +458,9 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, case YP_NODE_ASSOC_NODE: { yp_assoc_node_t *assoc_node = (yp_assoc_node_t *) node; yp_compile_node(iseq, assoc_node->key, ret, src, popped, compile_context); - yp_compile_node(iseq, assoc_node->value, ret, src, popped, compile_context); + if (assoc_node->value) { + yp_compile_node(iseq, assoc_node->value, ret, src, popped, compile_context); + } return; } case YP_NODE_ASSOC_SPLAT_NODE: { @@ -703,7 +765,13 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; } case YP_NODE_IF_NODE: { - yp_compile_if(iseq, node, ret, src, popped, compile_context); + const int line = (int)yp_newline_list_line_column(&(parser->newline_list), node->location.start).line; + yp_if_node_t *if_node = (yp_if_node_t *)node; + yp_statements_node_t *node_body = if_node->statements; + yp_node_t *node_else = if_node->consequent; + yp_node_t *predicate = if_node->predicate; + + yp_compile_if(iseq, line, node_body, node_else, predicate, ret, src, popped, compile_context); return; } case YP_NODE_IMAGINARY_NODE: { @@ -1118,7 +1186,9 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN (ret, &dummy_line_node, nop); ADD_LABEL(ret, start); - yp_compile_node(iseq, (yp_node_t *)(scope_node->body), ret, src, popped, &scope_compile_context); + if (scope_node->body) { + yp_compile_node(iseq, (yp_node_t *)(scope_node->body), ret, src, popped, &scope_compile_context); + } ADD_LABEL(ret, end); ADD_TRACE(ret, RUBY_EVENT_B_RETURN); @@ -1234,77 +1304,31 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; } case YP_NODE_UNLESS_NODE: { - yp_compile_if(iseq, node, ret, src, popped, compile_context); + const int line = (int)yp_newline_list_line_column(&(parser->newline_list), node->location.start).line; + yp_unless_node_t *unless_node = (yp_unless_node_t *)node; + yp_statements_node_t *node_body = unless_node->statements; + yp_node_t *node_else = (yp_node_t *)(unless_node->consequent); + yp_node_t *predicate = unless_node->predicate; + + yp_compile_if(iseq, line, node_body, node_else, predicate, ret, src, popped, compile_context); + return; + } + case YP_NODE_UNTIL_NODE: { + yp_until_node_t *until_node = (yp_until_node_t *)node; + yp_statements_node_t *statements = until_node->statements; + yp_node_t *predicate = until_node->predicate; + yp_node_flags_t flags = node->flags; + + yp_compile_while(iseq, lineno, flags, node->type, statements, predicate, ret, src, popped, compile_context); return; } case YP_NODE_WHILE_NODE: { yp_while_node_t *while_node = (yp_while_node_t *)node; + yp_statements_node_t *statements = while_node->statements; + yp_node_t *predicate = while_node->predicate; + yp_node_flags_t flags = node->flags; - LABEL *prev_start_label = ISEQ_COMPILE_DATA(iseq)->start_label; - LABEL *prev_end_label = ISEQ_COMPILE_DATA(iseq)->end_label; - LABEL *prev_redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label; - int prev_loopval_popped = ISEQ_COMPILE_DATA(iseq)->loopval_popped; - - // TODO: Deal with ensures in here - LABEL *next_label = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(lineno); /* next */ - LABEL *redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label = NEW_LABEL(lineno); /* redo */ - LABEL *break_label = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(lineno); /* break */ - LABEL *end_label = NEW_LABEL(lineno); - LABEL *adjust_label = NEW_LABEL(lineno); - - LABEL *next_catch_label = NEW_LABEL(lineno); - LABEL *tmp_label = NULL; - - ISEQ_COMPILE_DATA(iseq)->loopval_popped = 0; - - // begin; end while true - if (while_node->base.flags & YP_LOOP_FLAGS_BEGIN_MODIFIER) { - ADD_INSNL(ret, &dummy_line_node, jump, next_label); - } - else { - // while true; end - tmp_label = NEW_LABEL(lineno); - ADD_INSNL(ret, &dummy_line_node, jump, tmp_label); - } - - ADD_LABEL(ret, adjust_label); - ADD_INSN(ret, &dummy_line_node, putnil); - ADD_LABEL(ret, next_catch_label); - ADD_INSN(ret, &dummy_line_node, pop); - ADD_INSNL(ret, &dummy_line_node, jump, next_label); - if (tmp_label) ADD_LABEL(ret, tmp_label); - - ADD_LABEL(ret, redo_label); - if (while_node->statements) { - yp_compile_node(iseq, (yp_node_t *)while_node->statements, ret, src, Qfalse, compile_context); - } - - ADD_LABEL(ret, next_label); - - yp_compile_branch_condition(iseq, ret, while_node->predicate, redo_label, end_label, src, popped, compile_context); - - ADD_LABEL(ret, end_label); - ADD_ADJUST_RESTORE(ret, adjust_label); - - ADD_INSN(ret, &dummy_line_node, putnil); - - ADD_LABEL(ret, break_label); - - if (popped) { - ADD_INSN(ret, &dummy_line_node, pop); - } - - ADD_CATCH_ENTRY(CATCH_TYPE_BREAK, redo_label, break_label, NULL, - break_label); - ADD_CATCH_ENTRY(CATCH_TYPE_NEXT, redo_label, break_label, NULL, - next_catch_label); - ADD_CATCH_ENTRY(CATCH_TYPE_REDO, redo_label, break_label, NULL, - ISEQ_COMPILE_DATA(iseq)->redo_label); - - ISEQ_COMPILE_DATA(iseq)->start_label = prev_start_label; - ISEQ_COMPILE_DATA(iseq)->end_label = prev_end_label; - ISEQ_COMPILE_DATA(iseq)->redo_label = prev_redo_label; - ISEQ_COMPILE_DATA(iseq)->loopval_popped = prev_loopval_popped; + yp_compile_while(iseq, lineno, flags, node->type, statements, predicate, ret, src, popped, compile_context); return; } case YP_NODE_X_STRING_NODE: { From 27024004fa9804631c6f21e2022bb2dd690e8c5c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 29 Aug 2023 19:31:53 -0400 Subject: [PATCH 113/208] Fix string2cstr in lldb_cruby.py [ci skip] --- misc/lldb_cruby.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/lldb_cruby.py b/misc/lldb_cruby.py index 95e03c620901b0..e667a466287ddf 100755 --- a/misc/lldb_cruby.py +++ b/misc/lldb_cruby.py @@ -197,12 +197,11 @@ def string2cstr(rstring): flags = rstring.GetValueForExpressionPath(".basic->flags").unsigned if flags & RUBY_T_MASK != RUBY_T_STRING: raise TypeError("not a string") + clen = int(rstring.GetValueForExpressionPath(".len").value, 0) if flags & RUBY_FL_USER1: cptr = int(rstring.GetValueForExpressionPath(".as.heap.ptr").value, 0) - clen = int(rstring.GetValueForExpressionPath(".as.heap.len").value, 0) else: cptr = int(rstring.GetValueForExpressionPath(".as.embed.ary").location, 0) - clen = int(rstring.GetValueForExpressionPath(".as.embed.len").value, 0) return cptr, clen def output_string(debugger, result, rstring): From 6a876a61d7495ff64805cdda2756c5fc38846ef8 Mon Sep 17 00:00:00 2001 From: Josh Nichols Date: Mon, 28 Aug 2023 20:02:08 -0400 Subject: [PATCH 114/208] [rubygems/rubygems] (Further) Improve Bundler::Settings#[] performance and memory usage I previously identified and improved this method over in https://github.com/rubygems/rubygems/pull/6884 but while reviewing another memory_profiler profile, I realized another gain we can eek out. This method keeps comes up in part because `configs` is allocating a new Hash every time. My last change took advantage of that by using `map!` on it. `configs` is called quite often, including in this `[]` method, so there's a benefit to memoizing it. Back in `[]`, logically we are trying to find the first Hash in `configs` that has a value for the given key. Currently, we end up `map` and `compact` to just get that value. Instead, we can use a loop over `configs`, and break when we find the value for the key. https://github.com/rubygems/rubygems/commit/b913cfc87b --- lib/bundler/settings.rb | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index f5999481a8f73b..e31f207d90ff4c 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -102,10 +102,13 @@ def initialize(root = nil) def [](name) key = key_for(name) - values = configs.values - values.map! {|config| config[key] } - values.compact! - value = values.first + value = nil + configs.each do |_, config| + if config[key] + value = config[key] + break + end + end converted_value(value, name) end @@ -316,7 +319,7 @@ def key_for(key) private def configs - { + @configs ||= { :temporary => @temporary, :local => @local_config, :env => @env_config, From e747e2c36b76262740da6070434a6768723397b2 Mon Sep 17 00:00:00 2001 From: Josh Nichols Date: Tue, 29 Aug 2023 19:01:32 -0400 Subject: [PATCH 115/208] [rubygems/rubygems] Update bundler/lib/bundler/settings.rb https://github.com/rubygems/rubygems/commit/75ffa8ef76 Co-authored-by: Martin Emde --- lib/bundler/settings.rb | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index e31f207d90ff4c..92b4a6a4e37787 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -104,10 +104,9 @@ def [](name) value = nil configs.each do |_, config| - if config[key] - value = config[key] - break - end + value = config[key] + next if value.nil? + break end converted_value(value, name) From ad2a464e8fc6bd4daa5b20b3c69d94d52464a665 Mon Sep 17 00:00:00 2001 From: Soutaro Matsumoto Date: Wed, 30 Aug 2023 11:10:41 +0900 Subject: [PATCH 116/208] Bundle RBS 3.2.1 (#8306) --- gems/bundled_gems | 4 ++-- tool/rbs_skip_tests | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/gems/bundled_gems b/gems/bundled_gems index af08c6c82d1c3e..f11380027ac77c 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -17,7 +17,7 @@ net-pop 0.1.2 https://github.com/ruby/net-pop net-smtp 0.3.3 https://github.com/ruby/net-smtp matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime -rbs 3.1.3 https://github.com/ruby/rbs -typeprof 0.21.7 https://github.com/ruby/typeprof aabc019684d8b4a1ed66c2a1ca48da7bbb18dcc0 +rbs 3.2.1 https://github.com/ruby/rbs +typeprof 0.21.8 https://github.com/ruby/typeprof debug 1.8.0 https://github.com/ruby/debug racc 1.7.1 https://github.com/ruby/racc diff --git a/tool/rbs_skip_tests b/tool/rbs_skip_tests index aefdb23a792bef..7a67e156ef55fe 100644 --- a/tool/rbs_skip_tests +++ b/tool/rbs_skip_tests @@ -24,6 +24,7 @@ test_collection_update(RBS::CliTest) running tests without Bundler test_subtract(RBS::CliTest) running tests without Bundler test_subtract_several_subtrahends(RBS::CliTest) running tests without Bundler test_subtract_write(RBS::CliTest) running tests without Bundler +test_subtract_write_removes_definition_if_empty(RBS::CliTest) runnint tests without Bundler test_aref(FiberSingletonTest) the method should not accept String keys From a83152d4dbef7be51543e6c3d7a55fabb4917248 Mon Sep 17 00:00:00 2001 From: git Date: Wed, 30 Aug 2023 02:11:23 +0000 Subject: [PATCH 117/208] Update bundled gems list at ad2a464e8fc6bd4daa5b20b3c69d94 [ci skip] --- NEWS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 9f6012394efe38..12cd8dd5abf843 100644 --- a/NEWS.md +++ b/NEWS.md @@ -105,8 +105,8 @@ The following bundled gems are updated. * rexml 3.2.6 * rss 0.3.0 * net-imap 0.3.7 -* rbs 3.1.3 -* typeprof 0.21.7 +* rbs 3.2.1 +* typeprof 0.21.8 * debug 1.8.0 The following default gem is now bundled. From 2e648bfee4178399170437156af80d5f64ee96b9 Mon Sep 17 00:00:00 2001 From: yui-knk Date: Tue, 29 Aug 2023 21:32:34 +0900 Subject: [PATCH 118/208] [DOC] Detailed explanation when one line pattern matching is a void value expression --- parse.y | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parse.y b/parse.y index ab416540eace4b..36ba905db12e61 100644 --- a/parse.y +++ b/parse.y @@ -12114,7 +12114,7 @@ value_expr_check(struct parser_params *p, NODE *node) if (node->nd_body->nd_body) { return NULL; } - /* single line pattern matching */ + /* single line pattern matching with "=>" operator */ return void_node ? void_node : node; case NODE_BLOCK: From c05737e09f1809715a672148258dd225cede331f Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 30 Aug 2023 14:59:17 +0900 Subject: [PATCH 119/208] sync_default_gems.rb: Transform by proc --- tool/sync_default_gems.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 2ac66f919500ca..ec70b8675298d1 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -81,7 +81,7 @@ module SyncDefaultGems yaml: "ruby/yaml", yarp: ["ruby/yarp", "main"], zlib: 'ruby/zlib', - }.transform_keys {|k| k.to_s} + }.transform_keys(&:to_s) CLASSICAL_DEFAULT_BRANCH = "master" From acedbcb1b4eb6b362f11e783bff53c237d05afc6 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 30 Aug 2023 15:00:00 +0900 Subject: [PATCH 120/208] sync_default_gems.rb: Fix typo in replace_rdoc_ref_all --- tool/sync_default_gems.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index ec70b8675298d1..4c3f5a6422af75 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -122,9 +122,10 @@ def replace_rdoc_ref(file) end def replace_rdoc_ref_all - result = pipe_readlines(%W"git status porcelain -z -- *.c *.rb *.rdoc") + result = pipe_readlines(%W"git status --porcelain -z -- *.c *.rb *.rdoc") result.map! {|line| line[/\A.M (.*)/, 1]} result.compact! + return if result.empty? result = pipe_readlines(%W"git grep -z -l -F [https://docs.ruby-lang.org/en/master/ --" + result) result.inject(false) {|changed, file| changed | replace_rdoc_ref(file)} end From bcc905100f1079e191632cfd02319c10af82dac0 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Tue, 20 Sep 2022 16:10:56 +0200 Subject: [PATCH 121/208] BasicSocket#recv* return `nil` rather than an empty packet [Bug #19012] man recvmsg(2) states: > Return Value > These calls return the number of bytes received, or -1 if an error occurred. > The return value will be 0 when the peer has performed an orderly shutdown. Not too sure how one is supposed to make the difference between a packet of size 0 and a closed connection. --- ext/socket/ancdata.c | 4 ++ ext/socket/init.c | 20 ++++++ ext/socket/rubysocket.h | 2 + .../library/socket/basicsocket/recv_spec.rb | 19 ++++++ .../library/socket/basicsocket/send_spec.rb | 4 +- .../socket/basicsocket/shutdown_spec.rb | 20 +++--- spec/ruby/library/socket/fixtures/classes.rb | 2 +- test/socket/test_unix.rb | 68 ++++++++++++++++++- 8 files changed, 124 insertions(+), 15 deletions(-) diff --git a/ext/socket/ancdata.c b/ext/socket/ancdata.c index 7406177de2cb6a..6ef040b692f338 100644 --- a/ext/socket/ancdata.c +++ b/ext/socket/ancdata.c @@ -1555,6 +1555,10 @@ bsock_recvmsg_internal(VALUE sock, ss = rb_recvmsg(fptr->fd, &mh, flags); + if (ss == 0 && !rsock_is_dgram(fptr)) { + return Qnil; + } + if (ss == -1) { int e; if (!nonblock && rb_io_maybe_wait_readable(errno, fptr->self, RUBY_IO_TIMEOUT_DEFAULT)) { diff --git a/ext/socket/init.c b/ext/socket/init.c index 557d4374a51e0c..e9dc6f84839f25 100644 --- a/ext/socket/init.c +++ b/ext/socket/init.c @@ -116,6 +116,7 @@ recvfrom_blocking(void *data) ssize_t ret; ret = recvfrom(arg->fd, RSTRING_PTR(arg->str), arg->length, arg->flags, &arg->buf.addr, &arg->alen); + if (ret != -1 && len0 < arg->alen) arg->alen = len0; @@ -147,6 +148,18 @@ recvfrom_locktmp(VALUE v) return rb_thread_io_blocking_region(recvfrom_blocking, arg, arg->fd); } +int +rsock_is_dgram(rb_io_t *fptr) +{ + int socktype; + socklen_t optlen = (socklen_t)sizeof(socktype); + int ret = getsockopt(fptr->fd, SOL_SOCKET, SO_TYPE, (void*)&socktype, &optlen); + if (ret == -1) { + rb_sys_fail("getsockopt(SO_TYPE)"); + } + return socktype == SOCK_DGRAM; +} + VALUE rsock_s_recvfrom(VALUE socket, int argc, VALUE *argv, enum sock_recv_type from) { @@ -187,6 +200,9 @@ rsock_s_recvfrom(VALUE socket, int argc, VALUE *argv, enum sock_recv_type from) slen = (long)rb_str_locktmp_ensure(str, recvfrom_locktmp, (VALUE)&arg); + if (slen == 0 && !rsock_is_dgram(fptr)) { + return Qnil; + } if (slen >= 0) break; if (!rb_io_maybe_wait_readable(errno, socket, RUBY_IO_TIMEOUT_DEFAULT)) @@ -259,6 +275,10 @@ rsock_s_recvfrom_nonblock(VALUE sock, VALUE len, VALUE flg, VALUE str, if (slen != -1 && len0 < alen) alen = len0; + if (slen == 0 && !rsock_is_dgram(fptr)) { + return Qnil; + } + if (slen < 0) { int e = errno; switch (e) { diff --git a/ext/socket/rubysocket.h b/ext/socket/rubysocket.h index 5f803ba0da3c6f..7c5739808d1618 100644 --- a/ext/socket/rubysocket.h +++ b/ext/socket/rubysocket.h @@ -459,6 +459,8 @@ VALUE rsock_write_nonblock(VALUE sock, VALUE buf, VALUE ex); void rsock_make_fd_nonblock(int fd); +int rsock_is_dgram(rb_io_t *fptr); + #if !defined HAVE_INET_NTOP && ! defined _WIN32 const char *inet_ntop(int, const void *, char *, size_t); #elif defined __MINGW32__ diff --git a/spec/ruby/library/socket/basicsocket/recv_spec.rb b/spec/ruby/library/socket/basicsocket/recv_spec.rb index b6ccda5d00878d..a56114f4ab57d6 100644 --- a/spec/ruby/library/socket/basicsocket/recv_spec.rb +++ b/spec/ruby/library/socket/basicsocket/recv_spec.rb @@ -32,6 +32,25 @@ ScratchPad.recorded.should == 'hello' end + ruby_version_is "3.3" do + it "returns nil on a closed stream socket" do + t = Thread.new do + client = @server.accept + packet = client.recv(10) + client.close + packet + end + + Thread.pass while t.status and t.status != "sleep" + t.status.should_not be_nil + + socket = TCPSocket.new('127.0.0.1', @port) + socket.close + + t.value.should be_nil + end + end + platform_is_not :solaris do it "accepts flags to specify unusual receiving behaviour" do t = Thread.new do diff --git a/spec/ruby/library/socket/basicsocket/send_spec.rb b/spec/ruby/library/socket/basicsocket/send_spec.rb index 868801df304a1c..041ce03d72c7ee 100644 --- a/spec/ruby/library/socket/basicsocket/send_spec.rb +++ b/spec/ruby/library/socket/basicsocket/send_spec.rb @@ -22,7 +22,7 @@ client = @server.accept loop do got = client.recv(5) - break if got.empty? + break if got.nil? || got.empty? data << got end client.close @@ -67,7 +67,7 @@ client = @server.accept loop do got = client.recv(5) - break if got.empty? + break if got.nil? || got.empty? data << got end client.close diff --git a/spec/ruby/library/socket/basicsocket/shutdown_spec.rb b/spec/ruby/library/socket/basicsocket/shutdown_spec.rb index 41d9581bdeb3b1..c78b32de38c3cd 100644 --- a/spec/ruby/library/socket/basicsocket/shutdown_spec.rb +++ b/spec/ruby/library/socket/basicsocket/shutdown_spec.rb @@ -23,7 +23,7 @@ it 'shuts down a socket for reading' do @client.shutdown(Socket::SHUT_RD) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for writing' do @@ -35,7 +35,7 @@ it 'shuts down a socket for reading and writing' do @client.shutdown(Socket::SHUT_RDWR) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty -> { @client.write('hello') }.should raise_error(Errno::EPIPE) end @@ -49,13 +49,13 @@ it 'shuts down a socket for reading using :RD' do @client.shutdown(:RD) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading using :SHUT_RD' do @client.shutdown(:SHUT_RD) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for writing using :WR' do @@ -73,7 +73,7 @@ it 'shuts down a socket for reading and writing' do @client.shutdown(:RDWR) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty -> { @client.write('hello') }.should raise_error(Errno::EPIPE) end @@ -87,13 +87,13 @@ it 'shuts down a socket for reading using "RD"' do @client.shutdown('RD') - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading using "SHUT_RD"' do @client.shutdown('SHUT_RD') - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for writing using "WR"' do @@ -123,7 +123,7 @@ @client.shutdown(@dummy) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading using "SHUT_RD"' do @@ -131,7 +131,7 @@ @client.shutdown(@dummy) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty end it 'shuts down a socket for reading and writing' do @@ -139,7 +139,7 @@ @client.shutdown(@dummy) - @client.recv(1).should be_empty + @client.recv(1).to_s.should be_empty -> { @client.write('hello') }.should raise_error(Errno::EPIPE) end diff --git a/spec/ruby/library/socket/fixtures/classes.rb b/spec/ruby/library/socket/fixtures/classes.rb index 406bd7c7101e5e..786629d2eff048 100644 --- a/spec/ruby/library/socket/fixtures/classes.rb +++ b/spec/ruby/library/socket/fixtures/classes.rb @@ -113,7 +113,7 @@ def service(socket) begin data = socket.recv(1024) - return if data.empty? + return if data.nil? || data.empty? log "SpecTCPServer received: #{data.inspect}" return if data == "QUIT" diff --git a/test/socket/test_unix.rb b/test/socket/test_unix.rb index e206339db09596..9d9faa4387544e 100644 --- a/test/socket/test_unix.rb +++ b/test/socket/test_unix.rb @@ -420,9 +420,10 @@ def test_dgram_pair s1.recv_nonblock(10) fail rescue => e - assert(IO::EAGAINWaitReadable === e) - assert(IO::WaitReadable === e) + assert_kind_of(IO::EWOULDBLOCKWaitReadable, e) + assert_kind_of(IO::WaitReadable, e) end + s2.send("", 0) s2.send("haha", 0) s2.send("", 0) @@ -446,6 +447,69 @@ def test_dgram_pair s2.close if s2 end + def test_stream_pair + s1, s2 = UNIXSocket.pair(Socket::SOCK_STREAM) + begin + s1.recv_nonblock(10) + fail + rescue => e + assert_kind_of(IO::EWOULDBLOCKWaitReadable, e) + assert_kind_of(IO::WaitReadable, e) + end + + s2.send("", 0) + s2.send("haha", 0) + assert_equal("haha", s1.recv(10)) + assert_raise(IO::EWOULDBLOCKWaitReadable) { s1.recv_nonblock(10) } + + buf = "".dup + s2.send("BBBBBB", 0) + IO.select([s1]) + rv = s1.recv(100, 0, buf) + assert_equal buf.object_id, rv.object_id + assert_equal "BBBBBB", rv + + s2.close + assert_nil(s1.recv(10)) + rescue Errno::EPROTOTYPE => error + omit error.message + ensure + s1.close if s1 + s2.close if s2 + end + + def test_seqpacket_pair + s1, s2 = UNIXSocket.pair(Socket::SOCK_SEQPACKET) + begin + s1.recv_nonblock(10) + fail + rescue => e + assert_kind_of(IO::EWOULDBLOCKWaitReadable, e) + assert_kind_of(IO::WaitReadable, e) + end + + s2.send("", 0) + s2.send("haha", 0) + assert_equal(nil, s1.recv(10)) # no way to distinguish empty packet from EOF with SOCK_SEQPACKET + assert_equal("haha", s1.recv(10)) + assert_raise(IO::EWOULDBLOCKWaitReadable) { s1.recv_nonblock(10) } + + buf = "".dup + s2.send("BBBBBB", 0) + IO.select([s1]) + rv = s1.recv(100, 0, buf) + assert_equal buf.object_id, rv.object_id + assert_equal "BBBBBB", rv + + s2.close + assert_nil(s1.recv(10)) + rescue Errno::EPROTOTYPE, Errno::EPROTONOSUPPORT => error + omit error.message + ensure + s1.close if s1 + s2.close if s2 + end + def test_dgram_pair_sendrecvmsg_errno_set if /mswin|mingw/ =~ RUBY_PLATFORM omit("AF_UNIX + SOCK_DGRAM is not supported on windows") From 36a3899e9d464e243648694fa28398abbb4641ef Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Wed, 30 Aug 2023 10:17:02 +0100 Subject: [PATCH 122/208] [ruby/irb] Bump version to 1.8.0 (https://github.com/ruby/irb/pull/700) https://github.com/ruby/irb/commit/a061744ed3 --- lib/irb/version.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/irb/version.rb b/lib/irb/version.rb index 27103fe0bd6eb7..04a9ccfe292c12 100644 --- a/lib/irb/version.rb +++ b/lib/irb/version.rb @@ -5,7 +5,7 @@ # module IRB # :nodoc: - VERSION = "1.7.4" + VERSION = "1.8.0" @RELEASE_VERSION = VERSION - @LAST_UPDATE_DATE = "2023-07-15" + @LAST_UPDATE_DATE = "2023-08-30" end From 00fdb4e12e1933bdb110aeecc08099b4875c91ce Mon Sep 17 00:00:00 2001 From: git Date: Wed, 30 Aug 2023 09:18:22 +0000 Subject: [PATCH 123/208] Update default gems list at 36a3899e9d464e243648694fa28398 [ci skip] --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 12cd8dd5abf843..6d439762ac5416 100644 --- a/NEWS.md +++ b/NEWS.md @@ -85,7 +85,7 @@ The following default gems are updated. * erb 4.0.3 * fiddle 1.1.2 * fileutils 1.7.1 -* irb 1.7.4 +* irb 1.8.0 * nkf 0.1.3 * optparse 0.4.0.pre.1 * psych 5.1.0 From 74f4d2683e22fe1507ff4a95648f53be86d90d41 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sat, 26 Aug 2023 23:51:34 +0900 Subject: [PATCH 124/208] sync_default_gems.rb: Remove Java templates --- tool/sync_default_gems.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 4c3f5a6422af75..78d9f1b5a2daaa 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -417,6 +417,7 @@ def sync_default_gems(gem) cp_r("#{upstream}/config.yml", "yarp/") cp_r("#{upstream}/templates", "yarp/") + rm_rf("yarp/templates/java") rm("yarp/extconf.rb") mv("yarp_init.c", "yarp/") From c521b6f823154eeb6135881cec510fbff088b7ac Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Wed, 30 Aug 2023 10:47:59 -0400 Subject: [PATCH 125/208] [ruby/yarp] add tests for `Location#join` https://github.com/ruby/yarp/commit/b01711396f --- test/yarp/ruby_api_test.rb | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index f02026541de7de..73b8825df09ad1 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -31,6 +31,33 @@ def test_literal_value_method assert_equal 0.5ri, parse_expression("0.5ri").value end + def test_location_join + recv, args_node, _ = parse_expression("1234 + 567").child_nodes + arg = args_node.arguments[0] + + joined = recv.location.join(arg.location) + assert_equal 0, joined.start_offset + assert_equal 10, joined.length + + e = assert_raises RuntimeError do + arg.location.join(recv.location) + end + assert_equal "Incompatible locations", e.message + + other_recv, other_args_node, _ = parse_expression("1234 + 567").child_nodes + other_arg = other_args_node.arguments[0] + + e = assert_raises RuntimeError do + other_arg.location.join(recv.location) + end + assert_equal "Incompatible sources", e.message + + e = assert_raises RuntimeError do + recv.location.join(other_arg.location) + end + assert_equal "Incompatible sources", e.message + end + private def parse_expression(source) From e58fed128b737119e7e0f44292e8b0f0c8e691fb Mon Sep 17 00:00:00 2001 From: Maxime Chevalier-Boisvert Date: Wed, 30 Aug 2023 11:14:51 -0400 Subject: [PATCH 126/208] YJIT: shrink Context from 29 to 21 bytes by reducing space used by TempMapping (#8321) * YJIT: merge tempmapping and temp types into a single-byte encoding YJIT: refactor to shrink Context by 8 bytes * Add tests, fix bug in TempMapping::map_to_local() * Update yjit/src/core.rs Co-authored-by: Takashi Kokubun * Update yjit/src/core.rs Co-authored-by: Takashi Kokubun * Fewer transmutes where `as` would suffice. Also repr(u8) * Update yjit/src/core.rs Co-authored-by: Takashi Kokubun * Update yjit/src/core.rs Co-authored-by: Takashi Kokubun * Update yjit/src/core.rs Co-authored-by: Takashi Kokubun --------- Co-authored-by: Takashi Kokubun Co-authored-by: Alan Wu --- yjit/src/codegen.rs | 32 +++--- yjit/src/core.rs | 236 +++++++++++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 107 deletions(-) diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 354b8e5fd213af..c506b4360c10a6 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -342,12 +342,14 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { // Verify stack operand types let top_idx = cmp::min(ctx.get_stack_size(), MAX_TEMP_TYPES as u8); for i in 0..top_idx { - let (learned_mapping, learned_type) = ctx.get_opnd_mapping(StackOpnd(i)); + let learned_mapping = ctx.get_opnd_mapping(StackOpnd(i)); + let learned_type = ctx.get_opnd_type(StackOpnd(i)); + let stack_val = jit.peek_at_stack(ctx, i as isize); let val_type = Type::from(stack_val); - match learned_mapping { - TempMapping::MapToSelf => { + match learned_mapping.get_kind() { + TempMappingKind::MapToSelf => { if self_val != stack_val { panic!( "verify_ctx: stack value was mapped to self, but values did not match!\n stack: {}\n self: {}", @@ -356,8 +358,8 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { ); } } - TempMapping::MapToLocal(local_idx) => { - let local_idx: u8 = local_idx.into(); + TempMappingKind::MapToLocal => { + let local_idx: u8 = learned_mapping.get_local_idx().into(); let local_val = jit.peek_at_local(local_idx.into()); if local_val != stack_val { panic!( @@ -368,7 +370,7 @@ fn verify_ctx(jit: &JITState, ctx: &Context) { ); } } - TempMapping::MapToStack => {} + TempMappingKind::MapToStack => {} } // If the actual type differs from the learned type @@ -1009,9 +1011,9 @@ fn gen_dup( _ocb: &mut OutlinedCb, ) -> Option { let dup_val = asm.stack_opnd(0); - let (mapping, tmp_type) = asm.ctx.get_opnd_mapping(dup_val.into()); + let mapping = asm.ctx.get_opnd_mapping(dup_val.into()); - let loc0 = asm.stack_push_mapping((mapping, tmp_type)); + let loc0 = asm.stack_push_mapping(mapping); asm.mov(loc0, dup_val); Some(KeepCompiling) @@ -2327,7 +2329,7 @@ fn gen_setinstancevariable( return None; } - let (_, stack_type) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let stack_type = asm.ctx.get_opnd_type(StackOpnd(0)); // Check if the comptime class uses a custom allocator let custom_allocator = unsafe { rb_get_alloc_func(comptime_val_klass) }; @@ -8968,8 +8970,8 @@ mod tests { let status = gen_swap(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); - let (_, tmp_type_next) = asm.ctx.get_opnd_mapping(StackOpnd(1)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); + let tmp_type_next = asm.ctx.get_opnd_type(StackOpnd(1)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::Fixnum); @@ -8981,7 +8983,7 @@ mod tests { let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); let status = gen_putnil(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::Nil); @@ -9000,7 +9002,7 @@ mod tests { let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::True); @@ -9020,7 +9022,7 @@ mod tests { let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); assert_eq!(status, Some(KeepCompiling)); assert_eq!(tmp_type_top, Type::Fixnum); @@ -9034,7 +9036,7 @@ mod tests { jit.opcode = YARVINSN_putobject_INT2FIX_0_.as_usize(); let status = gen_putobject_int2fix(&mut jit, &mut asm, &mut ocb); - let (_, tmp_type_top) = asm.ctx.get_opnd_mapping(StackOpnd(0)); + let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); // Right now we're not testing the generated machine code to make sure a literal 1 or 0 was pushed. I've checked locally. assert_eq!(status, Some(KeepCompiling)); diff --git a/yjit/src/core.rs b/yjit/src/core.rs index c0577f08e8266d..805b7dd68599ce 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -18,13 +18,14 @@ use std::cell::*; use std::collections::HashSet; use std::fmt; use std::mem; +use std::mem::transmute; use std::ops::Range; use std::rc::Rc; use mem::MaybeUninit; use std::ptr; use ptr::NonNull; use YARVOpnd::*; -use TempMapping::*; +use TempMappingKind::*; use crate::invariants::*; // Maximum number of temp value types we keep track of @@ -39,6 +40,7 @@ pub type IseqIdx = u16; // Represent the type of a value (local/stack/self) in YJIT #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)] +#[repr(u8)] pub enum Type { Unknown, UnknownImm, @@ -286,63 +288,92 @@ pub enum TypeDiff { Incompatible, } -// Potential mapping of a value on the temporary stack to -// self, a local variable or constant so that we can track its type #[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)] -pub enum TempMapping { - MapToStack, // Normal stack value - MapToSelf, // Temp maps to the self operand - MapToLocal(LocalIndex), // Temp maps to a local variable with index - //ConstMapping, // Small constant (0, 1, 2, Qnil, Qfalse, Qtrue) +#[repr(u8)] +pub enum TempMappingKind +{ + MapToStack = 0, + MapToSelf = 1, + MapToLocal = 2, } -// Index used by MapToLocal. Using this instead of u8 makes TempMapping 1 byte. +// Potential mapping of a value on the temporary stack to +// self, a local variable or constant so that we can track its type +// +// The highest two bits represent TempMappingKind, and the rest of +// the bits are used differently across different kinds. +// * MapToStack: The lowest 5 bits are used for mapping Type. +// * MapToSelf: The remaining bits are not used; the type is stored in self_type. +// * MapToLocal: The lowest 3 bits store the index of a local variable. #[derive(Copy, Clone, Eq, Hash, PartialEq, Debug)] -pub enum LocalIndex { - Local0, - Local1, - Local2, - Local3, - Local4, - Local5, - Local6, - Local7, -} +pub struct TempMapping(u8); -impl From for u8 { - fn from(idx: LocalIndex) -> Self { - match idx { - LocalIndex::Local0 => 0, - LocalIndex::Local1 => 1, - LocalIndex::Local2 => 2, - LocalIndex::Local3 => 3, - LocalIndex::Local4 => 4, - LocalIndex::Local5 => 5, - LocalIndex::Local6 => 6, - LocalIndex::Local7 => 7, - } +impl TempMapping { + pub fn map_to_stack(t: Type) -> TempMapping + { + let kind_bits = TempMappingKind::MapToStack as u8; + let type_bits = t as u8; + assert!(type_bits <= 0b11111); + let bits = (kind_bits << 6) | (type_bits & 0b11111); + TempMapping(bits) } -} -impl From for LocalIndex { - fn from(idx: u8) -> Self { - match idx { - 0 => LocalIndex::Local0, - 1 => LocalIndex::Local1, - 2 => LocalIndex::Local2, - 3 => LocalIndex::Local3, - 4 => LocalIndex::Local4, - 5 => LocalIndex::Local5, - 6 => LocalIndex::Local6, - 7 => LocalIndex::Local7, - _ => unreachable!("{idx} was larger than {MAX_LOCAL_TYPES}"), + pub fn map_to_self() -> TempMapping + { + let kind_bits = TempMappingKind::MapToSelf as u8; + let bits = kind_bits << 6; + TempMapping(bits) + } + + pub fn map_to_local(local_idx: u8) -> TempMapping + { + let kind_bits = TempMappingKind::MapToLocal as u8; + assert!(local_idx <= 0b111); + let bits = (kind_bits << 6) | (local_idx & 0b111); + TempMapping(bits) + } + + pub fn without_type(&self) -> TempMapping + { + if self.get_kind() != TempMappingKind::MapToStack { + return *self; } + + TempMapping::map_to_stack(Type::Unknown) + } + + pub fn get_kind(&self) -> TempMappingKind + { + // Take the two highest bits + let TempMapping(bits) = self; + let kind_bits = bits >> 6; + assert!(kind_bits <= 2); + unsafe { transmute::(kind_bits) } + } + + pub fn get_type(&self) -> Type + { + assert!(self.get_kind() == TempMappingKind::MapToStack); + + // Take the 5 lowest bits + let TempMapping(bits) = self; + let type_bits = bits & 0b11111; + unsafe { transmute::(type_bits) } + } + + pub fn get_local_idx(&self) -> u8 + { + assert!(self.get_kind() == TempMappingKind::MapToLocal); + + // Take the 3 lowest bits + let TempMapping(bits) = self; + bits & 0b111 } } impl Default for TempMapping { fn default() -> Self { - MapToStack + TempMapping::map_to_stack(Type::Unknown) } } @@ -425,9 +456,6 @@ pub struct Context { // Local variable types we keep track of local_types: [Type; MAX_LOCAL_TYPES], - // Temporary variable types we keep track of - temp_types: [Type; MAX_TEMP_TYPES], - // Type we track for self self_type: Type, @@ -1647,10 +1675,11 @@ impl Context { let mapping = self.temp_mapping[stack_idx]; - match mapping { + match mapping.get_kind() { MapToSelf => self.self_type, - MapToStack => self.temp_types[(self.stack_size - 1 - idx) as usize], - MapToLocal(idx) => { + MapToStack => mapping.get_type(), + MapToLocal => { + let idx = mapping.get_local_idx(); assert!((idx as usize) < MAX_LOCAL_TYPES); return self.local_types[idx as usize]; } @@ -1687,11 +1716,15 @@ impl Context { let mapping = self.temp_mapping[stack_idx]; - match mapping { + match mapping.get_kind() { MapToSelf => self.self_type.upgrade(opnd_type), - MapToStack => self.temp_types[stack_idx].upgrade(opnd_type), - MapToLocal(idx) => { - let idx = idx as usize; + MapToStack => { + let mut temp_type = mapping.get_type(); + temp_type.upgrade(opnd_type); + self.temp_mapping[stack_idx] = TempMapping::map_to_stack(temp_type); + } + MapToLocal => { + let idx = mapping.get_local_idx() as usize; assert!(idx < MAX_LOCAL_TYPES); self.local_types[idx].upgrade(opnd_type); } @@ -1705,29 +1738,29 @@ impl Context { This is can be used with stack_push_mapping or set_opnd_mapping to copy a stack value's type while maintaining the mapping. */ - pub fn get_opnd_mapping(&self, opnd: YARVOpnd) -> (TempMapping, Type) { + pub fn get_opnd_mapping(&self, opnd: YARVOpnd) -> TempMapping { let opnd_type = self.get_opnd_type(opnd); match opnd { - SelfOpnd => (MapToSelf, opnd_type), + SelfOpnd => TempMapping::map_to_self(), StackOpnd(idx) => { assert!(idx < self.stack_size); let stack_idx = (self.stack_size - 1 - idx) as usize; if stack_idx < MAX_TEMP_TYPES { - (self.temp_mapping[stack_idx], opnd_type) + self.temp_mapping[stack_idx] } else { // We can't know the source of this stack operand, so we assume it is // a stack-only temporary. type will be UNKNOWN assert!(opnd_type == Type::Unknown); - (MapToStack, opnd_type) + TempMapping::map_to_stack(opnd_type) } } } } /// Overwrite both the type and mapping of a stack operand. - pub fn set_opnd_mapping(&mut self, opnd: YARVOpnd, (mapping, opnd_type): (TempMapping, Type)) { + pub fn set_opnd_mapping(&mut self, opnd: YARVOpnd, mapping: TempMapping) { match opnd { SelfOpnd => unreachable!("self always maps to self"), StackOpnd(idx) => { @@ -1745,9 +1778,6 @@ impl Context { } self.temp_mapping[stack_idx] = mapping; - - // Only used when mapping == MAP_STACK - self.temp_types[stack_idx] = opnd_type; } } } @@ -1766,16 +1796,16 @@ impl Context { } // If any values on the stack map to this local we must detach them - for (i, mapping) in ctx.temp_mapping.iter_mut().enumerate() { - *mapping = match *mapping { - MapToStack => MapToStack, - MapToSelf => MapToSelf, - MapToLocal(idx) => { + for mapping in ctx.temp_mapping.iter_mut() { + *mapping = match mapping.get_kind() { + MapToStack => *mapping, + MapToSelf => *mapping, + MapToLocal => { + let idx = mapping.get_local_idx(); if idx as usize == local_idx { - ctx.temp_types[i] = ctx.local_types[idx as usize]; - MapToStack + TempMapping::map_to_stack(ctx.local_types[idx as usize]) } else { - MapToLocal(idx) + TempMapping::map_to_local(idx) } } } @@ -1789,14 +1819,10 @@ impl Context { pub fn clear_local_types(&mut self) { // When clearing local types we must detach any stack mappings to those // locals. Even if local values may have changed, stack values will not. - for (i, mapping) in self.temp_mapping.iter_mut().enumerate() { - *mapping = match *mapping { - MapToStack => MapToStack, - MapToSelf => MapToSelf, - MapToLocal(idx) => { - self.temp_types[i] = self.local_types[idx as usize]; - MapToStack - } + for mapping in self.temp_mapping.iter_mut() { + if mapping.get_kind() == MapToLocal { + let idx = mapping.get_local_idx(); + *mapping = TempMapping::map_to_stack(self.local_types[idx as usize]); } } @@ -1853,12 +1879,12 @@ impl Context { // For each value on the temp stack for i in 0..src.stack_size { - let (src_mapping, src_type) = src.get_opnd_mapping(StackOpnd(i)); - let (dst_mapping, dst_type) = dst.get_opnd_mapping(StackOpnd(i)); + let src_mapping = src.get_opnd_mapping(StackOpnd(i)); + let dst_mapping = dst.get_opnd_mapping(StackOpnd(i)); // If the two mappings aren't the same if src_mapping != dst_mapping { - if dst_mapping == MapToStack { + if dst_mapping.get_kind() == MapToStack { // We can safely drop information about the source of the temp // stack operand. diff += 1; @@ -1867,6 +1893,9 @@ impl Context { } } + let src_type = src.get_opnd_type(StackOpnd(i)); + let dst_type = dst.get_opnd_type(StackOpnd(i)); + diff += match src_type.diff(dst_type) { TypeDiff::Compatible(diff) => diff, TypeDiff::Incompatible => return TypeDiff::Incompatible, @@ -1896,10 +1925,10 @@ impl Context { impl Assembler { /// Push one new value on the temp stack with an explicit mapping /// Return a pointer to the new stack top - pub fn stack_push_mapping(&mut self, (mapping, temp_type): (TempMapping, Type)) -> Opnd { + pub fn stack_push_mapping(&mut self, mapping: TempMapping) -> Opnd { // If type propagation is disabled, store no types if get_option!(no_type_prop) { - return self.stack_push_mapping((mapping, Type::Unknown)); + return self.stack_push_mapping(mapping.without_type()); } let stack_size: usize = self.ctx.stack_size.into(); @@ -1907,9 +1936,9 @@ impl Assembler { // Keep track of the type and mapping of the value if stack_size < MAX_TEMP_TYPES { self.ctx.temp_mapping[stack_size] = mapping; - self.ctx.temp_types[stack_size] = temp_type; - if let MapToLocal(idx) = mapping { + if mapping.get_kind() == MapToLocal { + let idx = mapping.get_local_idx(); assert!((idx as usize) < MAX_LOCAL_TYPES); } } @@ -1928,12 +1957,12 @@ impl Assembler { /// Push one new value on the temp stack /// Return a pointer to the new stack top pub fn stack_push(&mut self, val_type: Type) -> Opnd { - return self.stack_push_mapping((MapToStack, val_type)); + return self.stack_push_mapping(TempMapping::map_to_stack(val_type)); } /// Push the self value on the stack pub fn stack_push_self(&mut self) -> Opnd { - return self.stack_push_mapping((MapToSelf, Type::Unknown)); + return self.stack_push_mapping(TempMapping::map_to_self()); } /// Push a local variable on the stack @@ -1942,7 +1971,7 @@ impl Assembler { return self.stack_push(Type::Unknown); } - return self.stack_push_mapping((MapToLocal((local_idx as u8).into()), Type::Unknown)); + return self.stack_push_mapping(TempMapping::map_to_local((local_idx as u8).into())); } // Pop N values off the stack @@ -1957,8 +1986,7 @@ impl Assembler { let idx: usize = (self.ctx.stack_size as usize) - i - 1; if idx < MAX_TEMP_TYPES { - self.ctx.temp_types[idx] = Type::Unknown; - self.ctx.temp_mapping[idx] = MapToStack; + self.ctx.temp_mapping[idx] = TempMapping::map_to_stack(Type::Unknown); } } @@ -1976,7 +2004,6 @@ impl Assembler { for i in method_name_index..(self.ctx.stack_size - 1) as usize { if i + 1 < MAX_TEMP_TYPES { - self.ctx.temp_types[i] = self.ctx.temp_types[i + 1]; self.ctx.temp_mapping[i] = self.ctx.temp_mapping[i + 1]; } } @@ -3153,6 +3180,31 @@ impl RefUnchecked for Cell { mod tests { use crate::core::*; + #[test] + fn tempmapping_size() { + assert_eq!(mem::size_of::(), 1); + } + + #[test] + fn tempmapping() { + let t = TempMapping::map_to_stack(Type::Unknown); + assert_eq!(t.get_kind(), MapToStack); + assert_eq!(t.get_type(), Type::Unknown); + + let t = TempMapping::map_to_stack(Type::TString); + assert_eq!(t.get_kind(), MapToStack); + assert_eq!(t.get_type(), Type::TString); + + let t = TempMapping::map_to_local(7); + assert_eq!(t.get_kind(), MapToLocal); + assert_eq!(t.get_local_idx(), 7); + } + + #[test] + fn context_size() { + assert_eq!(mem::size_of::(), 21); + } + #[test] fn types() { // Valid src => dst From c74039eec71b16c75965a36451ca14b18a81b278 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 31 Aug 2023 00:38:27 +0900 Subject: [PATCH 127/208] use assert_raise --- test/yarp/ruby_api_test.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 73b8825df09ad1..b74da653f42d09 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -39,7 +39,7 @@ def test_location_join assert_equal 0, joined.start_offset assert_equal 10, joined.length - e = assert_raises RuntimeError do + e = assert_rais RuntimeError do arg.location.join(recv.location) end assert_equal "Incompatible locations", e.message @@ -47,12 +47,12 @@ def test_location_join other_recv, other_args_node, _ = parse_expression("1234 + 567").child_nodes other_arg = other_args_node.arguments[0] - e = assert_raises RuntimeError do + e = assert_rais RuntimeError do other_arg.location.join(recv.location) end assert_equal "Incompatible sources", e.message - e = assert_raises RuntimeError do + e = assert_rais RuntimeError do recv.location.join(other_arg.location) end assert_equal "Incompatible sources", e.message From 9d8d2b81d21750774c60fd15b535a75ce61f04e7 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Wed, 30 Aug 2023 11:42:26 -0400 Subject: [PATCH 128/208] [ruby/yarp] use a more idiomatic form of `assert_raises` https://github.com/ruby/yarp/commit/687213d2e3 --- test/yarp/ruby_api_test.rb | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index b74da653f42d09..4e952b8b25a25c 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -39,23 +39,32 @@ def test_location_join assert_equal 0, joined.start_offset assert_equal 10, joined.length +<<<<<<< HEAD e = assert_rais RuntimeError do +======= + assert_raises RuntimeError, "Incompatible locations" do +>>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) arg.location.join(recv.location) end - assert_equal "Incompatible locations", e.message other_recv, other_args_node, _ = parse_expression("1234 + 567").child_nodes other_arg = other_args_node.arguments[0] +<<<<<<< HEAD e = assert_rais RuntimeError do +======= + assert_raises RuntimeError, "Incompatible sources" do +>>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) other_arg.location.join(recv.location) end - assert_equal "Incompatible sources", e.message +<<<<<<< HEAD e = assert_rais RuntimeError do +======= + assert_raises RuntimeError, "Incompatible sources" do +>>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) recv.location.join(other_arg.location) end - assert_equal "Incompatible sources", e.message end private From c5c0a3cf1772c9345373ecc4d7558f216b4ceb62 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 12:08:00 -0400 Subject: [PATCH 129/208] [ruby/yarp] Use assert_raise https://github.com/ruby/yarp/commit/b85e01d77d --- test/yarp/ruby_api_test.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 4e952b8b25a25c..79a5e2ab0a6c8c 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -39,17 +39,22 @@ def test_location_join assert_equal 0, joined.start_offset assert_equal 10, joined.length +<<<<<<< HEAD <<<<<<< HEAD e = assert_rais RuntimeError do ======= assert_raises RuntimeError, "Incompatible locations" do >>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) +======= + assert_raise RuntimeError, "Incompatible locations" do +>>>>>>> b85e01d77d2 (Use assert_raise) arg.location.join(recv.location) end other_recv, other_args_node, _ = parse_expression("1234 + 567").child_nodes other_arg = other_args_node.arguments[0] +<<<<<<< HEAD <<<<<<< HEAD e = assert_rais RuntimeError do ======= @@ -63,6 +68,13 @@ def test_location_join ======= assert_raises RuntimeError, "Incompatible sources" do >>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) +======= + assert_raise RuntimeError, "Incompatible sources" do + other_arg.location.join(recv.location) + end + + assert_raise RuntimeError, "Incompatible sources" do +>>>>>>> b85e01d77d2 (Use assert_raise) recv.location.join(other_arg.location) end end From 06f54c9a8c3058883e4c42c5b0d3c6e498d2f3ad Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 12:43:56 -0400 Subject: [PATCH 130/208] Fix merge error on ruby_api_test.rb --- test/yarp/ruby_api_test.rb | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 79a5e2ab0a6c8c..236407ccac413a 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -39,42 +39,18 @@ def test_location_join assert_equal 0, joined.start_offset assert_equal 10, joined.length -<<<<<<< HEAD -<<<<<<< HEAD - e = assert_rais RuntimeError do -======= - assert_raises RuntimeError, "Incompatible locations" do ->>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) -======= assert_raise RuntimeError, "Incompatible locations" do ->>>>>>> b85e01d77d2 (Use assert_raise) arg.location.join(recv.location) end other_recv, other_args_node, _ = parse_expression("1234 + 567").child_nodes other_arg = other_args_node.arguments[0] -<<<<<<< HEAD -<<<<<<< HEAD - e = assert_rais RuntimeError do -======= - assert_raises RuntimeError, "Incompatible sources" do ->>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) - other_arg.location.join(recv.location) - end - -<<<<<<< HEAD - e = assert_rais RuntimeError do -======= - assert_raises RuntimeError, "Incompatible sources" do ->>>>>>> 687213d2e38 (use a more idiomatic form of `assert_raises`) -======= assert_raise RuntimeError, "Incompatible sources" do other_arg.location.join(recv.location) end assert_raise RuntimeError, "Incompatible sources" do ->>>>>>> b85e01d77d2 (Use assert_raise) recv.location.join(other_arg.location) end end From c4998bc3f2f1978fbbe3f192f3d692523ee266d3 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Aug 2023 08:38:33 -0400 Subject: [PATCH 131/208] [ruby/yarp] Desugar ||= more accurately Class variables, global variables, constants, and constant paths should actually desugar to `defined?` instead of just reading the value. https://github.com/ruby/yarp/commit/551a59b876 --- lib/yarp/desugar_visitor.rb | 140 +++++++++++++++--------------- test/yarp/desugar_visitor_test.rb | 8 +- 2 files changed, 74 insertions(+), 74 deletions(-) diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb index 83a9bb5336a300..8fe488368e9079 100644 --- a/lib/yarp/desugar_visitor.rb +++ b/lib/yarp/desugar_visitor.rb @@ -8,26 +8,16 @@ class DesugarVisitor < MutationVisitor # # @@foo && @@foo = bar def visit_class_variable_and_write_node(node) - AndNode.new( - ClassVariableReadNode.new(node.name_loc), - ClassVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode) end # @@foo ||= bar # # becomes # - # @@foo || @@foo = bar + # defined?(@@foo) ? @@foo : @@foo = bar def visit_class_variable_or_write_node(node) - OrNode.new( - ClassVariableReadNode.new(node.name_loc), - ClassVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode) end # @@foo += bar @@ -36,7 +26,7 @@ def visit_class_variable_or_write_node(node) # # @@foo = @@foo + bar def visit_class_variable_operator_write_node(node) - desugar_operator_write_node(node, ClassVariableWriteNode, ClassVariableReadNode) + desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode) end # Foo &&= bar @@ -45,12 +35,7 @@ def visit_class_variable_operator_write_node(node) # # Foo && Foo = bar def visit_constant_and_write_node(node) - AndNode.new( - ConstantReadNode.new(node.name_loc), - ConstantWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_and_write_node(node, ConstantReadNode, ConstantWriteNode) end # Foo ||= bar @@ -59,12 +44,7 @@ def visit_constant_and_write_node(node) # # Foo || Foo = bar def visit_constant_or_write_node(node) - OrNode.new( - ConstantReadNode.new(node.name_loc), - ConstantWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_or_write_defined_node(node, ConstantReadNode, ConstantWriteNode) end # Foo += bar @@ -73,7 +53,7 @@ def visit_constant_or_write_node(node) # # Foo = Foo + bar def visit_constant_operator_write_node(node) - desugar_operator_write_node(node, ConstantWriteNode, ConstantReadNode) + desugar_operator_write_node(node, ConstantReadNode, ConstantWriteNode) end # Foo::Bar &&= baz @@ -96,9 +76,19 @@ def visit_constant_path_and_write_node(node) # # Foo::Bar || Foo::Bar = baz def visit_constant_path_or_write_node(node) - OrNode.new( - node.target, - ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location), + IfNode.new( + node.operator_loc, + DefinedNode.new(nil, node.target, nil, node.operator_loc, node.target.location), + StatementsNode.new([node.target], node.location), + ElseNode.new( + node.operator_loc, + StatementsNode.new( + [ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location)], + node.location + ), + node.operator_loc, + node.location + ), node.operator_loc, node.location ) @@ -135,12 +125,7 @@ def visit_constant_path_operator_write_node(node) # # $foo && $foo = bar def visit_global_variable_and_write_node(node) - AndNode.new( - GlobalVariableReadNode.new(node.name_loc), - GlobalVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_and_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) end # $foo ||= bar @@ -149,12 +134,7 @@ def visit_global_variable_and_write_node(node) # # $foo || $foo = bar def visit_global_variable_or_write_node(node) - OrNode.new( - GlobalVariableReadNode.new(node.name_loc), - GlobalVariableWriteNode.new(node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_or_write_defined_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) end # $foo += bar @@ -163,7 +143,7 @@ def visit_global_variable_or_write_node(node) # # $foo = $foo + bar def visit_global_variable_operator_write_node(node) - desugar_operator_write_node(node, GlobalVariableWriteNode, GlobalVariableReadNode) + desugar_operator_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) end # @foo &&= bar @@ -172,12 +152,7 @@ def visit_global_variable_operator_write_node(node) # # @foo && @foo = bar def visit_instance_variable_and_write_node(node) - AndNode.new( - InstanceVariableReadNode.new(node.name, node.name_loc), - InstanceVariableWriteNode.new(node.name, node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_and_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, arguments: [node.name]) end # @foo ||= bar @@ -186,12 +161,7 @@ def visit_instance_variable_and_write_node(node) # # @foo || @foo = bar def visit_instance_variable_or_write_node(node) - OrNode.new( - InstanceVariableReadNode.new(node.name, node.name_loc), - InstanceVariableWriteNode.new(node.name, node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_or_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, arguments: [node.name]) end # @foo += bar @@ -200,7 +170,7 @@ def visit_instance_variable_or_write_node(node) # # @foo = @foo + bar def visit_instance_variable_operator_write_node(node) - desugar_operator_write_node(node, InstanceVariableWriteNode, InstanceVariableReadNode, arguments: [node.name]) + desugar_operator_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, arguments: [node.name]) end # foo &&= bar @@ -209,12 +179,7 @@ def visit_instance_variable_operator_write_node(node) # # foo && foo = bar def visit_local_variable_and_write_node(node) - AndNode.new( - LocalVariableReadNode.new(node.name, node.depth, node.name_loc), - LocalVariableWriteNode.new(node.name, node.depth, node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_and_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, arguments: [node.name, node.depth]) end # foo ||= bar @@ -223,12 +188,7 @@ def visit_local_variable_and_write_node(node) # # foo || foo = bar def visit_local_variable_or_write_node(node) - OrNode.new( - LocalVariableReadNode.new(node.name, node.depth, node.name_loc), - LocalVariableWriteNode.new(node.name, node.depth, node.name_loc, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) + desugar_or_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, arguments: [node.name, node.depth]) end # foo += bar @@ -237,13 +197,23 @@ def visit_local_variable_or_write_node(node) # # foo = foo + bar def visit_local_variable_operator_write_node(node) - desugar_operator_write_node(node, LocalVariableWriteNode, LocalVariableReadNode, arguments: [node.name, node.depth]) + desugar_operator_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, arguments: [node.name, node.depth]) end private + # Desugar `x &&= y` to `x && x = y` + def desugar_and_write_node(node, read_class, write_class, arguments: []) + AndNode.new( + read_class.new(*arguments, node.name_loc), + write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + # Desugar `x += y` to `x = x + y` - def desugar_operator_write_node(node, write_class, read_class, arguments: []) + def desugar_operator_write_node(node, read_class, write_class, arguments: []) write_class.new( *arguments, node.name_loc, @@ -263,5 +233,35 @@ def desugar_operator_write_node(node, write_class, read_class, arguments: []) node.location ) end + + # Desugar `x ||= y` to `x || x = y` + def desugar_or_write_node(node, read_class, write_class, arguments: []) + OrNode.new( + read_class.new(*arguments, node.name_loc), + write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + + # Don't desugar `x ||= y` to `defined?(x) ? x : x = y` + def desugar_or_write_defined_node(node, read_class, write_class) + IfNode.new( + node.operator_loc, + DefinedNode.new(nil, read_class.new(node.name_loc), nil, node.operator_loc, node.name_loc), + StatementsNode.new([read_class.new(node.name_loc)], node.location), + ElseNode.new( + node.operator_loc, + StatementsNode.new( + [write_class.new(node.name_loc, node.value, node.operator_loc, node.location)], + node.location + ), + node.operator_loc, + node.location + ), + node.operator_loc, + node.location + ) + end end end diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index a893422a7c3fa6..de635966a1dfb3 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -14,10 +14,10 @@ def test_and_write end def test_or_write - assert_desugars("(OrNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo ||= bar") - assert_desugars("(OrNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar ||= baz") - assert_desugars("(OrNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo ||= bar") - assert_desugars("(OrNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo ||= bar") + assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") + assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") + assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") + assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") From 1edb03788d27cf214365b849778a72c8c52c158f Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Wed, 30 Aug 2023 10:18:09 -0700 Subject: [PATCH 132/208] Stop using -v for rjit test-all It outputs way too many lines. It's hard to download the output from GitHub Actions. --- .github/workflows/rjit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rjit.yml b/.github/workflows/rjit.yml index eea20d6be95aa0..66a6805abce2b1 100644 --- a/.github/workflows/rjit.yml +++ b/.github/workflows/rjit.yml @@ -95,7 +95,7 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-v --tty=no' + RUBY_TESTOPTS: '-q --tty=no' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-spec From f652c05a59506dfe931f23db4965f7e40033de5e Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 10:40:00 -0400 Subject: [PATCH 133/208] [ruby/yarp] debug: ensure valgrind will work when calling YARP.dump https://github.com/ruby/yarp/commit/deba3420d5 --- yarp/extension.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/yarp/extension.c b/yarp/extension.c index b46a1f8226a2d8..f959dba31ac234 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -83,7 +83,21 @@ dump(int argc, VALUE *argv, VALUE self) { yp_string_t input; input_load_string(&input, string); - return dump_input(&input, check_string(filepath)); + +#ifdef YARP_DEBUG_MODE_BUILD + size_t length = yp_string_length(&input); + char* dup = malloc(length); + memcpy(dup, yp_string_source(&input), length); + yp_string_constant_init(&input, dup, length); +#endif + + VALUE value = dump_input(&input, check_string(filepath)); + +#ifdef YARP_DEBUG_MODE_BUILD + free(dup); +#endif + + return value; } // Dump the AST corresponding to the given file to a string. From 440cdceffb7efef0aa93ad519b056a25d610f52d Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 16:36:18 -0400 Subject: [PATCH 134/208] [ruby/yarp] test: new test file for capturing interesting fuzzer snippets Note that we call `YARP.dump` for these fuzzer tests to better match the fuzz.parse harness, which also serializes. https://github.com/ruby/yarp/commit/032ad047e9 --- test/yarp/fuzzer_test.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/yarp/fuzzer_test.rb diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb new file mode 100644 index 00000000000000..3aed73ecd54f13 --- /dev/null +++ b/test/yarp/fuzzer_test.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +# These tests are simply to exercise snippets found by the fuzzer that caused invalid memory access. +class FuzzerTest < Test::Unit::TestCase + class << self + def snippet(name, source) + test "fuzzer #{name}" do + YARP.dump(source) + end + end + end +end From 3da139d28428d9889f6b6680400242f400e3175e Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 28 Aug 2023 18:01:47 -0400 Subject: [PATCH 135/208] [ruby/yarp] fix: "$" at the end of a file Previously this resulted in invalid memory access as well as a cascading failed assertion: src/enc/yp_unicode.c:2224: yp_utf_8_codepoint: Assertion `n >= 1' failed. Found by the fuzzer. https://github.com/ruby/yarp/commit/a34c534440 --- test/yarp/errors_test.rb | 6 ++++++ test/yarp/fuzzer_test.rb | 2 ++ yarp/yarp.c | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index 074dd7129e6e68..e461afbbd26fc1 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -1106,6 +1106,12 @@ def test_duplicated_parameter_names assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] end + def test_unterminated_global_variable + assert_errors expression("$"), "$", [ + ["Invalid global variable.", 0..1] + ] + end + private def assert_errors(expected, source, errors) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 3aed73ecd54f13..dd8a25864c550c 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -11,4 +11,6 @@ def snippet(name, source) end end end + + snippet "incomplete global variable", "$" end diff --git a/yarp/yarp.c b/yarp/yarp.c index 089de42b9a83f3..4b8ff07a985479 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -5124,6 +5124,11 @@ lex_numeric(yp_parser_t *parser) { static yp_token_type_t lex_global_variable(yp_parser_t *parser) { + if (parser->current.end >= parser->end) { + yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid global variable."); + return YP_TOKEN_GLOBAL_VARIABLE; + } + switch (*parser->current.end) { case '~': // $~: match-data case '*': // $*: argv From 476f38d62dd24caf80c586b642a25ede66b13fab Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 15:12:06 -0400 Subject: [PATCH 136/208] [ruby/yarp] fix: ":" at the end of a file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/c781c9fcd2 --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index dd8a25864c550c..6962e6be762220 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -13,4 +13,5 @@ def snippet(name, source) end snippet "incomplete global variable", "$" + snippet "incomplete symbol", ":" end diff --git a/yarp/yarp.c b/yarp/yarp.c index 4b8ff07a985479..15511ebc9e1143 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -6581,7 +6581,7 @@ parser_lex(yp_parser_t *parser) { LEX(YP_TOKEN_COLON_COLON); } - if (lex_state_end_p(parser) || yp_char_is_whitespace(*parser->current.end) || peek(parser) == '#') { + if (lex_state_end_p(parser) || yp_char_is_whitespace(peek(parser)) || peek(parser) == '#') { lex_state_set(parser, YP_LEX_STATE_BEG); LEX(YP_TOKEN_COLON); } From 6f8126faebeddf8a93a7c0041c096bf584efcefb Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 15:38:48 -0400 Subject: [PATCH 137/208] [ruby/yarp] fix: string escape char "\" at the end of a file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/178862e2ca --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 6962e6be762220..e2aa9dab617c19 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -14,4 +14,5 @@ def snippet(name, source) snippet "incomplete global variable", "$" snippet "incomplete symbol", ":" + snippet "incomplete escaped string", '"\\' end diff --git a/yarp/yarp.c b/yarp/yarp.c index 15511ebc9e1143..91c814af9a400d 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -7224,6 +7224,12 @@ parser_lex(yp_parser_t *parser) { breakpoint = yp_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1)); break; case '\\': { + // Check that we're not at the end of the file. + if (breakpoint + 1 >= parser->end) { + breakpoint = NULL; + break; + } + // If we hit escapes, then we need to treat the next token // literally. In this case we'll skip past the next character and // find the next breakpoint. From bd0268372e09eb45d088c44a534a5302bdca9796 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 17:01:09 -0400 Subject: [PATCH 138/208] [ruby/yarp] fix: trailing comment at end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/a1c9404906 --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index e2aa9dab617c19..8da1a2dc76bced 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -15,4 +15,5 @@ def snippet(name, source) snippet "incomplete global variable", "$" snippet "incomplete symbol", ":" snippet "incomplete escaped string", '"\\' + snippet "trailing comment", "1\n#\n" end diff --git a/yarp/yarp.c b/yarp/yarp.c index 91c814af9a400d..ce991003f18178 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -5908,7 +5908,7 @@ parser_lex(yp_parser_t *parser) { // Here we look for a "." or "&." following a "\n". const char *following = next_newline(next_content, parser->end - next_content); - while (following && (following < parser->end)) { + while (following && (following + 1 < parser->end)) { following++; following += yp_strspn_inline_whitespace(following, parser->end - following); From c83552a596a34808651efca29a4f480bb5c579c6 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 22:30:12 -0400 Subject: [PATCH 139/208] [ruby/yarp] fix: trailing asterisk at end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/c86b4907b4 --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 8da1a2dc76bced..97f128f08b09be 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -16,4 +16,5 @@ def snippet(name, source) snippet "incomplete symbol", ":" snippet "incomplete escaped string", '"\\' snippet "trailing comment", "1\n#\n" + snippet "trailing asterisk", "a *" end diff --git a/yarp/yarp.c b/yarp/yarp.c index ce991003f18178..0d6b55a65ecc4a 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -380,6 +380,9 @@ lex_state_arg_p(yp_parser_t *parser) { static inline bool lex_state_spcarg_p(yp_parser_t *parser, bool space_seen) { + if (parser->current.end >= parser->end) { + return false; + } return lex_state_arg_p(parser) && space_seen && !yp_char_is_whitespace(*parser->current.end); } From 46e47404a8734efd7f312e29e2fb705dd2f40291 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 22:40:29 -0400 Subject: [PATCH 140/208] [ruby/yarp] fix: trailing decimal, binary, octal, and hex numbers at end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/af5b85a27a --- test/yarp/fuzzer_test.rb | 4 ++++ yarp/yarp.c | 12 ++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 97f128f08b09be..6f8cde0d09dd7d 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -17,4 +17,8 @@ def snippet(name, source) snippet "incomplete escaped string", '"\\' snippet "trailing comment", "1\n#\n" snippet "trailing asterisk", "a *" + snippet "incomplete decimal number", "0d" + snippet "incomplete binary number", "0b" + snippet "incomplete octal number", "0o" + snippet "incomplete hex number", "0x" end diff --git a/yarp/yarp.c b/yarp/yarp.c index 0d6b55a65ecc4a..6b2a3c64e861c0 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -4997,7 +4997,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0d1111 is a decimal number case 'd': case 'D': - if (yp_char_is_decimal_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_decimal_digit(peek(parser))) { parser->current.end += yp_strspn_decimal_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid decimal number."); @@ -5008,7 +5009,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0b1111 is a binary number case 'b': case 'B': - if (yp_char_is_binary_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_binary_digit(peek(parser))) { parser->current.end += yp_strspn_binary_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid binary number."); @@ -5019,7 +5021,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0o1111 is an octal number case 'o': case 'O': - if (yp_char_is_octal_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_octal_digit(peek(parser))) { parser->current.end += yp_strspn_octal_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid octal number."); @@ -5043,7 +5046,8 @@ lex_numeric_prefix(yp_parser_t *parser) { // 0x1111 is a hexadecimal number case 'x': case 'X': - if (yp_char_is_hexadecimal_digit(*++parser->current.end)) { + parser->current.end++; + if (yp_char_is_hexadecimal_digit(peek(parser))) { parser->current.end += yp_strspn_hexadecimal_number(parser->current.end, parser->end - parser->current.end); } else { yp_diagnostic_list_append(&parser->error_list, parser->current.start, parser->current.end, "Invalid hexadecimal number."); From 7cebb9b737eddced828073453004720f9970be6a Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 29 Aug 2023 23:03:16 -0400 Subject: [PATCH 141/208] [ruby/yarp] fix: incomplete escape in list at the end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/78ed75ed75 --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 6f8cde0d09dd7d..52d7f77e9f16ce 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -21,4 +21,5 @@ def snippet(name, source) snippet "incomplete binary number", "0b" snippet "incomplete octal number", "0o" snippet "incomplete hex number", "0x" + snippet "incomplete escaped list", "%w[\\" end diff --git a/yarp/yarp.c b/yarp/yarp.c index 6b2a3c64e861c0..3b2f29bf0120b5 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -6952,6 +6952,12 @@ parser_lex(yp_parser_t *parser) { // literally. In this case we'll skip past the next character // and find the next breakpoint. if (*breakpoint == '\\') { + // Check that we're not at the end of the file. + if (breakpoint + 1 >= parser->end) { + breakpoint = NULL; + continue; + } + yp_unescape_type_t unescape_type = lex_mode->as.list.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); From 341f47a6dd3690754fe9660bc248875c7b810260 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 08:31:33 -0400 Subject: [PATCH 142/208] [ruby/yarp] fix: incomplete escape in regex at the end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/55b9dfb41c --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 52d7f77e9f16ce..2d851ff886d3d1 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -22,4 +22,5 @@ def snippet(name, source) snippet "incomplete octal number", "0o" snippet "incomplete hex number", "0x" snippet "incomplete escaped list", "%w[\\" + snippet "incomplete escaped regex", "/a\\" end diff --git a/yarp/yarp.c b/yarp/yarp.c index 3b2f29bf0120b5..3fa143f31e9b9b 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -7091,6 +7091,12 @@ parser_lex(yp_parser_t *parser) { // literally. In this case we'll skip past the next character // and find the next breakpoint. if (*breakpoint == '\\') { + // Check that we're not at the end of the file. + if (breakpoint + 1 >= parser->end) { + breakpoint = NULL; + continue; + } + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, YP_UNESCAPE_ALL, false); // If the result is an escaped newline ... From ae7f9075592ea3570dfba831d086c423301fbcb7 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 13:52:03 -0400 Subject: [PATCH 143/208] [ruby/yarp] fix: heredoc with incomplete escape at end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/ec4abd87f4 --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 2d851ff886d3d1..6dfbf94f395931 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -23,4 +23,5 @@ def snippet(name, source) snippet "incomplete hex number", "0x" snippet "incomplete escaped list", "%w[\\" snippet "incomplete escaped regex", "/a\\" + snippet "unterminated heredoc with unterminated escape at end of file", "<= parser->end) { + breakpoint = NULL; + break; + } + // If we hit an escape, then we need to skip past // however many characters the escape takes up. However // it's important that if \n or \r\n are escaped that we From 2d009805e71e76c4f7678409aea804e18df01446 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 14:03:03 -0400 Subject: [PATCH 144/208] [ruby/yarp] fix: comment followed by whitespace at end of file Previously this resulted in invalid memory access. Found by the fuzzer. https://github.com/ruby/yarp/commit/b248553dd6 --- test/yarp/fuzzer_test.rb | 1 + yarp/yarp.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 6dfbf94f395931..8d818897a6841a 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -16,6 +16,7 @@ def snippet(name, source) snippet "incomplete symbol", ":" snippet "incomplete escaped string", '"\\' snippet "trailing comment", "1\n#\n" + snippet "comment followed by whitespace at end of file", "1\n#\n " snippet "trailing asterisk", "a *" snippet "incomplete decimal number", "0d" snippet "incomplete binary number", "0b" diff --git a/yarp/yarp.c b/yarp/yarp.c index 37d55cd4672a48..2422fb45719e59 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -5921,7 +5921,7 @@ parser_lex(yp_parser_t *parser) { // If this is not followed by a comment, then we can break out // of this loop. - if (*following != '#') break; + if (peek_at(parser, following) != '#') break; // If there is a comment, then we need to find the end of the // comment and continue searching from there. From eac3da173acf801638656c57a4554773c3af5ac0 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 14:35:59 -0400 Subject: [PATCH 145/208] [ruby/yarp] Fix test-unit API in fuzzer test https://github.com/ruby/yarp/commit/d24f62566e --- test/yarp/fuzzer_test.rb | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 8d818897a6841a..61845b91f76466 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -2,27 +2,25 @@ require_relative "test_helper" -# These tests are simply to exercise snippets found by the fuzzer that caused invalid memory access. -class FuzzerTest < Test::Unit::TestCase - class << self - def snippet(name, source) - test "fuzzer #{name}" do - YARP.dump(source) - end +module YARP + # These tests are simply to exercise snippets found by the fuzzer that caused invalid memory access. + class FuzzerTest < Test::Unit::TestCase + def self.snippet(name, source) + define_method(:"test_fuzzer_#{name}") { YARP.dump(source) } end - end - snippet "incomplete global variable", "$" - snippet "incomplete symbol", ":" - snippet "incomplete escaped string", '"\\' - snippet "trailing comment", "1\n#\n" - snippet "comment followed by whitespace at end of file", "1\n#\n " - snippet "trailing asterisk", "a *" - snippet "incomplete decimal number", "0d" - snippet "incomplete binary number", "0b" - snippet "incomplete octal number", "0o" - snippet "incomplete hex number", "0x" - snippet "incomplete escaped list", "%w[\\" - snippet "incomplete escaped regex", "/a\\" - snippet "unterminated heredoc with unterminated escape at end of file", "< Date: Tue, 29 Aug 2023 10:48:20 -0400 Subject: [PATCH 146/208] [ruby/yarp] Switch from handling const char * to const uint8_t * https://github.com/ruby/yarp/commit/465e7bb0a9 --- yarp/defines.h | 3 +- yarp/diagnostic.c | 2 +- yarp/diagnostic.h | 6 +- yarp/enc/yp_big5.c | 57 ++----- yarp/enc/yp_encoding.h | 20 +-- yarp/enc/yp_euc_jp.c | 59 ++----- yarp/enc/yp_gbk.c | 65 +++----- yarp/enc/yp_shift_jis.c | 59 ++----- yarp/enc/yp_tables.c | 74 +++++---- yarp/enc/yp_unicode.c | 45 +++--- yarp/enc/yp_windows_31j.c | 59 ++----- yarp/extension.c | 12 +- yarp/parser.h | 38 ++--- yarp/regexp.c | 42 ++--- yarp/regexp.h | 2 +- yarp/templates/ext/yarp/api_node.c.erb | 10 +- yarp/templates/include/yarp/ast.h.erb | 8 +- yarp/templates/src/prettyprint.c.erb | 2 +- yarp/templates/src/serialize.c.erb | 6 +- yarp/unescape.c | 133 ++++++++------- yarp/unescape.h | 4 +- yarp/util/yp_buffer.c | 9 +- yarp/util/yp_buffer.h | 3 + yarp/util/yp_char.c | 68 ++++---- yarp/util/yp_char.h | 32 ++-- yarp/util/yp_constant_pool.c | 6 +- yarp/util/yp_constant_pool.h | 4 +- yarp/util/yp_memchr.c | 2 +- yarp/util/yp_newline_list.c | 9 +- yarp/util/yp_newline_list.h | 10 +- yarp/util/yp_string.c | 42 ++--- yarp/util/yp_string.h | 8 +- yarp/util/yp_string_list.c | 6 - yarp/util/yp_string_list.h | 3 - yarp/util/yp_strncasecmp.c | 9 +- yarp/util/yp_strpbrk.c | 16 +- yarp/util/yp_strpbrk.h | 2 +- yarp/yarp.c | 216 ++++++++++++------------- yarp/yarp.h | 8 +- 39 files changed, 504 insertions(+), 655 deletions(-) diff --git a/yarp/defines.h b/yarp/defines.h index c08d578422dd5b..5fe3530d914fe4 100644 --- a/yarp/defines.h +++ b/yarp/defines.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,6 @@ # define snprintf _snprintf #endif -int yp_strncasecmp(const char *string1, const char *string2, size_t length); +int yp_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length); #endif diff --git a/yarp/diagnostic.c b/yarp/diagnostic.c index 8bd888e379bcea..b216d96a330974 100644 --- a/yarp/diagnostic.c +++ b/yarp/diagnostic.c @@ -2,7 +2,7 @@ // Append an error to the given list of diagnostic. bool -yp_diagnostic_list_append(yp_list_t *list, const char *start, const char *end, const char *message) { +yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, const char *message) { yp_diagnostic_t *diagnostic = (yp_diagnostic_t *) malloc(sizeof(yp_diagnostic_t)); if (diagnostic == NULL) return false; diff --git a/yarp/diagnostic.h b/yarp/diagnostic.h index bcbee5380ca98e..58228d8493ad31 100644 --- a/yarp/diagnostic.h +++ b/yarp/diagnostic.h @@ -10,13 +10,13 @@ // This struct represents a diagnostic found during parsing. typedef struct { yp_list_node_t node; - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; const char *message; } yp_diagnostic_t; // Append a diagnostic to the given list of diagnostics. -bool yp_diagnostic_list_append(yp_list_t *list, const char *start, const char *end, const char *message); +bool yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, const char *message); // Deallocate the internal state of the given diagnostic list. void yp_diagnostic_list_free(yp_list_t *list); diff --git a/yarp/enc/yp_big5.c b/yarp/enc/yp_big5.c index a33f5ce50493ac..a7c879cd0abb16 100644 --- a/yarp/enc/yp_big5.c +++ b/yarp/enc/yp_big5.c @@ -1,69 +1,42 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_big5_codepoint_t; - -static yp_big5_codepoint_t -yp_big5_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_big5_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80) { - *width = 1; - return *uc; + if (*b < 0x80) { + return 1; } // These are the double byte characters. - if ((n > 1) && (uc[0] >= 0xA1 && uc[0] <= 0xFE) && (uc[1] >= 0x40 && uc[1] <= 0xFE)) { - *width = 2; - return (yp_big5_codepoint_t) (uc[0] << 8 | uc[1]); + if ((n > 1) && (b[0] >= 0xA1 && b[0] <= 0xFE) && (b[1] >= 0x40 && b[1] <= 0xFE)) { + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_big5_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_big5_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_big5_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_big5_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_big5_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_big5_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_big5_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_big5_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_big5_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_big5_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return false; } diff --git a/yarp/enc/yp_encoding.h b/yarp/enc/yp_encoding.h index 7c4ce28c9428b7..9e8e7e01f60dba 100644 --- a/yarp/enc/yp_encoding.h +++ b/yarp/enc/yp_encoding.h @@ -16,22 +16,22 @@ typedef struct { // Return the number of bytes that the next character takes if it is valid // in the encoding. Does not read more than n bytes. It is assumed that n is // at least 1. - size_t (*char_width)(const char *c, ptrdiff_t n); + size_t (*char_width)(const uint8_t *b, ptrdiff_t n); // Return the number of bytes that the next character takes if it is valid // in the encoding and is alphabetical. Does not read more than n bytes. It // is assumed that n is at least 1. - size_t (*alpha_char)(const char *c, ptrdiff_t n); + size_t (*alpha_char)(const uint8_t *b, ptrdiff_t n); // Return the number of bytes that the next character takes if it is valid // in the encoding and is alphanumeric. Does not read more than n bytes. It // is assumed that n is at least 1. - size_t (*alnum_char)(const char *c, ptrdiff_t n); + size_t (*alnum_char)(const uint8_t *b, ptrdiff_t n); // Return true if the next character is valid in the encoding and is an // uppercase character. Does not read more than n bytes. It is assumed that // n is at least 1. - bool (*isupper_char)(const char *c, ptrdiff_t n); + bool (*isupper_char)(const uint8_t *b, ptrdiff_t n); // The name of the encoding. This should correspond to a value that can be // passed to Encoding.find in Ruby. @@ -49,18 +49,18 @@ typedef struct { // These functions are reused by some other encodings, so they are defined here // so they can be shared. -size_t yp_encoding_ascii_alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n); -size_t yp_encoding_ascii_alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n); -bool yp_encoding_ascii_isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n); +size_t yp_encoding_ascii_alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n); +size_t yp_encoding_ascii_alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n); +bool yp_encoding_ascii_isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n); // These functions are shared between the actual encoding and the fast path in // the parser so they need to be internally visible. -size_t yp_encoding_utf_8_alpha_char(const char *c, ptrdiff_t n); -size_t yp_encoding_utf_8_alnum_char(const char *c, ptrdiff_t n); +size_t yp_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n); +size_t yp_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n); // This lookup table is referenced in both the UTF-8 encoding file and the // parser directly in order to speed up the default encoding processing. -extern unsigned char yp_encoding_unicode_table[256]; +extern uint8_t yp_encoding_unicode_table[256]; // These are the encodings that are supported by the parser. They are defined in // their own files in the src/enc directory. diff --git a/yarp/enc/yp_euc_jp.c b/yarp/enc/yp_euc_jp.c index ebcd6a784992df..f6f80d528bcde0 100644 --- a/yarp/enc/yp_euc_jp.c +++ b/yarp/enc/yp_euc_jp.c @@ -1,75 +1,48 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_euc_jp_codepoint_t; - -static yp_euc_jp_codepoint_t -yp_euc_jp_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_euc_jp_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80) { - *width = 1; - return *uc; + if (*b < 0x80) { + return 1; } // These are the double byte characters. if ( (n > 1) && ( - ((uc[0] == 0x8E) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || - ((uc[0] >= 0xA1 && uc[0] <= 0xFE) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) + ((b[0] == 0x8E) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || + ((b[0] >= 0xA1 && b[0] <= 0xFE) && (b[1] >= 0xA1 && b[1] <= 0xFE)) ) ) { - *width = 2; - return (yp_euc_jp_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_euc_jp_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_euc_jp_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_euc_jp_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_euc_jp_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_euc_jp_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_euc_jp_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_euc_jp_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_euc_jp_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_euc_jp_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_euc_jp_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return 0; } diff --git a/yarp/enc/yp_gbk.c b/yarp/enc/yp_gbk.c index 31e88756db5865..71de318612d45d 100644 --- a/yarp/enc/yp_gbk.c +++ b/yarp/enc/yp_gbk.c @@ -1,78 +1,51 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_gbk_codepoint_t; - -static yp_gbk_codepoint_t -yp_gbk_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_gbk_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80) { - *width = 1; - return *uc; + if (*b < 0x80) { + return 1; } // These are the double byte characters. if ( (n > 1) && ( - ((uc[0] >= 0xA1 && uc[0] <= 0xA9) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || // GBK/1 - ((uc[0] >= 0xB0 && uc[0] <= 0xF7) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || // GBK/2 - ((uc[0] >= 0x81 && uc[0] <= 0xA0) && (uc[1] >= 0x40 && uc[1] <= 0xFE) && (uc[1] != 0x7F)) || // GBK/3 - ((uc[0] >= 0xAA && uc[0] <= 0xFE) && (uc[1] >= 0x40 && uc[1] <= 0xA0) && (uc[1] != 0x7F)) || // GBK/4 - ((uc[0] >= 0xA8 && uc[0] <= 0xA9) && (uc[1] >= 0x40 && uc[1] <= 0xA0) && (uc[1] != 0x7F)) // GBK/5 + ((b[0] >= 0xA1 && b[0] <= 0xA9) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || // GBK/1 + ((b[0] >= 0xB0 && b[0] <= 0xF7) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || // GBK/2 + ((b[0] >= 0x81 && b[0] <= 0xA0) && (b[1] >= 0x40 && b[1] <= 0xFE) && (b[1] != 0x7F)) || // GBK/3 + ((b[0] >= 0xAA && b[0] <= 0xFE) && (b[1] >= 0x40 && b[1] <= 0xA0) && (b[1] != 0x7F)) || // GBK/4 + ((b[0] >= 0xA8 && b[0] <= 0xA9) && (b[1] >= 0x40 && b[1] <= 0xA0) && (b[1] != 0x7F)) // GBK/5 ) ) { - *width = 2; - return (yp_gbk_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_gbk_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_gbk_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_gbk_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_gbk_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_gbk_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_gbk_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_gbk_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_gbk_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_gbk_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_gbk_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return false; } diff --git a/yarp/enc/yp_shift_jis.c b/yarp/enc/yp_shift_jis.c index 1f361b9e705cd2..e6ca10d1fdc5b1 100644 --- a/yarp/enc/yp_shift_jis.c +++ b/yarp/enc/yp_shift_jis.c @@ -1,73 +1,46 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_shift_jis_codepoint_t; - -static yp_shift_jis_codepoint_t -yp_shift_jis_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_shift_jis_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80 || (*uc >= 0xA1 && *uc <= 0xDF)) { - *width = 1; - return *uc; + if (*b < 0x80 || (*b >= 0xA1 && *b <= 0xDF)) { + return 1; } // These are the double byte characters. if ( (n > 1) && - ((uc[0] >= 0x81 && uc[0] <= 0x9F) || (uc[0] >= 0xE0 && uc[0] <= 0xFC)) && - (uc[1] >= 0x40 && uc[1] <= 0xFC) + ((b[0] >= 0x81 && b[0] <= 0x9F) || (b[0] >= 0xE0 && b[0] <= 0xFC)) && + (b[1] >= 0x40 && b[1] <= 0xFC) ) { - *width = 2; - return (yp_shift_jis_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_shift_jis_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_shift_jis_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_shift_jis_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_shift_jis_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_shift_jis_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_shift_jis_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_shift_jis_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_shift_jis_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_shift_jis_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_shift_jis_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return 0; } diff --git a/yarp/enc/yp_tables.c b/yarp/enc/yp_tables.c index 057f2b3f818259..5504cd5419ac93 100644 --- a/yarp/enc/yp_tables.c +++ b/yarp/enc/yp_tables.c @@ -2,7 +2,7 @@ // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ASCII character. -static unsigned char yp_encoding_ascii_table[256] = { +static uint8_t yp_encoding_ascii_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -24,7 +24,7 @@ static unsigned char yp_encoding_ascii_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-1 character. -static unsigned char yp_encoding_iso_8859_1_table[256] = { +static uint8_t yp_encoding_iso_8859_1_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -46,7 +46,7 @@ static unsigned char yp_encoding_iso_8859_1_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-2 character. -static unsigned char yp_encoding_iso_8859_2_table[256] = { +static uint8_t yp_encoding_iso_8859_2_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -68,7 +68,7 @@ static unsigned char yp_encoding_iso_8859_2_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-3 character. -static unsigned char yp_encoding_iso_8859_3_table[256] = { +static uint8_t yp_encoding_iso_8859_3_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -90,7 +90,7 @@ static unsigned char yp_encoding_iso_8859_3_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-4 character. -static unsigned char yp_encoding_iso_8859_4_table[256] = { +static uint8_t yp_encoding_iso_8859_4_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -112,7 +112,7 @@ static unsigned char yp_encoding_iso_8859_4_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-5 character. -static unsigned char yp_encoding_iso_8859_5_table[256] = { +static uint8_t yp_encoding_iso_8859_5_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -134,7 +134,7 @@ static unsigned char yp_encoding_iso_8859_5_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-6 character. -static unsigned char yp_encoding_iso_8859_6_table[256] = { +static uint8_t yp_encoding_iso_8859_6_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -156,7 +156,7 @@ static unsigned char yp_encoding_iso_8859_6_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-7 character. -static unsigned char yp_encoding_iso_8859_7_table[256] = { +static uint8_t yp_encoding_iso_8859_7_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -178,7 +178,7 @@ static unsigned char yp_encoding_iso_8859_7_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-8 character. -static unsigned char yp_encoding_iso_8859_8_table[256] = { +static uint8_t yp_encoding_iso_8859_8_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -200,7 +200,7 @@ static unsigned char yp_encoding_iso_8859_8_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-9 character. -static unsigned char yp_encoding_iso_8859_9_table[256] = { +static uint8_t yp_encoding_iso_8859_9_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -222,7 +222,7 @@ static unsigned char yp_encoding_iso_8859_9_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-10 character. -static unsigned char yp_encoding_iso_8859_10_table[256] = { +static uint8_t yp_encoding_iso_8859_10_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -244,7 +244,7 @@ static unsigned char yp_encoding_iso_8859_10_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-11 character. -static unsigned char yp_encoding_iso_8859_11_table[256] = { +static uint8_t yp_encoding_iso_8859_11_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -266,7 +266,7 @@ static unsigned char yp_encoding_iso_8859_11_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-13 character. -static unsigned char yp_encoding_iso_8859_13_table[256] = { +static uint8_t yp_encoding_iso_8859_13_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -288,7 +288,7 @@ static unsigned char yp_encoding_iso_8859_13_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-14 character. -static unsigned char yp_encoding_iso_8859_14_table[256] = { +static uint8_t yp_encoding_iso_8859_14_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -310,7 +310,7 @@ static unsigned char yp_encoding_iso_8859_14_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-15 character. -static unsigned char yp_encoding_iso_8859_15_table[256] = { +static uint8_t yp_encoding_iso_8859_15_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -332,7 +332,7 @@ static unsigned char yp_encoding_iso_8859_15_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding ISO-8859-16 character. -static unsigned char yp_encoding_iso_8859_16_table[256] = { +static uint8_t yp_encoding_iso_8859_16_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -354,7 +354,7 @@ static unsigned char yp_encoding_iso_8859_16_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding KOI8-R character. -static unsigned char yp_encoding_koi8_r_table[256] = { +static uint8_t yp_encoding_koi8_r_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -376,7 +376,7 @@ static unsigned char yp_encoding_koi8_r_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding windows-1251 character. -static unsigned char yp_encoding_windows_1251_table[256] = { +static uint8_t yp_encoding_windows_1251_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -398,7 +398,7 @@ static unsigned char yp_encoding_windows_1251_table[256] = { // Each element of the following table contains a bitfield that indicates a // piece of information about the corresponding windows-1252 character. -static unsigned char yp_encoding_windows_1252_table[256] = { +static uint8_t yp_encoding_windows_1252_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -419,34 +419,32 @@ static unsigned char yp_encoding_windows_1252_table[256] = { }; static size_t -yp_encoding_ascii_char_width(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - const unsigned char v = (const unsigned char) *c; - return v < 0x80 ? 1 : 0; +yp_encoding_ascii_char_width(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return *b < 0x80 ? 1 : 0; } size_t -yp_encoding_ascii_alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - return (yp_encoding_ascii_table[(const unsigned char) *c] & YP_ENCODING_ALPHABETIC_BIT); +yp_encoding_ascii_alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return (yp_encoding_ascii_table[*b] & YP_ENCODING_ALPHABETIC_BIT); } size_t -yp_encoding_ascii_alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - return (yp_encoding_ascii_table[(const unsigned char) *c] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; +yp_encoding_ascii_alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return (yp_encoding_ascii_table[*b] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; } bool -yp_encoding_ascii_isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - return (yp_encoding_ascii_table[(const unsigned char) *c] & YP_ENCODING_UPPERCASE_BIT); +yp_encoding_ascii_isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return (yp_encoding_ascii_table[*b] & YP_ENCODING_UPPERCASE_BIT); } static size_t -yp_encoding_koi8_r_char_width(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { - const unsigned char v = (const unsigned char) *c; - return ((v >= 0x20 && v <= 0x7E) || (v >= 0x80)) ? 1 : 0; +yp_encoding_koi8_r_char_width(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { + return ((*b >= 0x20 && *b <= 0x7E) || (*b >= 0x80)) ? 1 : 0; } static size_t -yp_encoding_single_char_width(YP_ATTRIBUTE_UNUSED const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { +yp_encoding_single_char_width(YP_ATTRIBUTE_UNUSED const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { return 1; } @@ -469,14 +467,14 @@ yp_encoding_t yp_encoding_ascii_8bit = { }; #define YP_ENCODING_TABLE(s, i, w) \ - static size_t yp_encoding_ ##i ## _alpha_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ - return (yp_encoding_ ##i ## _table[(const unsigned char) *c] & YP_ENCODING_ALPHABETIC_BIT); \ + static size_t yp_encoding_ ##i ## _alpha_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ + return (yp_encoding_ ##i ## _table[*b] & YP_ENCODING_ALPHABETIC_BIT); \ } \ - static size_t yp_encoding_ ##i ## _alnum_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ - return (yp_encoding_ ##i ## _table[(const unsigned char) *c] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; \ + static size_t yp_encoding_ ##i ## _alnum_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ + return (yp_encoding_ ##i ## _table[*b] & YP_ENCODING_ALPHANUMERIC_BIT) ? 1 : 0; \ } \ - static bool yp_encoding_ ##i ## _isupper_char(const char *c, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ - return (yp_encoding_ ##i ## _table[(const unsigned char) *c] & YP_ENCODING_UPPERCASE_BIT); \ + static bool yp_encoding_ ##i ## _isupper_char(const uint8_t *b, YP_ATTRIBUTE_UNUSED ptrdiff_t n) { \ + return (yp_encoding_ ##i ## _table[*b] & YP_ENCODING_UPPERCASE_BIT); \ } \ yp_encoding_t yp_encoding_ ##i = { \ .name = s, \ diff --git a/yarp/enc/yp_unicode.c b/yarp/enc/yp_unicode.c index fc2f0336fb7bf3..bb4e041309baa7 100644 --- a/yarp/enc/yp_unicode.c +++ b/yarp/enc/yp_unicode.c @@ -10,7 +10,7 @@ typedef uint32_t yp_unicode_codepoint_t; // this table is different from other encodings where we used a lookup table // because the indices of those tables are the byte representations, not the // codepoints themselves. -unsigned char yp_encoding_unicode_table[256] = { +uint8_t yp_encoding_unicode_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x @@ -2220,7 +2220,7 @@ static const uint8_t yp_utf_8_dfa[] = { }; static yp_unicode_codepoint_t -yp_utf_8_codepoint(const unsigned char *c, ptrdiff_t n, size_t *width) { +yp_utf_8_codepoint(const uint8_t *b, ptrdiff_t n, size_t *width) { assert(n >= 1); size_t maximum = (size_t) n; @@ -2228,7 +2228,7 @@ yp_utf_8_codepoint(const unsigned char *c, ptrdiff_t n, size_t *width) { uint32_t state = 0; for (size_t index = 0; index < 4 && index < maximum; index++) { - uint32_t byte = c[index]; + uint32_t byte = b[index]; uint32_t type = yp_utf_8_dfa[byte]; codepoint = (state != 0) ? @@ -2247,60 +2247,55 @@ yp_utf_8_codepoint(const unsigned char *c, ptrdiff_t n, size_t *width) { } static size_t -yp_encoding_utf_8_char_width(const char *c, ptrdiff_t n) { +yp_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n) { size_t width; - const unsigned char *v = (const unsigned char *) c; - - yp_utf_8_codepoint(v, n, &width); + yp_utf_8_codepoint(b, n, &width); return width; } size_t -yp_encoding_utf_8_alpha_char(const char *c, ptrdiff_t n) { - const unsigned char *v = (const unsigned char *) c; - if (*v < 0x80) { - return (yp_encoding_unicode_table[*v] & YP_ENCODING_ALPHABETIC_BIT) ? 1 : 0; +yp_encoding_utf_8_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_ALPHABETIC_BIT) ? 1 : 0; } size_t width; - yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(v, n, &width); + yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(b, n, &width); if (codepoint <= 0xFF) { - return (yp_encoding_unicode_table[(unsigned char) codepoint] & YP_ENCODING_ALPHABETIC_BIT) ? width : 0; + return (yp_encoding_unicode_table[(uint8_t) codepoint] & YP_ENCODING_ALPHABETIC_BIT) ? width : 0; } else { return yp_unicode_codepoint_match(codepoint, unicode_alpha_codepoints, UNICODE_ALPHA_CODEPOINTS_LENGTH) ? width : 0; } } size_t -yp_encoding_utf_8_alnum_char(const char *c, ptrdiff_t n) { - const unsigned char *v = (const unsigned char *) c; - if (*v < 0x80) { - return (yp_encoding_unicode_table[*v] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? 1 : 0; +yp_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? 1 : 0; } size_t width; - yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(v, n, &width); + yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(b, n, &width); if (codepoint <= 0xFF) { - return (yp_encoding_unicode_table[(unsigned char) codepoint] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? width : 0; + return (yp_encoding_unicode_table[(uint8_t) codepoint] & (YP_ENCODING_ALPHANUMERIC_BIT)) ? width : 0; } else { return yp_unicode_codepoint_match(codepoint, unicode_alnum_codepoints, UNICODE_ALNUM_CODEPOINTS_LENGTH) ? width : 0; } } static bool -yp_encoding_utf_8_isupper_char(const char *c, ptrdiff_t n) { - const unsigned char *v = (const unsigned char *) c; - if (*v < 0x80) { - return (yp_encoding_unicode_table[*v] & YP_ENCODING_UPPERCASE_BIT) ? true : false; +yp_encoding_utf_8_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_UPPERCASE_BIT) ? true : false; } size_t width; - yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(v, n, &width); + yp_unicode_codepoint_t codepoint = yp_utf_8_codepoint(b, n, &width); if (codepoint <= 0xFF) { - return (yp_encoding_unicode_table[(unsigned char) codepoint] & YP_ENCODING_UPPERCASE_BIT) ? true : false; + return (yp_encoding_unicode_table[(uint8_t) codepoint] & YP_ENCODING_UPPERCASE_BIT) ? true : false; } else { return yp_unicode_codepoint_match(codepoint, unicode_isupper_codepoints, UNICODE_ISUPPER_CODEPOINTS_LENGTH) ? true : false; } diff --git a/yarp/enc/yp_windows_31j.c b/yarp/enc/yp_windows_31j.c index 7062d7c39c3a90..0d346395353e1e 100644 --- a/yarp/enc/yp_windows_31j.c +++ b/yarp/enc/yp_windows_31j.c @@ -1,73 +1,46 @@ #include "yarp/enc/yp_encoding.h" -typedef uint16_t yp_windows_31j_codepoint_t; - -static yp_windows_31j_codepoint_t -yp_windows_31j_codepoint(const char *c, ptrdiff_t n, size_t *width) { - const unsigned char *uc = (const unsigned char *) c; - +static size_t +yp_encoding_windows_31j_char_width(const uint8_t *b, ptrdiff_t n) { // These are the single byte characters. - if (*uc < 0x80 || (*uc >= 0xA1 && *uc <= 0xDF)) { - *width = 1; - return *uc; + if (*b < 0x80 || (*b >= 0xA1 && *b <= 0xDF)) { + return 1; } // These are the double byte characters. if ( (n > 1) && - ((uc[0] >= 0x81 && uc[0] <= 0x9F) || (uc[0] >= 0xE0 && uc[0] <= 0xFC)) && - (uc[1] >= 0x40 && uc[1] <= 0xFC) + ((b[0] >= 0x81 && b[0] <= 0x9F) || (b[0] >= 0xE0 && b[0] <= 0xFC)) && + (b[1] >= 0x40 && b[1] <= 0xFC) ) { - *width = 2; - return (yp_windows_31j_codepoint_t) (uc[0] << 8 | uc[1]); + return 2; } - *width = 0; return 0; } static size_t -yp_encoding_windows_31j_char_width(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint(c, n, &width); - - return width; -} - -static size_t -yp_encoding_windows_31j_alpha_char(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint_t codepoint = yp_windows_31j_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alpha_char(&value, n); +yp_encoding_windows_31j_alpha_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_windows_31j_char_width(b, n) == 1) { + return yp_encoding_ascii_alpha_char(b, n); } else { return 0; } } static size_t -yp_encoding_windows_31j_alnum_char(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint_t codepoint = yp_windows_31j_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_alnum_char(&value, n); +yp_encoding_windows_31j_alnum_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_windows_31j_char_width(b, n) == 1) { + return yp_encoding_ascii_alnum_char(b, n); } else { return 0; } } static bool -yp_encoding_windows_31j_isupper_char(const char *c, ptrdiff_t n) { - size_t width; - yp_windows_31j_codepoint_t codepoint = yp_windows_31j_codepoint(c, n, &width); - - if (width == 1) { - const char value = (const char) codepoint; - return yp_encoding_ascii_isupper_char(&value, n); +yp_encoding_windows_31j_isupper_char(const uint8_t *b, ptrdiff_t n) { + if (yp_encoding_windows_31j_char_width(b, n) == 1) { + return yp_encoding_ascii_isupper_char(b, n); } else { return false; } diff --git a/yarp/extension.c b/yarp/extension.c index f959dba31ac234..de925f1509d7fd 100644 --- a/yarp/extension.c +++ b/yarp/extension.c @@ -260,7 +260,7 @@ parse_lex_input(yp_string_t *input, const char *filepath, bool return_nodes) { yp_parser_register_encoding_changed_callback(&parser, parse_lex_encoding_changed_callback); VALUE offsets = rb_ary_new(); - VALUE source_argv[] = { rb_str_new(yp_string_source(input), yp_string_length(input)), offsets }; + VALUE source_argv[] = { rb_str_new((const char *) yp_string_source(input), yp_string_length(input)), offsets }; VALUE source = rb_class_new_instance(2, source_argv, rb_cYARPSource); parse_lex_data_t parse_lex_data = { @@ -442,7 +442,7 @@ named_captures(VALUE self, VALUE source) { yp_string_list_t string_list; yp_string_list_init(&string_list); - if (!yp_regexp_named_capture_group_names(RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, &yp_encoding_utf_8)) { + if (!yp_regexp_named_capture_group_names((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), &string_list, false, &yp_encoding_utf_8)) { yp_string_list_free(&string_list); return Qnil; } @@ -450,7 +450,7 @@ named_captures(VALUE self, VALUE source) { VALUE names = rb_ary_new(); for (size_t index = 0; index < string_list.length; index++) { const yp_string_t *string = &string_list.strings[index]; - rb_ary_push(names, rb_str_new(yp_string_source(string), yp_string_length(string))); + rb_ary_push(names, rb_str_new((const char *) yp_string_source(string), yp_string_length(string))); } yp_string_list_free(&string_list); @@ -463,8 +463,8 @@ static VALUE unescape(VALUE source, yp_unescape_type_t unescape_type) { yp_string_t result; - if (yp_unescape_string(RSTRING_PTR(source), RSTRING_LEN(source), unescape_type, &result)) { - VALUE str = rb_str_new(yp_string_source(&result), yp_string_length(&result)); + if (yp_unescape_string((const uint8_t *) RSTRING_PTR(source), RSTRING_LEN(source), unescape_type, &result)) { + VALUE str = rb_str_new((const char *) yp_string_source(&result), yp_string_length(&result)); yp_string_free(&result); return str; } else { @@ -498,7 +498,7 @@ static VALUE memsize(VALUE self, VALUE string) { yp_parser_t parser; size_t length = RSTRING_LEN(string); - yp_parser_init(&parser, RSTRING_PTR(string), length, NULL); + yp_parser_init(&parser, (const uint8_t *) RSTRING_PTR(string), length, NULL); yp_node_t *node = yp_parse(&parser); yp_memsize_t memsize; diff --git a/yarp/parser.h b/yarp/parser.h index 2091be7fd92c8f..0ae01f78da6e53 100644 --- a/yarp/parser.h +++ b/yarp/parser.h @@ -109,14 +109,14 @@ typedef struct yp_lex_mode { // When lexing a list, it takes into account balancing the // terminator if the terminator is one of (), [], {}, or <>. - char incrementor; + uint8_t incrementor; // This is the terminator of the list literal. - char terminator; + uint8_t terminator; // This is the character set that should be used to delimit the // tokens within the list. - char breakpoints[11]; + uint8_t breakpoints[11]; } list; struct { @@ -125,14 +125,14 @@ typedef struct yp_lex_mode { // When lexing a regular expression, it takes into account balancing // the terminator if the terminator is one of (), [], {}, or <>. - char incrementor; + uint8_t incrementor; // This is the terminator of the regular expression. - char terminator; + uint8_t terminator; // This is the character set that should be used to delimit the // tokens within the regular expression. - char breakpoints[6]; + uint8_t breakpoints[6]; } regexp; struct { @@ -149,21 +149,21 @@ typedef struct yp_lex_mode { // When lexing a string, it takes into account balancing the // terminator if the terminator is one of (), [], {}, or <>. - char incrementor; + uint8_t incrementor; // This is the terminator of the string. It is typically either a // single or double quote. - char terminator; + uint8_t terminator; // This is the character set that should be used to delimit the // tokens within the string. - char breakpoints[6]; + uint8_t breakpoints[6]; } string; struct { // These pointers point to the beginning and end of the heredoc // identifier. - const char *ident_start; + const uint8_t *ident_start; size_t ident_length; yp_heredoc_quote_t quote; @@ -171,7 +171,7 @@ typedef struct yp_lex_mode { // This is the pointer to the character where lexing should resume // once the heredoc has been completely processed. - const char *next_start; + const uint8_t *next_start; } heredoc; } as; @@ -239,8 +239,8 @@ typedef enum { // This is a node in the linked list of comments that we've found while parsing. typedef struct yp_comment { yp_list_node_t node; - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; yp_comment_type_t type; } yp_comment_t; @@ -252,7 +252,7 @@ typedef void (*yp_encoding_changed_callback_t)(yp_parser_t *parser); // the ability here to call out to a user-defined function to get an encoding // struct. If the function returns something that isn't NULL, we set that to // our encoding and use it to parse identifiers. -typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const char *name, size_t width); +typedef yp_encoding_t *(*yp_encoding_decode_callback_t)(yp_parser_t *parser, const uint8_t *name, size_t width); // When you are lexing through a file, the lexer needs all of the information // that the parser additionally provides (for example, the local table). So if @@ -316,21 +316,21 @@ struct yp_parser { size_t index; // the current index into the lexer mode stack } lex_modes; - const char *start; // the pointer to the start of the source - const char *end; // the pointer to the end of the source + const uint8_t *start; // the pointer to the start of the source + const uint8_t *end; // the pointer to the end of the source yp_token_t previous; // the previous token we were considering yp_token_t current; // the current token we're considering // This is a special field set on the parser when we need the parser to jump // to a specific location when lexing the next token, as opposed to just // using the end of the previous token. Normally this is NULL. - const char *next_start; + const uint8_t *next_start; // This field indicates the end of a heredoc whose identifier was found on // the current line. If another heredoc is found on the same line, then this // will be moved forward to the end of that heredoc. If no heredocs are // found on a line then this is NULL. - const char *heredoc_end; + const uint8_t *heredoc_end; yp_list_t comment_list; // the list of comments that have been found while parsing yp_list_t warning_list; // the list of warnings that have been found while parsing @@ -361,7 +361,7 @@ struct yp_parser { // This pointer indicates where a comment must start if it is to be // considered an encoding comment. - const char *encoding_comment_start; + const uint8_t *encoding_comment_start; // This is an optional callback that can be attached to the parser that will // be called whenever a new token is lexed by the parser. diff --git a/yarp/regexp.c b/yarp/regexp.c index 4d6b67ebe633a4..2aeadc1bfcfef0 100644 --- a/yarp/regexp.c +++ b/yarp/regexp.c @@ -2,9 +2,9 @@ // This is the parser that is going to handle parsing regular expressions. typedef struct { - const char *start; - const char *cursor; - const char *end; + const uint8_t *start; + const uint8_t *cursor; + const uint8_t *end; yp_string_list_t *named_captures; bool encoding_changed; yp_encoding_t *encoding; @@ -12,7 +12,7 @@ typedef struct { // This initializes a new parser with the given source. static void -yp_regexp_parser_init(yp_regexp_parser_t *parser, const char *start, const char *end, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { +yp_regexp_parser_init(yp_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { *parser = (yp_regexp_parser_t) { .start = start, .cursor = start, @@ -25,7 +25,7 @@ yp_regexp_parser_init(yp_regexp_parser_t *parser, const char *start, const char // This appends a new string to the list of named captures. static void -yp_regexp_parser_named_capture(yp_regexp_parser_t *parser, const char *start, const char *end) { +yp_regexp_parser_named_capture(yp_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) { yp_string_t string; yp_string_shared_init(&string, start, end); yp_string_list_append(parser->named_captures, &string); @@ -40,7 +40,7 @@ yp_regexp_char_is_eof(yp_regexp_parser_t *parser) { // Optionally accept a char and consume it if it exists. static inline bool -yp_regexp_char_accept(yp_regexp_parser_t *parser, char value) { +yp_regexp_char_accept(yp_regexp_parser_t *parser, uint8_t value) { if (!yp_regexp_char_is_eof(parser) && *parser->cursor == value) { parser->cursor++; return true; @@ -50,7 +50,7 @@ yp_regexp_char_accept(yp_regexp_parser_t *parser, char value) { // Expect a character to be present and consume it. static inline bool -yp_regexp_char_expect(yp_regexp_parser_t *parser, char value) { +yp_regexp_char_expect(yp_regexp_parser_t *parser, uint8_t value) { if (!yp_regexp_char_is_eof(parser) && *parser->cursor == value) { parser->cursor++; return true; @@ -60,12 +60,12 @@ yp_regexp_char_expect(yp_regexp_parser_t *parser, char value) { // This advances the current token to the next instance of the given character. static bool -yp_regexp_char_find(yp_regexp_parser_t *parser, char value) { +yp_regexp_char_find(yp_regexp_parser_t *parser, uint8_t value) { if (yp_regexp_char_is_eof(parser)) { return false; } - const char *end = (const char *) yp_memchr(parser->cursor, value, (size_t) (parser->end - parser->cursor), parser->encoding_changed, parser->encoding); + const uint8_t *end = (const uint8_t *) yp_memchr(parser->cursor, value, (size_t) (parser->end - parser->cursor), parser->encoding_changed, parser->encoding); if (end == NULL) { return false; } @@ -107,7 +107,7 @@ yp_regexp_char_find(yp_regexp_parser_t *parser, char value) { // consumed so we're in the start state. static bool yp_regexp_parse_range_quantifier(yp_regexp_parser_t *parser) { - const char *savepoint = parser->cursor; + const uint8_t *savepoint = parser->cursor; enum { YP_REGEXP_RANGE_QUANTIFIER_STATE_START, @@ -252,7 +252,7 @@ yp_regexp_parse_character_set(yp_regexp_parser_t *parser) { // A left bracket can either mean a POSIX class or a character set. static bool yp_regexp_parse_lbracket(yp_regexp_parser_t *parser) { - const char *reset = parser->cursor; + const uint8_t *reset = parser->cursor; if ((parser->cursor + 2 < parser->end) && parser->cursor[0] == '[' && parser->cursor[1] == ':') { parser->cursor++; @@ -287,7 +287,7 @@ typedef enum { // This is the set of options that are configurable on the regular expression. typedef struct { - unsigned char values[YP_REGEXP_OPTION_STATE_SLOTS]; + uint8_t values[YP_REGEXP_OPTION_STATE_SLOTS]; } yp_regexp_options_t; // Initialize a new set of options to their default values. @@ -305,9 +305,9 @@ yp_regexp_options_init(yp_regexp_options_t *options) { // Attempt to add the given option to the set of options. Returns true if it was // added, false if it was already present. static bool -yp_regexp_options_add(yp_regexp_options_t *options, unsigned char key) { +yp_regexp_options_add(yp_regexp_options_t *options, uint8_t key) { if (key >= YP_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= YP_REGEXP_OPTION_STATE_SLOT_MAXIMUM) { - key = (unsigned char) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); + key = (uint8_t) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); switch (options->values[key]) { case YP_REGEXP_OPTION_STATE_INVALID: @@ -328,9 +328,9 @@ yp_regexp_options_add(yp_regexp_options_t *options, unsigned char key) { // Attempt to remove the given option from the set of options. Returns true if // it was removed, false if it was already absent. static bool -yp_regexp_options_remove(yp_regexp_options_t *options, unsigned char key) { +yp_regexp_options_remove(yp_regexp_options_t *options, uint8_t key) { if (key >= YP_REGEXP_OPTION_STATE_SLOT_MINIMUM && key <= YP_REGEXP_OPTION_STATE_SLOT_MAXIMUM) { - key = (unsigned char) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); + key = (uint8_t) (key - YP_REGEXP_OPTION_STATE_SLOT_MINIMUM); switch (options->values[key]) { case YP_REGEXP_OPTION_STATE_INVALID: @@ -431,7 +431,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { parser->cursor++; break; default: { // named capture group - const char *start = parser->cursor; + const uint8_t *start = parser->cursor; if (!yp_regexp_char_find(parser, '>')) { return false; } @@ -441,7 +441,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { } break; case '\'': { // named capture group - const char *start = ++parser->cursor; + const uint8_t *start = ++parser->cursor; if (!yp_regexp_char_find(parser, '\'')) { return false; } @@ -456,7 +456,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { break; case 'i': case 'm': case 'x': case 'd': case 'a': case 'u': // options while (!yp_regexp_char_is_eof(parser) && *parser->cursor != '-' && *parser->cursor != ':' && *parser->cursor != ')') { - if (!yp_regexp_options_add(&options, (unsigned char) *parser->cursor)) { + if (!yp_regexp_options_add(&options, *parser->cursor)) { return false; } parser->cursor++; @@ -474,7 +474,7 @@ yp_regexp_parse_group(yp_regexp_parser_t *parser) { case '-': parser->cursor++; while (!yp_regexp_char_is_eof(parser) && *parser->cursor != ':' && *parser->cursor != ')') { - if (!yp_regexp_options_remove(&options, (unsigned char) *parser->cursor)) { + if (!yp_regexp_options_remove(&options, *parser->cursor)) { return false; } parser->cursor++; @@ -573,7 +573,7 @@ yp_regexp_parse_pattern(yp_regexp_parser_t *parser) { // Parse a regular expression and extract the names of all of the named capture // groups. YP_EXPORTED_FUNCTION bool -yp_regexp_named_capture_group_names(const char *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { +yp_regexp_named_capture_group_names(const uint8_t *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding) { yp_regexp_parser_t parser; yp_regexp_parser_init(&parser, source, source + size, named_captures, encoding_changed, encoding); return yp_regexp_parse_pattern(&parser); diff --git a/yarp/regexp.h b/yarp/regexp.h index 5a2f13047e552f..6807c58398080b 100644 --- a/yarp/regexp.h +++ b/yarp/regexp.h @@ -14,6 +14,6 @@ // Parse a regular expression and extract the names of all of the named capture // groups. -YP_EXPORTED_FUNCTION bool yp_regexp_named_capture_group_names(const char *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding); +YP_EXPORTED_FUNCTION bool yp_regexp_named_capture_group_names(const uint8_t *source, size_t size, yp_string_list_t *named_captures, bool encoding_changed, yp_encoding_t *encoding); #endif diff --git a/yarp/templates/ext/yarp/api_node.c.erb b/yarp/templates/ext/yarp/api_node.c.erb index 599bf2e9ffb2b7..fb25919ca38a62 100644 --- a/yarp/templates/ext/yarp/api_node.c.erb +++ b/yarp/templates/ext/yarp/api_node.c.erb @@ -12,7 +12,7 @@ static VALUE rb_cYARP<%= node.name %>; <%- end -%> static VALUE -yp_location_new(yp_parser_t *parser, const char *start, const char *end, VALUE source) { +yp_location_new(yp_parser_t *parser, const uint8_t *start, const uint8_t *end, VALUE source) { VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(end - start) }; return rb_class_new_instance(3, argv, rb_cYARPLocation); } @@ -24,7 +24,7 @@ yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALU VALUE argv[] = { ID2SYM(type), - rb_enc_str_new(token->start, token->end - token->start, encoding), + rb_enc_str_new((const char *) token->start, token->end - token->start, encoding), location }; @@ -33,13 +33,13 @@ yp_token_new(yp_parser_t *parser, yp_token_t *token, rb_encoding *encoding, VALU static VALUE yp_string_new(yp_string_t *string, rb_encoding *encoding) { - return rb_enc_str_new(yp_string_source(string), yp_string_length(string), encoding); + return rb_enc_str_new((const char *) yp_string_source(string), yp_string_length(string), encoding); } // Create a YARP::Source object from the given parser. VALUE yp_source_new(yp_parser_t *parser) { - VALUE source = rb_str_new(parser->start, parser->end - parser->start); + VALUE source = rb_str_new((const char *) parser->start, parser->end - parser->start); VALUE offsets = rb_ary_new_capa(parser->newline_list.size); for (size_t index = 0; index < parser->newline_list.size; index++) { @@ -85,7 +85,7 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) { yp_constant_t constant = parser->constant_pool.constants[index]; if (constant.id != 0) { - constants[constant.id - 1] = rb_intern3(constant.start, constant.length, encoding); + constants[constant.id - 1] = rb_intern3((const char *) constant.start, constant.length, encoding); } } diff --git a/yarp/templates/include/yarp/ast.h.erb b/yarp/templates/include/yarp/ast.h.erb index 6fe3bc2c2456bb..6eeadb49d292d4 100644 --- a/yarp/templates/include/yarp/ast.h.erb +++ b/yarp/templates/include/yarp/ast.h.erb @@ -21,15 +21,15 @@ typedef enum yp_token_type { // type and location information. typedef struct { yp_token_type_t type; - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; } yp_token_t; // This represents a range of bytes in the source string to which a node or // token corresponds. typedef struct { - const char *start; - const char *end; + const uint8_t *start; + const uint8_t *end; } yp_location_t; typedef struct { diff --git a/yarp/templates/src/prettyprint.c.erb b/yarp/templates/src/prettyprint.c.erb index cf2f12f2aeff73..ded483c04aed38 100644 --- a/yarp/templates/src/prettyprint.c.erb +++ b/yarp/templates/src/prettyprint.c.erb @@ -36,7 +36,7 @@ prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) { } <%- when StringParam -%> yp_buffer_append_str(buffer, "\"", 1); - yp_buffer_append_str(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>)); + yp_buffer_append_bytes(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>)); yp_buffer_append_str(buffer, "\"", 1); <%- when NodeListParam -%> yp_buffer_append_str(buffer, "[", 1); diff --git a/yarp/templates/src/serialize.c.erb b/yarp/templates/src/serialize.c.erb index 73ad0c9fe77657..635757b982cee1 100644 --- a/yarp/templates/src/serialize.c.erb +++ b/yarp/templates/src/serialize.c.erb @@ -38,7 +38,7 @@ yp_serialize_string(yp_parser_t *parser, yp_string_t *string, yp_buffer_t *buffe uint32_t length = yp_sizet_to_u32(yp_string_length(string)); yp_buffer_append_u8(buffer, 2); yp_buffer_append_u32(buffer, length); - yp_buffer_append_str(buffer, yp_string_source(string), length); + yp_buffer_append_bytes(buffer, yp_string_source(string), length); break; } case YP_STRING_MAPPED: @@ -234,7 +234,7 @@ serialize_token(void *data, yp_parser_t *parser, yp_token_t *token) { } YP_EXPORTED_FUNCTION void -yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer) { +yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer) { yp_parser_t parser; yp_parser_init(&parser, source, size, filepath); @@ -261,7 +261,7 @@ yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffe // Parse and serialize both the AST and the tokens represented by the given // source to the given buffer. YP_EXPORTED_FUNCTION void -yp_parse_lex_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata) { +yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata) { yp_parser_t parser; yp_parser_init(&parser, source, size, NULL); if (metadata) yp_parser_metadata(&parser, metadata); diff --git a/yarp/unescape.c b/yarp/unescape.c index 7cf2631b9b943f..0d7833be5dd2f5 100644 --- a/yarp/unescape.c +++ b/yarp/unescape.c @@ -5,9 +5,9 @@ /******************************************************************************/ static inline bool -yp_char_is_hexadecimal_digits(const char *c, size_t length) { +yp_char_is_hexadecimal_digits(const uint8_t *string, size_t length) { for (size_t index = 0; index < length; index++) { - if (!yp_char_is_hexadecimal_digit(c[index])) { + if (!yp_char_is_hexadecimal_digit(string[index])) { return false; } } @@ -18,10 +18,8 @@ yp_char_is_hexadecimal_digits(const char *c, size_t length) { // expensive to go through the indirection of the function pointer. Instead we // provide a fast path that will check if we can just return 1. static inline size_t -yp_char_width(yp_parser_t *parser, const char *start, const char *end) { - const unsigned char *uc = (const unsigned char *) start; - - if (parser->encoding_changed || (*uc >= 0x80)) { +yp_char_width(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { + if (parser->encoding_changed || (*start >= 0x80)) { return parser->encoding.char_width(start, end - start); } else { return 1; @@ -33,7 +31,7 @@ yp_char_width(yp_parser_t *parser, const char *start, const char *end) { /******************************************************************************/ // This is a lookup table for unescapes that only take up a single character. -static const unsigned char unescape_chars[] = { +static const uint8_t unescape_chars[] = { ['\''] = '\'', ['\\'] = '\\', ['a'] = '\a', @@ -60,9 +58,8 @@ static const bool ascii_printable_chars[] = { }; static inline bool -char_is_ascii_printable(const char c) { - unsigned char v = (unsigned char) c; - return (v < 0x80) && ascii_printable_chars[v]; +char_is_ascii_printable(const uint8_t b) { + return (b < 0x80) && ascii_printable_chars[b]; } /******************************************************************************/ @@ -72,37 +69,37 @@ char_is_ascii_printable(const char c) { // Scan the 1-3 digits of octal into the value. Returns the number of digits // scanned. static inline size_t -unescape_octal(const char *backslash, unsigned char *value) { - *value = (unsigned char) (backslash[1] - '0'); +unescape_octal(const uint8_t *backslash, uint8_t *value) { + *value = (uint8_t) (backslash[1] - '0'); if (!yp_char_is_octal_digit(backslash[2])) { return 2; } - *value = (unsigned char) ((*value << 3) | (backslash[2] - '0')); + *value = (uint8_t) ((*value << 3) | (backslash[2] - '0')); if (!yp_char_is_octal_digit(backslash[3])) { return 3; } - *value = (unsigned char) ((*value << 3) | (backslash[3] - '0')); + *value = (uint8_t) ((*value << 3) | (backslash[3] - '0')); return 4; } // Convert a hexadecimal digit into its equivalent value. -static inline unsigned char -unescape_hexadecimal_digit(const char value) { - return (unsigned char) ((value <= '9') ? (value - '0') : (value & 0x7) + 9); +static inline uint8_t +unescape_hexadecimal_digit(const uint8_t value) { + return (uint8_t) ((value <= '9') ? (value - '0') : (value & 0x7) + 9); } // Scan the 1-2 digits of hexadecimal into the value. Returns the number of // digits scanned. static inline size_t -unescape_hexadecimal(const char *backslash, unsigned char *value) { +unescape_hexadecimal(const uint8_t *backslash, uint8_t *value) { *value = unescape_hexadecimal_digit(backslash[2]); if (!yp_char_is_hexadecimal_digit(backslash[3])) { return 3; } - *value = (unsigned char) ((*value << 4) | unescape_hexadecimal_digit(backslash[3])); + *value = (uint8_t) ((*value << 4) | unescape_hexadecimal_digit(backslash[3])); return 4; } @@ -110,7 +107,7 @@ unescape_hexadecimal(const char *backslash, unsigned char *value) { // digits scanned. This function assumes that the characters have already been // validated. static inline void -unescape_unicode(const char *string, size_t length, uint32_t *value) { +unescape_unicode(const uint8_t *string, size_t length, uint32_t *value) { *value = 0; for (size_t index = 0; index < length; index++) { if (index != 0) *value <<= 4; @@ -122,27 +119,25 @@ unescape_unicode(const char *string, size_t length, uint32_t *value) { // 32-bit value to write. Writes the UTF-8 representation of the value to the // string and returns the number of bytes written. static inline size_t -unescape_unicode_write(char *dest, uint32_t value, const char *start, const char *end, yp_list_t *error_list) { - unsigned char *bytes = (unsigned char *) dest; - +unescape_unicode_write(uint8_t *dest, uint32_t value, const uint8_t *start, const uint8_t *end, yp_list_t *error_list) { if (value <= 0x7F) { // 0xxxxxxx - bytes[0] = (unsigned char) value; + dest[0] = (uint8_t) value; return 1; } if (value <= 0x7FF) { // 110xxxxx 10xxxxxx - bytes[0] = (unsigned char) (0xC0 | (value >> 6)); - bytes[1] = (unsigned char) (0x80 | (value & 0x3F)); + dest[0] = (uint8_t) (0xC0 | (value >> 6)); + dest[1] = (uint8_t) (0x80 | (value & 0x3F)); return 2; } if (value <= 0xFFFF) { // 1110xxxx 10xxxxxx 10xxxxxx - bytes[0] = (unsigned char) (0xE0 | (value >> 12)); - bytes[1] = (unsigned char) (0x80 | ((value >> 6) & 0x3F)); - bytes[2] = (unsigned char) (0x80 | (value & 0x3F)); + dest[0] = (uint8_t) (0xE0 | (value >> 12)); + dest[1] = (uint8_t) (0x80 | ((value >> 6) & 0x3F)); + dest[2] = (uint8_t) (0x80 | (value & 0x3F)); return 3; } @@ -150,10 +145,10 @@ unescape_unicode_write(char *dest, uint32_t value, const char *start, const char // the input is invalid. if (value <= 0x10FFFF) { // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx - bytes[0] = (unsigned char) (0xF0 | (value >> 18)); - bytes[1] = (unsigned char) (0x80 | ((value >> 12) & 0x3F)); - bytes[2] = (unsigned char) (0x80 | ((value >> 6) & 0x3F)); - bytes[3] = (unsigned char) (0x80 | (value & 0x3F)); + dest[0] = (uint8_t) (0xF0 | (value >> 18)); + dest[1] = (uint8_t) (0x80 | ((value >> 12) & 0x3F)); + dest[2] = (uint8_t) (0x80 | ((value >> 6) & 0x3F)); + dest[3] = (uint8_t) (0x80 | (value & 0x3F)); return 4; } @@ -161,9 +156,9 @@ unescape_unicode_write(char *dest, uint32_t value, const char *start, const char // want to just crash, so instead we'll add an error to the error list and put // in a replacement character instead. yp_diagnostic_list_append(error_list, start, end, "Invalid Unicode escape sequence."); - bytes[0] = 0xEF; - bytes[1] = 0xBF; - bytes[2] = 0xBD; + dest[0] = 0xEF; + dest[1] = 0xBF; + dest[2] = 0xBD; return 3; } @@ -175,24 +170,22 @@ typedef enum { } yp_unescape_flag_t; // Unescape a single character value based on the given flags. -static inline unsigned char -unescape_char(const unsigned char value, const unsigned char flags) { - unsigned char unescaped = value; - +static inline uint8_t +unescape_char(uint8_t value, const uint8_t flags) { if (flags & YP_UNESCAPE_FLAG_CONTROL) { - unescaped &= 0x1f; + value &= 0x1f; } if (flags & YP_UNESCAPE_FLAG_META) { - unescaped |= 0x80; + value |= 0x80; } - return unescaped; + return value; } // Read a specific escape sequence into the given destination. -static const char * -unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backslash, const char *end, const unsigned char flags, bool write_to_str) { +static const uint8_t * +unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t *backslash, const uint8_t *end, const uint8_t flags, bool write_to_str) { switch (backslash[1]) { case 'a': case 'b': @@ -204,27 +197,27 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs case 't': case 'v': if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(unescape_chars[(unsigned char) backslash[1]], flags); + dest[(*dest_length)++] = unescape_char(unescape_chars[backslash[1]], flags); } return backslash + 2; // \nnn octal bit pattern, where nnn is 1-3 octal digits ([0-7]) case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - unsigned char value; - const char *cursor = backslash + unescape_octal(backslash, &value); + uint8_t value; + const uint8_t *cursor = backslash + unescape_octal(backslash, &value); if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(value, flags); + dest[(*dest_length)++] = unescape_char(value, flags); } return cursor; } // \xnn hexadecimal bit pattern, where nn is 1-2 hexadecimal digits ([0-9a-fA-F]) case 'x': { - unsigned char value; - const char *cursor = backslash + unescape_hexadecimal(backslash, &value); + uint8_t value; + const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value); if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(value, flags); + dest[(*dest_length)++] = unescape_char(value, flags); } return cursor; } @@ -237,14 +230,14 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs } if ((backslash + 3) < end && backslash[2] == '{') { - const char *unicode_cursor = backslash + 3; - const char *extra_codepoints_start = NULL; + const uint8_t *unicode_cursor = backslash + 3; + const uint8_t *extra_codepoints_start = NULL; int codepoints_count = 0; unicode_cursor += yp_strspn_whitespace(unicode_cursor, end - unicode_cursor); while ((*unicode_cursor != '}') && (unicode_cursor < end)) { - const char *unicode_start = unicode_cursor; + const uint8_t *unicode_start = unicode_cursor; size_t hexadecimal_length = yp_strspn_hexadecimal_digit(unicode_cursor, end - unicode_cursor); // \u{nnnn} character literal allows only 1-6 hexadecimal digits @@ -311,7 +304,7 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); case '?': if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(0x7f, flags); + dest[(*dest_length)++] = unescape_char(0x7f, flags); } return backslash + 3; default: { @@ -321,7 +314,7 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs } if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char((const unsigned char) backslash[2], flags | YP_UNESCAPE_FLAG_CONTROL); + dest[(*dest_length)++] = unescape_char(backslash[2], flags | YP_UNESCAPE_FLAG_CONTROL); } return backslash + 3; } @@ -349,7 +342,7 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); case '?': if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char(0x7f, flags); + dest[(*dest_length)++] = unescape_char(0x7f, flags); } return backslash + 4; default: @@ -359,7 +352,7 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs } if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char((const unsigned char) backslash[3], flags | YP_UNESCAPE_FLAG_CONTROL); + dest[(*dest_length)++] = unescape_char(backslash[3], flags | YP_UNESCAPE_FLAG_CONTROL); } return backslash + 4; } @@ -388,7 +381,7 @@ unescape(yp_parser_t *parser, char *dest, size_t *dest_length, const char *backs if (char_is_ascii_printable(backslash[3])) { if (write_to_str) { - dest[(*dest_length)++] = (char) unescape_char((const unsigned char) backslash[3], flags | YP_UNESCAPE_FLAG_META); + dest[(*dest_length)++] = unescape_char(backslash[3], flags | YP_UNESCAPE_FLAG_META); } return backslash + 4; } @@ -454,7 +447,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc return; } - const char *backslash = yp_memchr(string->source, '\\', string->length, parser->encoding_changed, &parser->encoding); + const uint8_t *backslash = yp_memchr(string->source, '\\', string->length, parser->encoding_changed, &parser->encoding); if (backslash == NULL) { // Here there are no escapes, so we can reference the source directly. @@ -463,21 +456,21 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // Here we have found an escape character, so we need to handle all escapes // within the string. - char *allocated = malloc(string->length); + uint8_t *allocated = malloc(string->length); if (allocated == NULL) { yp_diagnostic_list_append(&parser->error_list, string->source, string->source + string->length, "Failed to allocate memory for unescaping."); return; } // This is the memory address where we're putting the unescaped string. - char *dest = allocated; + uint8_t *dest = allocated; size_t dest_length = 0; // This is the current position in the source string that we're looking at. // It's going to move along behind the backslash so that we can copy each // segment of the string that doesn't contain an escape. - const char *cursor = string->source; - const char *end = string->source + string->length; + const uint8_t *cursor = string->source; + const uint8_t *end = string->source + string->length; // For each escape found in the source string, we will handle it and update // the moving cursor->backslash window. @@ -496,7 +489,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc switch (backslash[1]) { case '\\': case '\'': - dest[dest_length++] = (char) unescape_chars[(unsigned char) backslash[1]]; + dest[dest_length++] = unescape_chars[backslash[1]]; cursor = backslash + 2; break; default: @@ -542,7 +535,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // actually perform any string manipulations. Instead, it calculates how long // the unescaped character is, and returns that value size_t -yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { +yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *backslash, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { assert(unescape_type != YP_UNESCAPE_NONE); switch (backslash[1]) { @@ -558,11 +551,11 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_ // handle all of the different unescapes. assert(unescape_type == YP_UNESCAPE_ALL); - unsigned char flags = YP_UNESCAPE_FLAG_NONE; + uint8_t flags = YP_UNESCAPE_FLAG_NONE; if (expect_single_codepoint) flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; - const char *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, false); + const uint8_t *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, false); assert(cursor > backslash); return (size_t) (cursor - backslash); @@ -574,7 +567,7 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const char *backslash, yp_ // string, a type of unescaping, and a pointer to a result string. It returns a // boolean indicating whether or not the unescaping was successful. YP_EXPORTED_FUNCTION bool -yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) { +yp_unescape_string(const uint8_t *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result) { yp_parser_t parser; yp_parser_init(&parser, start, length, NULL); diff --git a/yarp/unescape.h b/yarp/unescape.h index 30c433febd525f..bf8b7e83ec08e6 100644 --- a/yarp/unescape.h +++ b/yarp/unescape.h @@ -35,10 +35,10 @@ YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_ // Accepts a source string and a type of unescaping and returns the unescaped version. // The caller must yp_string_free(result); after calling this function. -YP_EXPORTED_FUNCTION bool yp_unescape_string(const char *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result); +YP_EXPORTED_FUNCTION bool yp_unescape_string(const uint8_t *start, size_t length, yp_unescape_type_t unescape_type, yp_string_t *result); // Returns the number of bytes that encompass the first escape sequence in the // given string. -size_t yp_unescape_calculate_difference(yp_parser_t *parser, const char *value, yp_unescape_type_t unescape_type, bool expect_single_codepoint); +size_t yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *value, yp_unescape_type_t unescape_type, bool expect_single_codepoint); #endif diff --git a/yarp/util/yp_buffer.c b/yarp/util/yp_buffer.c index c9f06ae1c6dc0e..15cdef74f88073 100644 --- a/yarp/util/yp_buffer.c +++ b/yarp/util/yp_buffer.c @@ -63,8 +63,13 @@ yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length) { // Append a string to the buffer. void yp_buffer_append_str(yp_buffer_t *buffer, const char *value, size_t length) { - const void *source = value; - yp_buffer_append(buffer, source, length); + yp_buffer_append(buffer, value, length); +} + +// Append a list of bytes to the buffer. +void +yp_buffer_append_bytes(yp_buffer_t *buffer, const uint8_t *value, size_t length) { + yp_buffer_append(buffer, (const char *) value, length); } // Append a single byte to the buffer. diff --git a/yarp/util/yp_buffer.h b/yarp/util/yp_buffer.h index 095f62a833c565..c388e8d5ce0000 100644 --- a/yarp/util/yp_buffer.h +++ b/yarp/util/yp_buffer.h @@ -36,6 +36,9 @@ void yp_buffer_append_zeroes(yp_buffer_t *buffer, size_t length); // Append a string to the buffer. void yp_buffer_append_str(yp_buffer_t *buffer, const char *value, size_t length); +// Append a list of bytes to the buffer. +void yp_buffer_append_bytes(yp_buffer_t *buffer, const uint8_t *value, size_t length); + // Append a single byte to the buffer. void yp_buffer_append_u8(yp_buffer_t *buffer, uint8_t value); diff --git a/yarp/util/yp_char.c b/yarp/util/yp_char.c index d27a04104ec62e..e9f1ef45c20997 100644 --- a/yarp/util/yp_char.c +++ b/yarp/util/yp_char.c @@ -13,8 +13,8 @@ #define YP_NUMBER_BIT_HEXADECIMAL_DIGIT (1 << 6) #define YP_NUMBER_BIT_HEXADECIMAL_NUMBER (1 << 7) -static const unsigned char yp_char_table[256] = { -//0 1 2 3 4 5 6 7 8 9 A B C D E F +static const uint8_t yp_byte_table[256] = { +// 0 1 2 3 4 5 6 7 8 9 A B C D E F 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, 0, 0, // 0x 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 2x @@ -33,7 +33,7 @@ static const unsigned char yp_char_table[256] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Fx }; -static const unsigned char yp_number_table[256] = { +static const uint8_t yp_number_table[256] = { // 0 1 2 3 4 5 6 7 8 9 A B C D E F 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 1x @@ -54,20 +54,20 @@ static const unsigned char yp_number_table[256] = { }; static inline size_t -yp_strspn_char_kind(const char *string, ptrdiff_t length, unsigned char kind) { +yp_strspn_char_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) { if (length <= 0) return 0; size_t size = 0; size_t maximum = (size_t) length; - while (size < maximum && (yp_char_table[(unsigned char) string[size]] & kind)) size++; + while (size < maximum && (yp_byte_table[string[size]] & kind)) size++; return size; } // Returns the number of characters at the start of the string that are // whitespace. Disallows searching past the given maximum number of characters. size_t -yp_strspn_whitespace(const char *string, ptrdiff_t length) { +yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length) { return yp_strspn_char_kind(string, length, YP_CHAR_BIT_WHITESPACE); } @@ -75,13 +75,13 @@ yp_strspn_whitespace(const char *string, ptrdiff_t length) { // whitespace while also tracking the location of each newline. Disallows // searching past the given maximum number of characters. size_t -yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) { +yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline) { if (length <= 0) return 0; size_t size = 0; size_t maximum = (size_t) length; - while (size < maximum && (yp_char_table[(unsigned char) string[size]] & YP_CHAR_BIT_WHITESPACE)) { + while (size < maximum && (yp_byte_table[string[size]] & YP_CHAR_BIT_WHITESPACE)) { if (string[size] == '\n') { if (stop_at_newline) { return size + 1; @@ -100,42 +100,42 @@ yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_l // Returns the number of characters at the start of the string that are inline // whitespace. Disallows searching past the given maximum number of characters. size_t -yp_strspn_inline_whitespace(const char *string, ptrdiff_t length) { +yp_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length) { return yp_strspn_char_kind(string, length, YP_CHAR_BIT_INLINE_WHITESPACE); } // Returns the number of characters at the start of the string that are regexp // options. Disallows searching past the given maximum number of characters. size_t -yp_strspn_regexp_option(const char *string, ptrdiff_t length) { +yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length) { return yp_strspn_char_kind(string, length, YP_CHAR_BIT_REGEXP_OPTION); } static inline bool -yp_char_is_char_kind(const char c, unsigned char kind) { - return (yp_char_table[(unsigned char) c] & kind) != 0; +yp_char_is_char_kind(const uint8_t b, uint8_t kind) { + return (yp_byte_table[b] & kind) != 0; } // Returns true if the given character is a whitespace character. bool -yp_char_is_whitespace(const char c) { - return yp_char_is_char_kind(c, YP_CHAR_BIT_WHITESPACE); +yp_char_is_whitespace(const uint8_t b) { + return yp_char_is_char_kind(b, YP_CHAR_BIT_WHITESPACE); } // Returns true if the given character is an inline whitespace character. bool -yp_char_is_inline_whitespace(const char c) { - return yp_char_is_char_kind(c, YP_CHAR_BIT_INLINE_WHITESPACE); +yp_char_is_inline_whitespace(const uint8_t b) { + return yp_char_is_char_kind(b, YP_CHAR_BIT_INLINE_WHITESPACE); } static inline size_t -yp_strspn_number_kind(const char *string, ptrdiff_t length, unsigned char kind) { +yp_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) { if (length <= 0) return 0; size_t size = 0; size_t maximum = (size_t) length; - while (size < maximum && (yp_number_table[(unsigned char) string[size]] & kind)) size++; + while (size < maximum && (yp_number_table[string[size]] & kind)) size++; return size; } @@ -143,7 +143,7 @@ yp_strspn_number_kind(const char *string, ptrdiff_t length, unsigned char kind) // digits or underscores. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_binary_number(const char *string, ptrdiff_t length) { +yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_BINARY_NUMBER); } @@ -151,14 +151,14 @@ yp_strspn_binary_number(const char *string, ptrdiff_t length) { // digits or underscores. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_octal_number(const char *string, ptrdiff_t length) { +yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_OCTAL_NUMBER); } // Returns the number of characters at the start of the string that are decimal // digits. Disallows searching past the given maximum number of characters. size_t -yp_strspn_decimal_digit(const char *string, ptrdiff_t length) { +yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_DIGIT); } @@ -166,7 +166,7 @@ yp_strspn_decimal_digit(const char *string, ptrdiff_t length) { // digits or underscores. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_decimal_number(const char *string, ptrdiff_t length) { +yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_DECIMAL_NUMBER); } @@ -174,7 +174,7 @@ yp_strspn_decimal_number(const char *string, ptrdiff_t length) { // hexadecimal digits. Disallows searching past the given maximum number of // characters. size_t -yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length) { +yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_DIGIT); } @@ -182,37 +182,37 @@ yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length) { // hexadecimal digits or underscores. Disallows searching past the given maximum // number of characters. size_t -yp_strspn_hexadecimal_number(const char *string, ptrdiff_t length) { +yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length) { return yp_strspn_number_kind(string, length, YP_NUMBER_BIT_HEXADECIMAL_NUMBER); } static inline bool -yp_char_is_number_kind(const char c, unsigned char kind) { - return (yp_number_table[(unsigned char) c] & kind) != 0; +yp_char_is_number_kind(const uint8_t b, uint8_t kind) { + return (yp_number_table[b] & kind) != 0; } // Returns true if the given character is a binary digit. bool -yp_char_is_binary_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_BINARY_DIGIT); +yp_char_is_binary_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_BINARY_DIGIT); } // Returns true if the given character is an octal digit. bool -yp_char_is_octal_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_OCTAL_DIGIT); +yp_char_is_octal_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_OCTAL_DIGIT); } // Returns true if the given character is a decimal digit. bool -yp_char_is_decimal_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_DECIMAL_DIGIT); +yp_char_is_decimal_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_DECIMAL_DIGIT); } // Returns true if the given character is a hexadecimal digit. bool -yp_char_is_hexadecimal_digit(const char c) { - return yp_char_is_number_kind(c, YP_NUMBER_BIT_HEXADECIMAL_DIGIT); +yp_char_is_hexadecimal_digit(const uint8_t b) { + return yp_char_is_number_kind(b, YP_NUMBER_BIT_HEXADECIMAL_DIGIT); } #undef YP_CHAR_BIT_WHITESPACE diff --git a/yarp/util/yp_char.h b/yarp/util/yp_char.h index 010d34d6693c56..67ba31d34d687a 100644 --- a/yarp/util/yp_char.h +++ b/yarp/util/yp_char.h @@ -9,67 +9,67 @@ // Returns the number of characters at the start of the string that are // whitespace. Disallows searching past the given maximum number of characters. -size_t yp_strspn_whitespace(const char *string, ptrdiff_t length); +size_t yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are // whitespace while also tracking the location of each newline. Disallows // searching past the given maximum number of characters. size_t -yp_strspn_whitespace_newlines(const char *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool); +yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list, bool stop_at_newline); // Returns the number of characters at the start of the string that are inline // whitespace. Disallows searching past the given maximum number of characters. -size_t yp_strspn_inline_whitespace(const char *string, ptrdiff_t length); +size_t yp_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are decimal // digits. Disallows searching past the given maximum number of characters. -size_t yp_strspn_decimal_digit(const char *string, ptrdiff_t length); +size_t yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are // hexadecimal digits. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_hexadecimal_digit(const char *string, ptrdiff_t length); +size_t yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are octal // digits or underscores. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_octal_number(const char *string, ptrdiff_t length); +size_t yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are decimal // digits or underscores. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_decimal_number(const char *string, ptrdiff_t length); +size_t yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are // hexadecimal digits or underscores. Disallows searching past the given maximum // number of characters. -size_t yp_strspn_hexadecimal_number(const char *string, ptrdiff_t length); +size_t yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are regexp // options. Disallows searching past the given maximum number of characters. -size_t yp_strspn_regexp_option(const char *string, ptrdiff_t length); +size_t yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length); // Returns the number of characters at the start of the string that are binary // digits or underscores. Disallows searching past the given maximum number of // characters. -size_t yp_strspn_binary_number(const char *string, ptrdiff_t length); +size_t yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length); // Returns true if the given character is a whitespace character. -bool yp_char_is_whitespace(const char c); +bool yp_char_is_whitespace(const uint8_t b); // Returns true if the given character is an inline whitespace character. -bool yp_char_is_inline_whitespace(const char c); +bool yp_char_is_inline_whitespace(const uint8_t b); // Returns true if the given character is a binary digit. -bool yp_char_is_binary_digit(const char c); +bool yp_char_is_binary_digit(const uint8_t b); // Returns true if the given character is an octal digit. -bool yp_char_is_octal_digit(const char c); +bool yp_char_is_octal_digit(const uint8_t b); // Returns true if the given character is a decimal digit. -bool yp_char_is_decimal_digit(const char c); +bool yp_char_is_decimal_digit(const uint8_t b); // Returns true if the given character is a hexadecimal digit. -bool yp_char_is_hexadecimal_digit(const char c); +bool yp_char_is_hexadecimal_digit(const uint8_t b); #endif diff --git a/yarp/util/yp_constant_pool.c b/yarp/util/yp_constant_pool.c index df46c769921322..3ad241a9d1bb54 100644 --- a/yarp/util/yp_constant_pool.c +++ b/yarp/util/yp_constant_pool.c @@ -48,12 +48,12 @@ yp_constant_id_list_free(yp_constant_id_list_t *list) { // A relatively simple hash function (djb2) that is used to hash strings. We are // optimizing here for simplicity and speed. static inline size_t -yp_constant_pool_hash(const char *start, size_t length) { +yp_constant_pool_hash(const uint8_t *start, size_t length) { // This is a prime number used as the initial value for the hash function. size_t value = 5381; for (size_t index = 0; index < length; index++) { - value = ((value << 5) + value) + ((unsigned char) start[index]); + value = ((value << 5) + value) + start[index]; } return value; @@ -109,7 +109,7 @@ yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity) { // Insert a constant into a constant pool. Returns the id of the constant, or 0 // if any potential calls to resize fail. yp_constant_id_t -yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t length) { +yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length) { if (pool->size >= (pool->capacity / 4 * 3)) { if (!yp_constant_pool_resize(pool)) return 0; } diff --git a/yarp/util/yp_constant_pool.h b/yarp/util/yp_constant_pool.h index 3726ecc44a28d6..1ac23cf88bbf28 100644 --- a/yarp/util/yp_constant_pool.h +++ b/yarp/util/yp_constant_pool.h @@ -40,7 +40,7 @@ void yp_constant_id_list_free(yp_constant_id_list_t *list); typedef struct { yp_constant_id_t id; - const char *start; + const uint8_t *start; size_t length; size_t hash; } yp_constant_t; @@ -59,7 +59,7 @@ bool yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity); // Insert a constant into a constant pool. Returns the id of the constant, or 0 // if any potential calls to resize fail. -yp_constant_id_t yp_constant_pool_insert(yp_constant_pool_t *pool, const char *start, size_t length); +yp_constant_id_t yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t length); // Free the memory associated with a constant pool. void yp_constant_pool_free(yp_constant_pool_t *pool); diff --git a/yarp/util/yp_memchr.c b/yarp/util/yp_memchr.c index c323f37a6658e7..af9c14397e0b37 100644 --- a/yarp/util/yp_memchr.c +++ b/yarp/util/yp_memchr.c @@ -8,7 +8,7 @@ void * yp_memchr(const void *memory, int character, size_t number, bool encoding_changed, yp_encoding_t *encoding) { if (encoding_changed && encoding->multibyte && character >= YP_MEMCHR_TRAILING_BYTE_MINIMUM) { - const char *source = (const char *) memory; + const uint8_t *source = (const uint8_t *) memory; size_t index = 0; while (index < number) { diff --git a/yarp/util/yp_newline_list.c b/yarp/util/yp_newline_list.c index de353acf62a869..0a2050df059dd5 100644 --- a/yarp/util/yp_newline_list.c +++ b/yarp/util/yp_newline_list.c @@ -3,7 +3,7 @@ // Initialize a new newline list with the given capacity. Returns true if the // allocation of the offsets succeeds, otherwise returns false. bool -yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity) { +yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capacity) { list->offsets = (size_t *) calloc(capacity, sizeof(size_t)); if (list->offsets == NULL) return false; @@ -23,7 +23,7 @@ yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity // Append a new offset to the newline list. Returns true if the reallocation of // the offsets succeeds (if one was necessary), otherwise returns false. bool -yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { +yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor) { if (list->size == list->capacity) { list->capacity = (list->capacity * 3) / 2; list->offsets = (size_t *) realloc(list->offsets, list->capacity * sizeof(size_t)); @@ -33,6 +33,7 @@ yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { assert(*cursor == '\n'); assert(cursor >= list->start); size_t newline_offset = (size_t) (cursor - list->start + 1); + assert(list->size == 0 || newline_offset > list->offsets[list->size - 1]); list->offsets[list->size++] = newline_offset; @@ -41,7 +42,7 @@ yp_newline_list_append(yp_newline_list_t *list, const char *cursor) { // Conditionally append a new offset to the newline list, if the value passed in is a newline. bool -yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor) { +yp_newline_list_check_append(yp_newline_list_t *list, const uint8_t *cursor) { if (*cursor != '\n') { return true; } @@ -105,7 +106,7 @@ yp_newline_list_line_column_scan(yp_newline_list_t *list, size_t offset) { // list, the line and column of the closest offset less than the given offset // are returned. yp_line_column_t -yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor) { +yp_newline_list_line_column(yp_newline_list_t *list, const uint8_t *cursor) { assert(cursor >= list->start); size_t offset = (size_t) (cursor - list->start); yp_line_column_t result; diff --git a/yarp/util/yp_newline_list.h b/yarp/util/yp_newline_list.h index b7c8c1f3aace57..92313050084dc3 100644 --- a/yarp/util/yp_newline_list.h +++ b/yarp/util/yp_newline_list.h @@ -19,7 +19,7 @@ // A list of offsets of newlines in a string. The offsets are assumed to be // sorted/inserted in ascending order. typedef struct { - const char *start; + const uint8_t *start; size_t *offsets; size_t size; @@ -41,19 +41,19 @@ typedef struct { // Initialize a new newline list with the given capacity. Returns true if the // allocation of the offsets succeeds, otherwise returns false. -bool yp_newline_list_init(yp_newline_list_t *list, const char *start, size_t capacity); +bool yp_newline_list_init(yp_newline_list_t *list, const uint8_t *start, size_t capacity); // Append a new offset to the newline list. Returns true if the reallocation of // the offsets succeeds (if one was necessary), otherwise returns false. -bool yp_newline_list_append(yp_newline_list_t *list, const char *cursor); +bool yp_newline_list_append(yp_newline_list_t *list, const uint8_t *cursor); // Conditionally append a new offset to the newline list, if the value passed in is a newline. -bool yp_newline_list_check_append(yp_newline_list_t *list, const char *cursor); +bool yp_newline_list_check_append(yp_newline_list_t *list, const uint8_t *cursor); // Returns the line and column of the given offset. If the offset is not in the // list, the line and column of the closest offset less than the given offset // are returned. -yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const char *cursor); +yp_line_column_t yp_newline_list_line_column(yp_newline_list_t *list, const uint8_t *cursor); // Free the internal memory allocated for the newline list. void yp_newline_list_free(yp_newline_list_t *list); diff --git a/yarp/util/yp_string.c b/yarp/util/yp_string.c index bdd001d2b0f4f3..9ee25155a3bdfb 100644 --- a/yarp/util/yp_string.c +++ b/yarp/util/yp_string.c @@ -12,18 +12,19 @@ // Initialize a shared string that is based on initial input. void -yp_string_shared_init(yp_string_t *string, const char *start, const char *end) { +yp_string_shared_init(yp_string_t *string, const uint8_t *start, const uint8_t *end) { assert(start <= end); + *string = (yp_string_t) { .type = YP_STRING_SHARED, - .source = (char*) start, + .source = start, .length = (size_t) (end - start) }; } // Initialize an owned string that is responsible for freeing allocated memory. void -yp_string_owned_init(yp_string_t *string, char *source, size_t length) { +yp_string_owned_init(yp_string_t *string, uint8_t *source, size_t length) { *string = (yp_string_t) { .type = YP_STRING_OWNED, .source = source, @@ -36,13 +37,13 @@ void yp_string_constant_init(yp_string_t *string, const char *source, size_t length) { *string = (yp_string_t) { .type = YP_STRING_CONSTANT, - .source = (char*) source, + .source = (const uint8_t *) source, .length = length }; } static void -yp_string_mapped_init_internal(yp_string_t *string, char *source, size_t length) { +yp_string_mapped_init_internal(yp_string_t *string, uint8_t *source, size_t length) { *string = (yp_string_t) { .type = YP_STRING_MAPPED, .source = source, @@ -67,13 +68,13 @@ yp_string_ensure_owned(yp_string_t *string) { if (string->type == YP_STRING_OWNED) return; size_t length = yp_string_length(string); - const char *source = yp_string_source(string); + const uint8_t *source = yp_string_source(string); - char *memory = malloc(length); + uint8_t *memory = malloc(length); if (!memory) return; yp_string_owned_init(string, memory, length); - memcpy(string->source, source, length); + memcpy((void *) string->source, source, length); } // Returns the length associated with the string. @@ -83,7 +84,7 @@ yp_string_length(const yp_string_t *string) { } // Returns the start pointer associated with the string. -YP_EXPORTED_FUNCTION const char * +YP_EXPORTED_FUNCTION const uint8_t * yp_string_source(const yp_string_t *string) { return string->source; } @@ -91,15 +92,16 @@ yp_string_source(const yp_string_t *string) { // Free the associated memory of the given string. YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string) { + void *memory = (void *) string->source; + if (string->type == YP_STRING_OWNED) { - free(string->source); + free(memory); } else if (string->type == YP_STRING_MAPPED && string->length) { - void *memory = (void *) string->source; - #if defined(_WIN32) +#if defined(_WIN32) UnmapViewOfFile(memory); - #else +#else munmap(memory, string->length); - #endif +#endif } } @@ -126,8 +128,8 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) { // the source to a constant empty string and return. if (file_size == 0) { CloseHandle(file); - char empty_string[] = ""; - yp_string_mapped_init_internal(string, empty_string, 0); + uint8_t empty[] = ""; + yp_string_mapped_init_internal(string, empty, 0); return true; } @@ -140,7 +142,7 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) { } // Map the file into memory. - char *source = (char *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + uint8_t *source = (uint8_t *) MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); CloseHandle(mapping); CloseHandle(file); @@ -169,12 +171,12 @@ yp_string_mapped_init(yp_string_t *string, const char *filepath) { // mmap the file descriptor to virtually get the contents size_t size = (size_t) sb.st_size; - char *source = NULL; + uint8_t *source = NULL; if (size == 0) { close(fd); - char empty_string[] = ""; - yp_string_mapped_init_internal(string, empty_string, 0); + uint8_t empty[] = ""; + yp_string_mapped_init_internal(string, empty, 0); return true; } diff --git a/yarp/util/yp_string.h b/yarp/util/yp_string.h index 8d5a925232df59..bcdf8b66d99c60 100644 --- a/yarp/util/yp_string.h +++ b/yarp/util/yp_string.h @@ -12,17 +12,17 @@ // This struct represents a string value. typedef struct { enum { YP_STRING_SHARED, YP_STRING_OWNED, YP_STRING_CONSTANT, YP_STRING_MAPPED } type; - char *source; + const uint8_t *source; size_t length; } yp_string_t; #define YP_EMPTY_STRING ((yp_string_t) { .type = YP_STRING_CONSTANT, .source = NULL, .length = 0 }) // Initialize a shared string that is based on initial input. -void yp_string_shared_init(yp_string_t *string, const char *start, const char *end); +void yp_string_shared_init(yp_string_t *string, const uint8_t *start, const uint8_t *end); // Initialize an owned string that is responsible for freeing allocated memory. -void yp_string_owned_init(yp_string_t *string, char *source, size_t length); +void yp_string_owned_init(yp_string_t *string, uint8_t *source, size_t length); // Initialize a constant string that doesn't own its memory source. void yp_string_constant_init(yp_string_t *string, const char *source, size_t length); @@ -49,7 +49,7 @@ void yp_string_ensure_owned(yp_string_t *string); YP_EXPORTED_FUNCTION size_t yp_string_length(const yp_string_t *string); // Returns the start pointer associated with the string. -YP_EXPORTED_FUNCTION const char * yp_string_source(const yp_string_t *string); +YP_EXPORTED_FUNCTION const uint8_t * yp_string_source(const yp_string_t *string); // Free the associated memory of the given string. YP_EXPORTED_FUNCTION void yp_string_free(yp_string_t *string); diff --git a/yarp/util/yp_string_list.c b/yarp/util/yp_string_list.c index 74822729ff2e22..b03a3d259b1d45 100644 --- a/yarp/util/yp_string_list.c +++ b/yarp/util/yp_string_list.c @@ -1,11 +1,5 @@ #include "yarp/util/yp_string_list.h" -// Allocate a new yp_string_list_t. -yp_string_list_t * -yp_string_list_alloc(void) { - return (yp_string_list_t *) malloc(sizeof(yp_string_list_t)); -} - // Initialize a yp_string_list_t with its default values. void yp_string_list_init(yp_string_list_t *string_list) { diff --git a/yarp/util/yp_string_list.h b/yarp/util/yp_string_list.h index ae252eb5d5b3e1..0009a27a60cce4 100644 --- a/yarp/util/yp_string_list.h +++ b/yarp/util/yp_string_list.h @@ -13,9 +13,6 @@ typedef struct { size_t capacity; } yp_string_list_t; -// Allocate a new yp_string_list_t. -yp_string_list_t * yp_string_list_alloc(void); - // Initialize a yp_string_list_t with its default values. YP_EXPORTED_FUNCTION void yp_string_list_init(yp_string_list_t *string_list); diff --git a/yarp/util/yp_strncasecmp.c b/yarp/util/yp_strncasecmp.c index 899bba4eaa633f..1cbaf904f49c73 100644 --- a/yarp/util/yp_strncasecmp.c +++ b/yarp/util/yp_strncasecmp.c @@ -1,18 +1,15 @@ #include #include +#include int -yp_strncasecmp(const char *string1, const char *string2, size_t length) { +yp_strncasecmp(const uint8_t *string1, const uint8_t *string2, size_t length) { size_t offset = 0; int difference = 0; while (offset < length && string1[offset] != '\0') { if (string2[offset] == '\0') return string1[offset]; - - unsigned char left = (unsigned char) string1[offset]; - unsigned char right = (unsigned char) string2[offset]; - - if ((difference = tolower(left) - tolower(right)) != 0) return difference; + if ((difference = tolower(string1[offset]) - tolower(string2[offset])) != 0) return difference; offset++; } diff --git a/yarp/util/yp_strpbrk.c b/yarp/util/yp_strpbrk.c index 14a032f3f59708..7c0015d2890fcf 100644 --- a/yarp/util/yp_strpbrk.c +++ b/yarp/util/yp_strpbrk.c @@ -1,12 +1,12 @@ #include "yarp/util/yp_strpbrk.h" // This is the slow path that does care about the encoding. -static inline const char * -yp_strpbrk_multi_byte(yp_parser_t *parser, const char *source, const char *charset, size_t maximum) { +static inline const uint8_t * +yp_strpbrk_multi_byte(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) { size_t index = 0; while (index < maximum) { - if (strchr(charset, source[index]) != NULL) { + if (strchr((const char *) charset, source[index]) != NULL) { return source + index; } @@ -22,12 +22,12 @@ yp_strpbrk_multi_byte(yp_parser_t *parser, const char *source, const char *chars } // This is the fast path that does not care about the encoding. -static inline const char * -yp_strpbrk_single_byte(const char *source, const char *charset, size_t maximum) { +static inline const uint8_t * +yp_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t maximum) { size_t index = 0; while (index < maximum) { - if (strchr(charset, source[index]) != NULL) { + if (strchr((const char *) charset, source[index]) != NULL) { return source + index; } @@ -54,8 +54,8 @@ yp_strpbrk_single_byte(const char *source, const char *charset, size_t maximum) // characters that are trailing bytes of multi-byte characters. For example, in // Shift-JIS, the backslash character can be a trailing byte. In that case we // need to take a slower path and iterate one multi-byte character at a time. -const char * -yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length) { +const uint8_t * +yp_strpbrk(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) { if (length <= 0) { return NULL; } else if (parser->encoding_changed && parser->encoding.multibyte) { diff --git a/yarp/util/yp_strpbrk.h b/yarp/util/yp_strpbrk.h index 7a664d5452115f..d0bdd5bec000cc 100644 --- a/yarp/util/yp_strpbrk.h +++ b/yarp/util/yp_strpbrk.h @@ -24,6 +24,6 @@ // characters that are trailing bytes of multi-byte characters. For example, in // Shift-JIS, the backslash character can be a trailing byte. In that case we // need to take a slower path and iterate one multi-byte character at a time. -const char * yp_strpbrk(yp_parser_t *parser, const char *source, const char *charset, ptrdiff_t length); +const uint8_t * yp_strpbrk(yp_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length); #endif diff --git a/yarp/yarp.c b/yarp/yarp.c index 2422fb45719e59..c9d7cd08b42ab7 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -167,8 +167,8 @@ debug_token(yp_token_t * token) { // Returns the incrementor character that should be used to increment the // nesting count if one is possible. -static inline char -lex_mode_incrementor(const char start) { +static inline uint8_t +lex_mode_incrementor(const uint8_t start) { switch (start) { case '(': case '[': @@ -182,8 +182,8 @@ lex_mode_incrementor(const char start) { // Returns the matching character that should be used to terminate a list // beginning with the given character. -static inline char -lex_mode_terminator(const char start) { +static inline uint8_t +lex_mode_terminator(const uint8_t start) { switch (start) { case '(': return ')'; @@ -221,9 +221,9 @@ lex_mode_push(yp_parser_t *parser, yp_lex_mode_t lex_mode) { // Push on a new list lex mode. static inline bool -lex_mode_push_list(yp_parser_t *parser, bool interpolation, char delimiter) { - char incrementor = lex_mode_incrementor(delimiter); - char terminator = lex_mode_terminator(delimiter); +lex_mode_push_list(yp_parser_t *parser, bool interpolation, uint8_t delimiter) { + uint8_t incrementor = lex_mode_incrementor(delimiter); + uint8_t terminator = lex_mode_terminator(delimiter); yp_lex_mode_t lex_mode = { .mode = YP_LEX_LIST, @@ -237,7 +237,7 @@ lex_mode_push_list(yp_parser_t *parser, bool interpolation, char delimiter) { // These are the places where we need to split up the content of the list. // We'll use strpbrk to find the first of these characters. - char *breakpoints = lex_mode.as.list.breakpoints; + uint8_t *breakpoints = lex_mode.as.list.breakpoints; memcpy(breakpoints, "\\ \t\f\r\v\n\0\0\0", sizeof(lex_mode.as.list.breakpoints)); // Now we'll add the terminator to the list of breakpoints. @@ -260,7 +260,7 @@ lex_mode_push_list(yp_parser_t *parser, bool interpolation, char delimiter) { // Push on a new regexp lex mode. static inline bool -lex_mode_push_regexp(yp_parser_t *parser, char incrementor, char terminator) { +lex_mode_push_regexp(yp_parser_t *parser, uint8_t incrementor, uint8_t terminator) { yp_lex_mode_t lex_mode = { .mode = YP_LEX_REGEXP, .as.regexp = { @@ -273,7 +273,7 @@ lex_mode_push_regexp(yp_parser_t *parser, char incrementor, char terminator) { // These are the places where we need to split up the content of the // regular expression. We'll use strpbrk to find the first of these // characters. - char *breakpoints = lex_mode.as.regexp.breakpoints; + uint8_t *breakpoints = lex_mode.as.regexp.breakpoints; memcpy(breakpoints, "\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); // First we'll add the terminator. @@ -289,7 +289,7 @@ lex_mode_push_regexp(yp_parser_t *parser, char incrementor, char terminator) { // Push on a new string lex mode. static inline bool -lex_mode_push_string(yp_parser_t *parser, bool interpolation, bool label_allowed, char incrementor, char terminator) { +lex_mode_push_string(yp_parser_t *parser, bool interpolation, bool label_allowed, uint8_t incrementor, uint8_t terminator) { yp_lex_mode_t lex_mode = { .mode = YP_LEX_STRING, .as.string = { @@ -303,7 +303,7 @@ lex_mode_push_string(yp_parser_t *parser, bool interpolation, bool label_allowed // These are the places where we need to split up the content of the // string. We'll use strpbrk to find the first of these characters. - char *breakpoints = lex_mode.as.string.breakpoints; + uint8_t *breakpoints = lex_mode.as.string.breakpoints; memcpy(breakpoints, "\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); // Now add in the terminator. @@ -423,7 +423,7 @@ debug_lex_state_set(yp_parser_t *parser, yp_lex_state_t state, char const * call // Retrieve the constant pool id for the given location. static inline yp_constant_id_t -yp_parser_constant_id_location(yp_parser_t *parser, const char *start, const char *end) { +yp_parser_constant_id_location(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { return yp_constant_pool_insert(&parser->constant_pool, start, (size_t) (end - start)); } @@ -615,7 +615,7 @@ yp_regular_expression_flags_create(const yp_token_t *closing) { yp_node_flags_t flags = 0; if (closing->type == YP_TOKEN_REGEXP_END) { - for (const char *flag = closing->start + 1; flag < closing->end; flag++) { + for (const uint8_t *flag = closing->start + 1; flag < closing->end; flag++) { switch (*flag) { case 'i': flags |= YP_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE; break; case 'm': flags |= YP_REGULAR_EXPRESSION_FLAGS_MULTI_LINE; break; @@ -657,7 +657,7 @@ yp_alloc_node(YP_ATTRIBUTE_UNUSED yp_parser_t *parser, size_t size) { // Allocate a new MissingNode node. static yp_missing_node_t * -yp_missing_node_create(yp_parser_t *parser, const char *start, const char *end) { +yp_missing_node_create(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { yp_missing_node_t *node = YP_ALLOC_NODE(parser, yp_missing_node_t); *node = (yp_missing_node_t) {{ .type = YP_NODE_MISSING_NODE, .location = { .start = start, .end = end } }}; return node; @@ -926,7 +926,7 @@ yp_array_pattern_node_requireds_append(yp_array_pattern_node_t *node, yp_node_t static yp_assoc_node_t * yp_assoc_node_create(yp_parser_t *parser, yp_node_t *key, const yp_token_t *operator, yp_node_t *value) { yp_assoc_node_t *node = YP_ALLOC_NODE(parser, yp_assoc_node_t); - const char *end; + const uint8_t *end; if (value != NULL) { end = value->location.end; @@ -1110,7 +1110,7 @@ static yp_block_parameters_node_t * yp_block_parameters_node_create(yp_parser_t *parser, yp_parameters_node_t *parameters, const yp_token_t *opening) { yp_block_parameters_node_t *node = YP_ALLOC_NODE(parser, yp_block_parameters_node_t); - const char *start; + const uint8_t *start; if (opening->type != YP_TOKEN_NOT_PROVIDED) { start = opening->start; } else if (parameters != NULL) { @@ -1119,7 +1119,7 @@ yp_block_parameters_node_create(yp_parser_t *parser, yp_parameters_node_t *param start = NULL; } - const char *end; + const uint8_t *end; if (parameters != NULL) { end = parameters->base.location.end; } else if (opening->type != YP_TOKEN_NOT_PROVIDED) { @@ -1878,7 +1878,7 @@ yp_def_node_create( const yp_token_t *end_keyword ) { yp_def_node_t *node = YP_ALLOC_NODE(parser, yp_def_node_t); - const char *end; + const uint8_t *end; if (end_keyword->type == YP_TOKEN_NOT_PROVIDED) { end = body->location.end; @@ -1933,7 +1933,7 @@ yp_defined_node_create(yp_parser_t *parser, const yp_token_t *lparen, yp_node_t static yp_else_node_t * yp_else_node_create(yp_parser_t *parser, const yp_token_t *else_keyword, yp_statements_node_t *statements, const yp_token_t *end_keyword) { yp_else_node_t *node = YP_ALLOC_NODE(parser, yp_else_node_t); - const char *end = NULL; + const uint8_t *end = NULL; if ((end_keyword->type == YP_TOKEN_NOT_PROVIDED) && (statements != NULL)) { end = statements->base.location.end; } else { @@ -2413,7 +2413,7 @@ yp_if_node_create(yp_parser_t *parser, yp_flip_flop(predicate); yp_if_node_t *node = YP_ALLOC_NODE(parser, yp_if_node_t); - const char *end; + const uint8_t *end; if (end_keyword->type != YP_TOKEN_NOT_PROVIDED) { end = end_keyword->end; } else if (consequent != NULL) { @@ -2596,7 +2596,7 @@ static yp_in_node_t * yp_in_node_create(yp_parser_t *parser, yp_node_t *pattern, yp_statements_node_t *statements, const yp_token_t *in_keyword, const yp_token_t *then_keyword) { yp_in_node_t *node = YP_ALLOC_NODE(parser, yp_in_node_t); - const char *end; + const uint8_t *end; if (statements != NULL) { end = statements->base.location.end; } else if (then_keyword->type != YP_TOKEN_NOT_PROVIDED) { @@ -3891,7 +3891,7 @@ yp_statements_node_body_length(yp_statements_node_t *node) { // Set the location of the given StatementsNode. static void -yp_statements_node_location_set(yp_statements_node_t *node, const char *start, const char *end) { +yp_statements_node_location_set(yp_statements_node_t *node, const uint8_t *start, const uint8_t *end) { node->base.location = (yp_location_t) { .start = start, .end = end }; } @@ -3957,7 +3957,7 @@ yp_super_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_argument assert(keyword->type == YP_TOKEN_KEYWORD_SUPER); yp_super_node_t *node = YP_ALLOC_NODE(parser, yp_super_node_t); - const char *end; + const uint8_t *end; if (arguments->block != NULL) { end = arguments->block->base.location.end; } else if (arguments->closing_loc.start != NULL) { @@ -4048,7 +4048,7 @@ yp_symbol_node_label_create(yp_parser_t *parser, const yp_token_t *token) { // Check if the given node is a label in a hash. static bool yp_symbol_node_label_p(yp_node_t *node) { - const char *end = NULL; + const uint8_t *end = NULL; switch (YP_NODE_TYPE(node)) { case YP_NODE_SYMBOL_NODE: @@ -4156,7 +4156,7 @@ yp_unless_node_create(yp_parser_t *parser, const yp_token_t *keyword, yp_node_t yp_flip_flop(predicate); yp_unless_node_t *node = YP_ALLOC_NODE(parser, yp_unless_node_t); - const char *end; + const uint8_t *end; if (statements != NULL) { end = statements->base.location.end; } else { @@ -4373,7 +4373,7 @@ static yp_yield_node_t * yp_yield_node_create(yp_parser_t *parser, const yp_token_t *keyword, const yp_location_t *lparen_loc, yp_arguments_node_t *arguments, const yp_location_t *rparen_loc) { yp_yield_node_t *node = YP_ALLOC_NODE(parser, yp_yield_node_t); - const char *end; + const uint8_t *end; if (rparen_loc->start != NULL) { end = rparen_loc->end; } else if (arguments != NULL) { @@ -4447,7 +4447,7 @@ yp_parser_local_depth(yp_parser_t *parser, yp_token_t *token) { // Add a local variable from a location to the current scope. static yp_constant_id_t -yp_parser_local_add_location(yp_parser_t *parser, const char *start, const char *end) { +yp_parser_local_add_location(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { yp_constant_id_t constant_id = yp_parser_constant_id_location(parser, start, end); if (!yp_constant_id_list_includes(&parser->current_scope->locals, constant_id)) { @@ -4496,15 +4496,13 @@ yp_parser_scope_pop(yp_parser_t *parser) { // reason we have the encoding_changed boolean to check if we need to go through // the function pointer or can just directly use the UTF-8 functions. static inline size_t -char_is_identifier_start(yp_parser_t *parser, const char *c) { - const unsigned char uc = (unsigned char) *c; - +char_is_identifier_start(yp_parser_t *parser, const uint8_t *b) { if (parser->encoding_changed) { - return parser->encoding.alpha_char(c, parser->end - c) || (uc == '_') || (uc >= 0x80); - } else if (uc < 0x80) { - return (yp_encoding_unicode_table[uc] & YP_ENCODING_ALPHABETIC_BIT ? 1 : 0) || (uc == '_'); + return parser->encoding.alpha_char(b, parser->end - b) || (*b == '_') || (*b >= 0x80); + } else if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_ALPHABETIC_BIT ? 1 : 0) || (*b == '_'); } else { - return (size_t) (yp_encoding_utf_8_alpha_char(c, parser->end - c) || 1u); + return (size_t) (yp_encoding_utf_8_alpha_char(b, parser->end - b) || 1u); } } @@ -4512,15 +4510,13 @@ char_is_identifier_start(yp_parser_t *parser, const char *c) { // the identifiers in a source file once the first character has been found. So // it's important that it be as fast as possible. static inline size_t -char_is_identifier(yp_parser_t *parser, const char *c) { - const unsigned char uc = (unsigned char) *c; - +char_is_identifier(yp_parser_t *parser, const uint8_t *b) { if (parser->encoding_changed) { - return parser->encoding.alnum_char(c, parser->end - c) || (uc == '_') || (uc >= 0x80); - } else if (uc < 0x80) { - return (yp_encoding_unicode_table[uc] & YP_ENCODING_ALPHANUMERIC_BIT ? 1 : 0) || (uc == '_'); + return parser->encoding.alnum_char(b, parser->end - b) || (*b == '_') || (*b >= 0x80); + } else if (*b < 0x80) { + return (yp_encoding_unicode_table[*b] & YP_ENCODING_ALPHANUMERIC_BIT ? 1 : 0) || (*b == '_'); } else { - return (size_t) (yp_encoding_utf_8_alnum_char(c, parser->end - c) || 1u); + return (size_t) (yp_encoding_utf_8_alnum_char(b, parser->end - b) || 1u); } } @@ -4542,15 +4538,15 @@ const unsigned int yp_global_name_punctuation_hash[(0x7e - 0x20 + 31) / 32] = { #undef PUNCT static inline bool -char_is_global_name_punctuation(const char c) { - const unsigned int i = (const unsigned int) c; +char_is_global_name_punctuation(const uint8_t b) { + const unsigned int i = (const unsigned int) b; if (i <= 0x20 || 0x7e < i) return false; - return (yp_global_name_punctuation_hash[(i - 0x20) / 32] >> (c % 32)) & 1; + return (yp_global_name_punctuation_hash[(i - 0x20) / 32] >> (i % 32)) & 1; } static inline bool -token_is_numbered_parameter(const char *start, const char *end) { +token_is_numbered_parameter(const uint8_t *start, const uint8_t *end) { return (end - start == 2) && (start[0] == '_') && (start[1] != '0') && (yp_char_is_decimal_digit(start[1])); } @@ -4604,8 +4600,8 @@ yp_do_loop_stack_p(yp_parser_t *parser) { // Get the next character in the source starting from +cursor+. If that position // is beyond the end of the source then return '\0'. -static inline char -peek_at(yp_parser_t *parser, const char *cursor) { +static inline uint8_t +peek_at(yp_parser_t *parser, const uint8_t *cursor) { if (cursor < parser->end) { return *cursor; } else { @@ -4616,33 +4612,33 @@ peek_at(yp_parser_t *parser, const char *cursor) { // Get the next character in the source starting from parser->current.end and // adding the given offset. If that position is beyond the end of the source // then return '\0'. -static inline char +static inline uint8_t peek_offset(yp_parser_t *parser, ptrdiff_t offset) { return peek_at(parser, parser->current.end + offset); } // Get the next character in the source starting from parser->current.end. If // that position is beyond the end of the source then return '\0'. -static inline char +static inline uint8_t peek(yp_parser_t *parser) { return peek_at(parser, parser->current.end); } // Get the next string of length len in the source starting from parser->current.end. // If the string extends beyond the end of the source, return the empty string "" -static inline const char* +static inline const uint8_t * peek_string(yp_parser_t *parser, size_t len) { if (parser->current.end + len <= parser->end) { return parser->current.end; } else { - return ""; + return (const uint8_t *) ""; } } // If the character to be read matches the given value, then returns true and // advanced the current pointer. static inline bool -match(yp_parser_t *parser, char value) { +match(yp_parser_t *parser, uint8_t value) { if (peek(parser) == value) { parser->current.end++; return true; @@ -4653,7 +4649,7 @@ match(yp_parser_t *parser, char value) { // Return the length of the line ending string starting at +cursor+, or 0 if it // is not a line ending. This function is intended to be CRLF/LF agnostic. static inline size_t -match_eol_at(yp_parser_t *parser, const char *cursor) { +match_eol_at(yp_parser_t *parser, const uint8_t *cursor) { if (peek_at(parser, cursor) == '\n') { return 1; } @@ -4680,8 +4676,8 @@ match_eol(yp_parser_t *parser) { } // Skip to the next newline character or NUL byte. -static inline const char * -next_newline(const char *cursor, ptrdiff_t length) { +static inline const uint8_t * +next_newline(const uint8_t *cursor, ptrdiff_t length) { assert(length >= 0); // Note that it's okay for us to use memchr here to look for \n because none @@ -4692,15 +4688,15 @@ next_newline(const char *cursor, ptrdiff_t length) { // Find the start of the encoding comment. This is effectively an inlined // version of strnstr with some modifications. -static inline const char * -parser_lex_encoding_comment_start(yp_parser_t *parser, const char *cursor, ptrdiff_t remaining) { +static inline const uint8_t * +parser_lex_encoding_comment_start(yp_parser_t *parser, const uint8_t *cursor, ptrdiff_t remaining) { assert(remaining >= 0); size_t length = (size_t) remaining; size_t key_length = strlen("coding:"); if (key_length > length) return NULL; - const char *cursor_limit = cursor + length - key_length + 1; + const uint8_t *cursor_limit = cursor + length - key_length + 1; while ((cursor = yp_memchr(cursor, 'c', (size_t) (cursor_limit - cursor), parser->encoding_changed, &parser->encoding)) != NULL) { if (memcmp(cursor, "coding", key_length - 1) == 0) { size_t whitespace_after_coding = yp_strspn_inline_whitespace(cursor + key_length - 1, parser->end - (cursor + key_length - 1)); @@ -4721,13 +4717,13 @@ parser_lex_encoding_comment_start(yp_parser_t *parser, const char *cursor, ptrdi // actions are necessary for it here. static void parser_lex_encoding_comment(yp_parser_t *parser) { - const char *start = parser->current.start + 1; - const char *end = next_newline(start, parser->end - start); + const uint8_t *start = parser->current.start + 1; + const uint8_t *end = next_newline(start, parser->end - start); if (end == NULL) end = parser->end; // These are the patterns we're going to match to find the encoding comment. // This is definitely not complete or even really correct. - const char *encoding_start = parser_lex_encoding_comment_start(parser, start, end - start); + const uint8_t *encoding_start = parser_lex_encoding_comment_start(parser, start, end - start); // If we didn't find anything that matched our patterns, then return. Note // that this does a _very_ poor job of actually finding the encoding, and @@ -4740,7 +4736,7 @@ parser_lex_encoding_comment(yp_parser_t *parser) { // Now determine the end of the encoding string. This is either the end of // the line, the first whitespace character, or a punctuation mark. - const char *encoding_end = yp_strpbrk(parser, encoding_start, " \t\f\r\v\n;,", end - encoding_start); + const uint8_t *encoding_end = yp_strpbrk(parser, encoding_start, (const uint8_t *) " \t\f\r\v\n;,", end - encoding_start); encoding_end = encoding_end == NULL ? end : encoding_end; // Finally, we can determine the width of the encoding string. @@ -4762,7 +4758,7 @@ parser_lex_encoding_comment(yp_parser_t *parser) { // Extensions like utf-8 can contain extra encoding details like, // utf-8-dos, utf-8-linux, utf-8-mac. We treat these all as utf-8 should // treat any encoding starting utf-8 as utf-8. - if ((encoding_start + 5 <= parser->end) && (yp_strncasecmp(encoding_start, "utf-8", 5) == 0)) { + if ((encoding_start + 5 <= parser->end) && (yp_strncasecmp(encoding_start, (const uint8_t *) "utf-8", 5) == 0)) { // We don't need to do anything here because the default encoding is // already UTF-8. We'll just return. return; @@ -4771,7 +4767,7 @@ parser_lex_encoding_comment(yp_parser_t *parser) { // Next, we're going to loop through each of the encodings that we handle // explicitly. If we found one that we understand, we'll use that value. #define ENCODING(value, prebuilt) \ - if (width == sizeof(value) - 1 && encoding_start + width <= parser->end && yp_strncasecmp(encoding_start, value, width) == 0) { \ + if (width == sizeof(value) - 1 && encoding_start + width <= parser->end && yp_strncasecmp(encoding_start, (const uint8_t *) value, width) == 0) { \ parser->encoding = prebuilt; \ parser->encoding_changed |= true; \ if (parser->encoding_changed_callback != NULL) parser->encoding_changed_callback(parser); \ @@ -5093,7 +5089,7 @@ lex_numeric(yp_parser_t *parser) { if (parser->current.end < parser->end) { type = lex_numeric_prefix(parser); - const char *end = parser->current.end; + const uint8_t *end = parser->current.end; yp_token_type_t suffix_type = type; if (type == YP_TOKEN_INTEGER) { @@ -5118,8 +5114,8 @@ lex_numeric(yp_parser_t *parser) { } } - const unsigned char uc = (const unsigned char) peek(parser); - if (uc != '\0' && (uc >= 0x80 || ((uc >= 'a' && uc <= 'z') || (uc >= 'A' && uc <= 'Z')) || uc == '_')) { + const uint8_t b = peek(parser); + if (b != '\0' && (b >= 0x80 || ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z')) || b == '_')) { parser->current.end = end; } else { type = suffix_type; @@ -5390,7 +5386,7 @@ current_token_starts_line(yp_parser_t *parser) { // this token type. // static yp_token_type_t -lex_interpolation(yp_parser_t *parser, const char *pound) { +lex_interpolation(yp_parser_t *parser, const uint8_t *pound) { // If there is no content following this #, then we're at the end of // the string and we can safely return string content. if (pound + 1 >= parser->end) { @@ -5411,7 +5407,7 @@ lex_interpolation(yp_parser_t *parser, const char *pound) { // If we're looking at a @ and there's another @, then we'll skip past the // second @. - const char *variable = pound + 2; + const uint8_t *variable = pound + 2; if (*variable == '@' && pound + 3 < parser->end) variable++; if (char_is_identifier_start(parser, variable)) { @@ -5447,7 +5443,7 @@ lex_interpolation(yp_parser_t *parser, const char *pound) { // This is the character that we're going to check to see if it is the // start of an identifier that would indicate that this is a global // variable. - const char *check = pound + 2; + const uint8_t *check = pound + 2; if (pound[2] == '-') { if (pound + 3 >= parser->end) { @@ -5638,7 +5634,7 @@ parser_comment(yp_parser_t *parser, yp_comment_type_t type) { static yp_token_type_t lex_embdoc(yp_parser_t *parser) { // First, lex out the EMBDOC_BEGIN token. - const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end); if (newline == NULL) { parser->current.end = parser->end; @@ -5663,7 +5659,7 @@ lex_embdoc(yp_parser_t *parser) { // token here. if (memcmp(parser->current.end, "=end", 4) == 0 && (parser->current.end + 4 == parser->end || yp_char_is_whitespace(parser->current.end[4]))) { - const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end); if (newline == NULL) { parser->current.end = parser->end; @@ -5683,7 +5679,7 @@ lex_embdoc(yp_parser_t *parser) { // Otherwise, we'll parse until the end of the line and return a line of // embedded documentation. - const char *newline = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *newline = next_newline(parser->current.end, parser->end - parser->current.end); if (newline == NULL) { parser->current.end = parser->end; @@ -5833,7 +5829,7 @@ parser_lex(yp_parser_t *parser) { LEX(YP_TOKEN_EOF); case '#': { // comments - const char *ending = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *ending = next_newline(parser->current.end, parser->end - parser->current.end); parser->current.end = ending == NULL ? parser->end : ending + 1; parser->current.type = YP_TOKEN_COMMENT; @@ -5902,7 +5898,7 @@ parser_lex(yp_parser_t *parser) { // (either . or &.) that starts the next line. If there is, then this // is going to become an ignored newline and we're going to instead // return the call operator. - const char *next_content = parser->next_start == NULL ? parser->current.end : parser->next_start; + const uint8_t *next_content = parser->next_start == NULL ? parser->current.end : parser->next_start; next_content += yp_strspn_inline_whitespace(next_content, parser->end - next_content); if (next_content < parser->end) { @@ -5913,7 +5909,7 @@ parser_lex(yp_parser_t *parser) { // Otherwise we'll return a regular newline. if (next_content[0] == '#') { // Here we look for a "." or "&." following a "\n". - const char *following = next_newline(next_content, parser->end - next_content); + const uint8_t *following = next_newline(next_content, parser->end - next_content); while (following && (following + 1 < parser->end)) { following++; @@ -6202,7 +6198,7 @@ parser_lex(yp_parser_t *parser) { !lex_state_end_p(parser) && (!lex_state_p(parser, YP_LEX_STATE_ARG_ANY) || lex_state_p(parser, YP_LEX_STATE_LABELED) || space_seen) ) { - const char *end = parser->current.end; + const uint8_t *end = parser->current.end; yp_heredoc_quote_t quote = YP_HEREDOC_QUOTE_NONE; yp_heredoc_indent_t indent = YP_HEREDOC_INDENT_NONE; @@ -6224,7 +6220,7 @@ parser_lex(yp_parser_t *parser) { quote = YP_HEREDOC_QUOTE_SINGLE; } - const char *ident_start = parser->current.end; + const uint8_t *ident_start = parser->current.end; size_t width = 0; if (parser->current.end >= parser->end) { @@ -6247,7 +6243,7 @@ parser_lex(yp_parser_t *parser) { } size_t ident_length = (size_t) (parser->current.end - ident_start); - if (quote != YP_HEREDOC_QUOTE_NONE && !match(parser, (char) quote)) { + if (quote != YP_HEREDOC_QUOTE_NONE && !match(parser, (uint8_t) quote)) { // TODO: handle unterminated heredoc } @@ -6263,7 +6259,7 @@ parser_lex(yp_parser_t *parser) { }); if (parser->heredoc_end == NULL) { - const char *body_start = next_newline(parser->current.end, parser->end - parser->current.end); + const uint8_t *body_start = next_newline(parser->current.end, parser->end - parser->current.end); if (body_start == NULL) { // If there is no newline after the heredoc identifier, then @@ -6905,8 +6901,8 @@ parser_lex(yp_parser_t *parser) { // Here we'll get a list of the places where strpbrk should break, // and then find the first one. yp_lex_mode_t *lex_mode = parser->lex_modes.current; - const char *breakpoints = lex_mode->as.list.breakpoints; - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoints = lex_mode->as.list.breakpoints; + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { // If we hit a null byte, skip directly past it. @@ -7028,8 +7024,8 @@ parser_lex(yp_parser_t *parser) { // These are the places where we need to split up the content of the // regular expression. We'll use strpbrk to find the first of these // characters. - const char *breakpoints = lex_mode->as.regexp.breakpoints; - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoints = lex_mode->as.regexp.breakpoints; + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { // If we hit a null byte, skip directly past it. @@ -7162,8 +7158,8 @@ parser_lex(yp_parser_t *parser) { // These are the places where we need to split up the content of the // string. We'll use strpbrk to find the first of these characters. - const char *breakpoints = parser->lex_modes.current->as.string.breakpoints; - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoints = parser->lex_modes.current->as.string.breakpoints; + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { // If we hit the incrementor, then we'll increment then nesting and @@ -7314,13 +7310,13 @@ parser_lex(yp_parser_t *parser) { // Now let's grab the information about the identifier off of the current // lex mode. - const char *ident_start = parser->lex_modes.current->as.heredoc.ident_start; + const uint8_t *ident_start = parser->lex_modes.current->as.heredoc.ident_start; size_t ident_length = parser->lex_modes.current->as.heredoc.ident_length; // If we are immediately following a newline and we have hit the // terminator, then we need to return the ending of the heredoc. if (current_token_starts_line(parser)) { - const char *start = parser->current.start; + const uint8_t *start = parser->current.start; if (parser->lex_modes.current->as.heredoc.indent != YP_HEREDOC_INDENT_NONE) { start += yp_strspn_inline_whitespace(start, parser->end - start); } @@ -7360,14 +7356,14 @@ parser_lex(yp_parser_t *parser) { // Otherwise we'll be parsing string content. These are the places where // we need to split up the content of the heredoc. We'll use strpbrk to // find the first of these characters. - char breakpoints[] = "\n\\#"; + uint8_t breakpoints[] = "\n\\#"; yp_heredoc_quote_t quote = parser->lex_modes.current->as.heredoc.quote; if (quote == YP_HEREDOC_QUOTE_SINGLE) { breakpoints[2] = '\0'; } - const char *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); + const uint8_t *breakpoint = yp_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end); while (breakpoint != NULL) { switch (*breakpoint) { @@ -7384,7 +7380,7 @@ parser_lex(yp_parser_t *parser) { yp_newline_list_append(&parser->newline_list, breakpoint); - const char *start = breakpoint + 1; + const uint8_t *start = breakpoint + 1; if (parser->lex_modes.current->as.heredoc.indent != YP_HEREDOC_INDENT_NONE) { start += yp_strspn_inline_whitespace(start, parser->end - start); } @@ -7966,10 +7962,11 @@ parse_target(yp_parser_t *parser, yp_node_t *target) { // the previous method name in, and append an =. size_t length = yp_string_length(&call->name); - char *name = calloc(length + 2, sizeof(char)); + uint8_t *name = calloc(length + 1, sizeof(uint8_t)); if (name == NULL) return NULL; - snprintf(name, length + 2, "%.*s=", (int) length, yp_string_source(&call->name)); + memcpy(name, yp_string_source(&call->name), length); + name[length] = '='; // Now switch the name to the new string. yp_string_free(&call->name); @@ -8123,10 +8120,11 @@ parse_write(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_nod // the previous method name in, and append an =. size_t length = yp_string_length(&call->name); - char *name = calloc(length + 2, sizeof(char)); + uint8_t *name = calloc(length + 1, sizeof(uint8_t)); if (name == NULL) return NULL; - snprintf(name, length + 2, "%.*s=", (int) length, yp_string_source(&call->name)); + memcpy(name, yp_string_source(&call->name), length); + name[length] = '='; // Now switch the name to the new string. yp_string_free(&call->name); @@ -9113,7 +9111,7 @@ parse_rescues(yp_parser_t *parser, yp_begin_node_t *parent_node) { // since we won't know the end until we've found all consequent // clauses. This sets the end location on all rescues once we know it if (current) { - const char *end_to_set = current->base.location.end; + const uint8_t *end_to_set = current->base.location.end; current = parent_node->rescue_clause; while (current) { current->base.location.end = end_to_set; @@ -9170,7 +9168,7 @@ parse_rescues_as_begin(yp_parser_t *parser, yp_statements_node_t *statements) { // All nodes within a begin node are optional, so we look // for the earliest possible node that we can use to set // the BeginNode's start location - const char * start = begin_node->base.location.start; + const uint8_t *start = begin_node->base.location.start; if (begin_node->statements) { start = begin_node->statements->base.location.start; } else if (begin_node->rescue_clause) { @@ -9845,7 +9843,7 @@ parse_heredoc_common_whitespace(yp_parser_t *parser, yp_node_list_t *nodes) { // variable. if (index == 0 || YP_NODE_TYPE_P(nodes->nodes[index - 1], YP_NODE_STRING_NODE)) { int cur_whitespace; - const char *cur_char = content_loc->start; + const uint8_t *cur_char = content_loc->start; while (cur_char && cur_char < content_loc->end) { // Any empty newlines aren't included in the minimum whitespace @@ -9936,15 +9934,15 @@ parse_heredoc_dedent(yp_parser_t *parser, yp_node_t *node, yp_heredoc_quote_t qu // destination to move bytes into. We'll also use it for bounds checking // since we don't require that these strings be null terminated. size_t dest_length = yp_string_length(string); - char *source_start = string->source; + uint8_t *source_start = (uint8_t *) string->source; - const char *source_cursor = source_start; - const char *source_end = source_cursor + dest_length; + const uint8_t *source_cursor = source_start; + const uint8_t *source_end = source_cursor + dest_length; // We're going to move bytes backward in the string when we get leading // whitespace, so we'll maintain a pointer to the current position in the // string that we're writing to. - char *dest_cursor = source_start; + uint8_t *dest_cursor = source_start; while (source_cursor < source_end) { // If we need to dedent the next element within the heredoc or the next @@ -9971,7 +9969,7 @@ parse_heredoc_dedent(yp_parser_t *parser, yp_node_t *node, yp_heredoc_quote_t qu // At this point we have dedented all that we need to, so we need to find // the next newline. - const char *breakpoint = next_newline(source_cursor, source_end - source_cursor); + const uint8_t *breakpoint = next_newline(source_cursor, source_end - source_cursor); if (breakpoint == NULL) { // If there isn't another newline, then we can just move the rest of the @@ -13587,7 +13585,7 @@ yp_parser_metadata(yp_parser_t *parser, const char *metadata) { uint32_t local_size = yp_metadata_read_u32(metadata); metadata += 4; - yp_parser_local_add_location(parser, metadata, metadata + local_size); + yp_parser_local_add_location(parser, (const uint8_t *) metadata, (const uint8_t *) (metadata + local_size)); metadata += local_size; } } @@ -13599,7 +13597,7 @@ yp_parser_metadata(yp_parser_t *parser, const char *metadata) { // Initialize a parser with the given start and end pointers. YP_EXPORTED_FUNCTION void -yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath) { +yp_parser_init(yp_parser_t *parser, const uint8_t *source, size_t size, const char *filepath) { assert(source != NULL); // Set filepath to the file that was passed @@ -13671,7 +13669,7 @@ yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char yp_newline_list_init(&parser->newline_list, source, newline_size < 4 ? 4 : newline_size); // Skip past the UTF-8 BOM if it exists. - if (size >= 3 && (unsigned char) source[0] == 0xef && (unsigned char) source[1] == 0xbb && (unsigned char) source[2] == 0xbf) { + if (size >= 3 && source[0] == 0xef && source[1] == 0xbb && source[2] == 0xbf) { parser->current.end += 3; parser->encoding_comment_start += 3; } @@ -13679,7 +13677,7 @@ yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char // If the first two bytes of the source are a shebang, then we'll indicate // that the encoding comment is at the end of the shebang. if (peek(parser) == '#' && peek_offset(parser, 1) == '!') { - const char *encoding_comment_start = next_newline(source, (ptrdiff_t) size); + const uint8_t *encoding_comment_start = next_newline(source, (ptrdiff_t) size); if (encoding_comment_start) { parser->encoding_comment_start = encoding_comment_start + 1; } @@ -13751,7 +13749,7 @@ yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { // Parse and serialize the AST represented by the given source to the given // buffer. YP_EXPORTED_FUNCTION void -yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata) { +yp_parse_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata) { yp_parser_t parser; yp_parser_init(&parser, source, size, NULL); if (metadata) yp_parser_metadata(&parser, metadata); diff --git a/yarp/yarp.h b/yarp/yarp.h index 1a8a47725a9adf..6488d01adcbfab 100644 --- a/yarp/yarp.h +++ b/yarp/yarp.h @@ -40,7 +40,7 @@ void yp_scope_node_init(yp_node_t *node, yp_scope_node_t *dest); YP_EXPORTED_FUNCTION const char * yp_version(void); // Initialize a parser with the given start and end pointers. -YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const char *source, size_t size, const char *filepath); +YP_EXPORTED_FUNCTION void yp_parser_init(yp_parser_t *parser, const uint8_t *source, size_t size, const char *filepath); // Register a callback that will be called whenever YARP changes the encoding it // is using to parse based on the magic comment. @@ -66,14 +66,14 @@ YP_EXPORTED_FUNCTION void yp_prettyprint(yp_parser_t *parser, yp_node_t *node, y YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer); // Parse the given source to the AST and serialize the AST to the given buffer. -YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata); +YP_EXPORTED_FUNCTION void yp_parse_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata); // Lex the given source and serialize to the given buffer. -YP_EXPORTED_FUNCTION void yp_lex_serialize(const char *source, size_t size, const char *filepath, yp_buffer_t *buffer); +YP_EXPORTED_FUNCTION void yp_lex_serialize(const uint8_t *source, size_t size, const char *filepath, yp_buffer_t *buffer); // Parse and serialize both the AST and the tokens represented by the given // source to the given buffer. -YP_EXPORTED_FUNCTION void yp_parse_lex_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata); +YP_EXPORTED_FUNCTION void yp_parse_lex_serialize(const uint8_t *source, size_t size, yp_buffer_t *buffer, const char *metadata); // Returns a string representation of the given token type. YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type); From 49dff732e8734dccc7e30102cadc13290618d54a Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 14:32:52 -0400 Subject: [PATCH 147/208] Update YARP APIs to handle uint8_t --- compile.c | 2 +- iseq.c | 2 +- yarp/yarp_compiler.c | 26 +++++++++++++------------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/compile.c b/compile.c index 2673df5dbb5fbb..4d344fff9c6aca 100644 --- a/compile.c +++ b/compile.c @@ -878,7 +878,7 @@ rb_iseq_compile_yarp_node(rb_iseq_t * iseq, const yp_node_t *yarp_pointer, yp_pa yp_constant_t constant = parser->constant_pool.constants[index]; if (constant.id != 0) { - constants[constant.id - 1] = rb_intern3(constant.start, constant.length, encoding); + constants[constant.id - 1] = rb_intern3((const char *) constant.start, constant.length, encoding); } } diff --git a/iseq.c b/iseq.c index 9082cd4861783c..4466de8bd105c9 100644 --- a/iseq.c +++ b/iseq.c @@ -1425,7 +1425,7 @@ iseqw_s_compile_yarp(int argc, VALUE *argv, VALUE self) size_t len = RSTRING_LEN(src); VALUE name = rb_fstring_lit(""); - yp_parser_init(&parser, RSTRING_PTR(src), len, ""); + yp_parser_init(&parser, (const uint8_t *) RSTRING_PTR(src), len, ""); yp_node_t *node = yp_parse(&parser); diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index fa71c12226decb..0c31cd83a0fd3e 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -19,8 +19,8 @@ yp_iseq_new_with_opt(yp_scope_node_t *node, yp_parser_t *parser, VALUE name, VAL static VALUE parse_number(const yp_node_t *node) { - const char *start = node->location.start; - const char *end = node->location.end; + const uint8_t *start = node->location.start; + const uint8_t *end = node->location.end; size_t length = end - start; char *buffer = malloc(length + 1); @@ -35,12 +35,12 @@ parse_number(const yp_node_t *node) { static inline VALUE parse_string(yp_string_t *string) { - return rb_str_new(yp_string_source(string), yp_string_length(string)); + return rb_str_new((const char *) yp_string_source(string), yp_string_length(string)); } static inline ID -parse_symbol(const char *start, const char *end) { - return rb_intern2(start, end - start); +parse_symbol(const uint8_t *start, const uint8_t *end) { + return rb_intern2((const char *) start, end - start); } static inline ID @@ -50,7 +50,7 @@ parse_node_symbol(yp_node_t *node) { static inline ID parse_string_symbol(yp_string_t *string) { - const char *start = yp_string_source(string); + const uint8_t *start = yp_string_source(string); return parse_symbol(start, start + yp_string_length(string)); } @@ -106,11 +106,11 @@ yp_static_literal_value(yp_node_t *node) static void yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_node_t *cond, - LABEL *then_label, LABEL *else_label, const char *src, bool popped, yp_compile_context_t *compile_context); + LABEL *then_label, LABEL *else_label, const uint8_t *src, bool popped, yp_compile_context_t *compile_context); static void yp_compile_logical(rb_iseq_t *iseq, LINK_ANCHOR *const ret, yp_node_t *cond, - LABEL *then_label, LABEL *else_label, const char *src, bool popped, yp_compile_context_t *compile_context) + LABEL *then_label, LABEL *else_label, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) { yp_parser_t *parser = compile_context->parser; yp_newline_list_t newline_list = parser->newline_list; @@ -140,11 +140,11 @@ yp_compile_logical(rb_iseq_t *iseq, LINK_ANCHOR *const ret, yp_node_t *cond, return; } -static void yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *context); +static void yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *context); static void yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_node_t *cond, - LABEL *then_label, LABEL *else_label, const char *src, bool popped, yp_compile_context_t *compile_context) + LABEL *then_label, LABEL *else_label, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) { yp_parser_t *parser = compile_context->parser; yp_newline_list_t newline_list = parser->newline_list; @@ -197,7 +197,7 @@ yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_no } static void -yp_compile_if(rb_iseq_t *iseq, const int line, yp_statements_node_t *node_body, yp_node_t *node_else, yp_node_t *predicate, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) { +yp_compile_if(rb_iseq_t *iseq, const int line, yp_statements_node_t *node_body, yp_node_t *node_else, yp_node_t *predicate, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) { NODE line_node = generate_dummy_line_node(line, line); DECL_ANCHOR(cond_seq); @@ -261,7 +261,7 @@ yp_compile_if(rb_iseq_t *iseq, const int line, yp_statements_node_t *node_body, } static void -yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_node_type type, yp_statements_node_t *statements, yp_node_t *predicate, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) +yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_node_type type, yp_statements_node_t *statements, yp_node_t *predicate, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) { NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); @@ -388,7 +388,7 @@ yp_compile_class_path(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const yp_node_t * * compile_context - Stores parser and local information */ static void -yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const char * src, bool popped, yp_compile_context_t *compile_context) +yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, yp_compile_context_t *compile_context) { yp_parser_t *parser = compile_context->parser; yp_newline_list_t newline_list = parser->newline_list; From 1ed70eb99f54a4221bf2e4644596922f676340cc Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 15:20:07 -0400 Subject: [PATCH 148/208] [ruby/yarp] Fix unused variable in YARPRubyAPITest https://github.com/ruby/yarp/commit/0556f971ce --- test/yarp/ruby_api_test.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 236407ccac413a..31b3f0fdec3b91 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -43,8 +43,7 @@ def test_location_join arg.location.join(recv.location) end - other_recv, other_args_node, _ = parse_expression("1234 + 567").child_nodes - other_arg = other_args_node.arguments[0] + other_arg = parse_expression("1234 + 567").arguments.arguments[0] assert_raise RuntimeError, "Incompatible sources" do other_arg.location.join(recv.location) From 082962e857833dfc16881cb4dfb44caa175590ba Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Fri, 26 May 2023 13:12:20 +1000 Subject: [PATCH 149/208] Work around a hang in fork(2) on FreeBSD See bug https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 On FreeBSDk, it's possible for fork(2) in a multithreaded process to hang because of a bug in the lock handling of the dynamic linker. This is now fixed on FreeBSD master, but it would be good if we could work around it for Ruby CI which is running 13.1. Setting LD_BIND_NOW seems to work around the problem (probably because the dynamic linker doesn't then need to resolve anything through the PLT when it's first called). --- tool/runruby.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tool/runruby.rb b/tool/runruby.rb index 1efe38fd1305e7..c6724f53d3f7b3 100755 --- a/tool/runruby.rb +++ b/tool/runruby.rb @@ -134,6 +134,9 @@ def File.realpath(*args) env[e] = [abs_archdir, ENV[e]].compact.join(File::PATH_SEPARATOR) end end +# Work around a bug in FreeBSD 13.2 which can cause fork(2) to hang +# See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=271490 +env['LD_BIND_NOW'] = 'yes' if /freebsd/ =~ RUBY_PLATFORM ENV.update env From 151e94fee5a54f07fa031942de72009c7f7b1d1d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 15:20:56 -0400 Subject: [PATCH 150/208] [ruby/yarp] Fix instance variable constant names https://github.com/ruby/yarp/commit/1f94f55fcb --- test/yarp/snapshots/methods.txt | 2 +- test/yarp/snapshots/patterns.txt | 2 +- test/yarp/snapshots/seattlerb/case_in.txt | 2 +- ...interpolation_and_carriage_return_escapes.txt | 2 +- ...ation_and_carriage_return_escapes_windows.txt | 2 +- test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt | 2 +- ...e_line_call_ivar_arg_no_parens_line_break.txt | 2 +- .../parse_line_call_ivar_line_break_paren.txt | 2 +- test/yarp/snapshots/strings.txt | 2 +- .../unparser/corpus/literal/assignment.txt | 10 +++++----- .../unparser/corpus/literal/defined.txt | 2 +- .../snapshots/unparser/corpus/literal/dstr.txt | 2 +- .../unparser/corpus/literal/literal.txt | 16 ++++++++-------- .../unparser/corpus/literal/variables.txt | 2 +- .../snapshots/unparser/corpus/semantic/dstr.txt | 4 ++-- .../unparser/corpus/semantic/literal.txt | 2 +- test/yarp/snapshots/variables.txt | 10 +++++----- .../snapshots/whitequark/array_words_interp.txt | 2 +- test/yarp/snapshots/whitequark/defined.txt | 2 +- test/yarp/snapshots/whitequark/ivar.txt | 2 +- test/yarp/snapshots/whitequark/ivasgn.txt | 2 +- test/yarp/snapshots/whitequark/masgn_splat.txt | 2 +- .../yarp/snapshots/whitequark/parser_bug_272.txt | 2 +- test/yarp/snapshots/whitequark/resbody_var.txt | 2 +- test/yarp/snapshots/whitequark/string_concat.txt | 2 +- test/yarp/snapshots/whitequark/string_dvar.txt | 2 +- test/yarp/snapshots/whitequark/var_op_asgn.txt | 2 +- yarp/yarp.c | 2 +- 28 files changed, 44 insertions(+), 44 deletions(-) diff --git a/test/yarp/snapshots/methods.txt b/test/yarp/snapshots/methods.txt index 32a7a4e7cd903b..70e030811c4e1c 100644 --- a/test/yarp/snapshots/methods.txt +++ b/test/yarp/snapshots/methods.txt @@ -199,7 +199,7 @@ ProgramNode(0...1194)( ), DefNode(190...204)( (199...200), - InstanceVariableReadNode(194...198)(:var), + InstanceVariableReadNode(194...198)(:@var), nil, nil, [], diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index f888d1498fcd09..63764bca0beff5 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -923,7 +923,7 @@ ProgramNode(0...3743)( "foo" ), PinnedVariableNode(961...966)( - InstanceVariableReadNode(962...966)(:bar), + InstanceVariableReadNode(962...966)(:@bar), (961...962) ), (958...960) diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index 5ea3d11b78e10f..2e508d8c9ae135 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -494,7 +494,7 @@ ProgramNode(0...747)( ArrayPatternNode(627...643)( nil, [PinnedVariableNode(628...631)( - InstanceVariableReadNode(629...631)(:a), + InstanceVariableReadNode(629...631)(:@a), (628...629) ), PinnedVariableNode(633...636)( diff --git a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt index 5aab2888ef81ad..452e5b605b8ef5 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes.txt @@ -6,7 +6,7 @@ ProgramNode(0...5)( [StringNode(6...11)(nil, (6...11), nil, "foo\r"), EmbeddedVariableNode(11...16)( (11...12), - InstanceVariableReadNode(12...16)(:bar) + InstanceVariableReadNode(12...16)(:@bar) ), StringNode(16...17)(nil, (16...17), nil, "\n")], (17...21) diff --git a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt index 04727011a2a369..99b42f6122a97e 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt @@ -6,7 +6,7 @@ ProgramNode(0...5)( [StringNode(7...12)(nil, (7...12), nil, "foo\r"), EmbeddedVariableNode(12...17)( (12...13), - InstanceVariableReadNode(13...17)(:bar) + InstanceVariableReadNode(13...17)(:@bar) ), StringNode(17...19)(nil, (17...19), nil, "\r\n")], (19...24) diff --git a/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt b/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt index f065032f3a2b2c..cf47f6dc7e5ded 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_ivar_env.txt @@ -2,7 +2,7 @@ ProgramNode(0...7)( [], StatementsNode(0...7)( [InstanceVariableWriteNode(0...7)( - :a, + :@a, (0...2), IntegerNode(5...7)(), (3...4) diff --git a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt index efbc413fe1485c..84ee2d450bd7aa 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_arg_no_parens_line_break.txt @@ -6,7 +6,7 @@ ProgramNode(0...4)( nil, (0...1), nil, - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:b)]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:@b)]), nil, nil, 0, diff --git a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt index 6466d9dd45b8d4..1f9d067c76bf7d 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_call_ivar_line_break_paren.txt @@ -6,7 +6,7 @@ ProgramNode(0...6)( nil, (0...1), (1...2), - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:b)]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:@b)]), (5...6), nil, 0, diff --git a/test/yarp/snapshots/strings.txt b/test/yarp/snapshots/strings.txt index d6f624624c0d40..ee8bc79f17eeaf 100644 --- a/test/yarp/snapshots/strings.txt +++ b/test/yarp/snapshots/strings.txt @@ -185,7 +185,7 @@ ProgramNode(0...498)( (414...415), [EmbeddedVariableNode(415...420)( (415...416), - InstanceVariableReadNode(416...420)(:foo) + InstanceVariableReadNode(416...420)(:@foo) )], (420...421) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 6166e0cd26814e..7e193e0e87dd25 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -66,8 +66,8 @@ ProgramNode(0...704)( (74...75) ), MultiWriteNode(85...102)( - [InstanceVariableTargetNode(86...88)(:a), - InstanceVariableTargetNode(90...92)(:b)], + [InstanceVariableTargetNode(86...88)(:@a), + InstanceVariableTargetNode(90...92)(:@b)], (94...95), ArrayNode(96...102)( [IntegerNode(97...98)(), IntegerNode(100...101)()], @@ -296,7 +296,7 @@ ProgramNode(0...704)( (306...307) ), InstanceVariableWriteNode(310...316)( - :a, + :@a, (310...312), IntegerNode(315...316)(), (313...314) @@ -606,7 +606,7 @@ ProgramNode(0...704)( (543...546) ), InstanceVariableOrWriteNode(551...561)( - :a, + :@a, (551...553), (554...557), StringNode(558...561)((558...560), (560...560), (560...561), "") @@ -700,7 +700,7 @@ ProgramNode(0...704)( (665...668) ), InstanceVariableOrWriteNode(687...704)( - :a, + :@a, (687...689), (690...693), InterpolatedStringNode(694...704)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/defined.txt b/test/yarp/snapshots/unparser/corpus/literal/defined.txt index 1d39c45c836728..9229badf44efbe 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/defined.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/defined.txt @@ -3,7 +3,7 @@ ProgramNode(0...56)( StatementsNode(0...56)( [DefinedNode(0...14)( (8...9), - InstanceVariableReadNode(9...13)(:foo), + InstanceVariableReadNode(9...13)(:@foo), (13...14), (0...8) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt index 49d08824bfce0f..1abbed681f80bd 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt @@ -75,7 +75,7 @@ ProgramNode(0...299)( [StringNode(160...161)(nil, (160...161), nil, "a"), EmbeddedVariableNode(161...164)( (161...162), - InstanceVariableReadNode(162...164)(:a) + InstanceVariableReadNode(162...164)(:@a) )], (164...165) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/literal.txt b/test/yarp/snapshots/unparser/corpus/literal/literal.txt index a38a7f4f888cd1..03f9aecbd56ec0 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/literal.txt @@ -153,7 +153,7 @@ ProgramNode(0...916)( (206...207), [EmbeddedVariableNode(207...210)( (207...208), - InstanceVariableReadNode(208...210)(:a) + InstanceVariableReadNode(208...210)(:@a) ), StringNode(210...211)(nil, (210...211), nil, " "), EmbeddedVariableNode(211...215)( @@ -268,7 +268,7 @@ ProgramNode(0...916)( EmbeddedStatementsNode(422...429)( (422...424), StatementsNode(424...428)( - [InstanceVariableReadNode(424...428)(:bar)] + [InstanceVariableReadNode(424...428)(:@bar)] ), (428...429) )], @@ -303,7 +303,7 @@ ProgramNode(0...916)( EmbeddedStatementsNode(519...526)( (519...521), StatementsNode(521...525)( - [InstanceVariableReadNode(521...525)(:bar)] + [InstanceVariableReadNode(521...525)(:@bar)] ), (525...526) )], @@ -316,7 +316,7 @@ ProgramNode(0...916)( EmbeddedStatementsNode(532...539)( (532...534), StatementsNode(534...538)( - [InstanceVariableReadNode(534...538)(:bar)] + [InstanceVariableReadNode(534...538)(:@bar)] ), (538...539) )], @@ -507,7 +507,7 @@ ProgramNode(0...916)( [IntegerNode(693...694)(), SplatNode(696...701)( (696...697), - InstanceVariableReadNode(697...701)(:foo) + InstanceVariableReadNode(697...701)(:@foo) )], (692...693), (701...702) @@ -515,7 +515,7 @@ ProgramNode(0...916)( ArrayNode(703...713)( [SplatNode(704...709)( (704...705), - InstanceVariableReadNode(705...709)(:foo) + InstanceVariableReadNode(705...709)(:@foo) ), IntegerNode(711...712)()], (703...704), @@ -524,11 +524,11 @@ ProgramNode(0...916)( ArrayNode(714...728)( [SplatNode(715...720)( (715...716), - InstanceVariableReadNode(716...720)(:foo) + InstanceVariableReadNode(716...720)(:@foo) ), SplatNode(722...727)( (722...723), - InstanceVariableReadNode(723...727)(:baz) + InstanceVariableReadNode(723...727)(:@baz) )], (714...715), (727...728) diff --git a/test/yarp/snapshots/unparser/corpus/literal/variables.txt b/test/yarp/snapshots/unparser/corpus/literal/variables.txt index 29857ec53c5591..051fb2f498941a 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/variables.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/variables.txt @@ -2,7 +2,7 @@ ProgramNode(0...66)( [], StatementsNode(0...66)( [CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), - InstanceVariableReadNode(2...4)(:a), + InstanceVariableReadNode(2...4)(:@a), ClassVariableReadNode(5...8)(), GlobalVariableReadNode(9...11)(), NumberedReferenceReadNode(12...14)(), diff --git a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt index 490b13e0bc874a..47d37b1c38c43e 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt @@ -208,7 +208,7 @@ ProgramNode(0...608)( EmbeddedStatementsNode(564...569)( (564...566), StatementsNode(566...568)( - [InstanceVariableReadNode(566...568)(:a)] + [InstanceVariableReadNode(566...568)(:@a)] ), (568...569) )], @@ -222,7 +222,7 @@ ProgramNode(0...608)( [StringNode(576...577)(nil, (576...577), nil, "a"), EmbeddedVariableNode(577...580)( (577...578), - InstanceVariableReadNode(578...580)(:a) + InstanceVariableReadNode(578...580)(:@a) )], (580...581) ), diff --git a/test/yarp/snapshots/unparser/corpus/semantic/literal.txt b/test/yarp/snapshots/unparser/corpus/semantic/literal.txt index b9df7fcbd9a182..5e18c54da412a6 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/literal.txt @@ -15,7 +15,7 @@ ProgramNode(0...131)( (71...74), [EmbeddedStatementsNode(74...81)( (74...76), - StatementsNode(76...80)([InstanceVariableReadNode(76...80)(:bar)]), + StatementsNode(76...80)([InstanceVariableReadNode(76...80)(:@bar)]), (80...81) ), StringNode(81...84)(nil, (81...84), nil, "baz")], diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index a6b6a444f9ecd3..2dd87366c5674e 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -30,9 +30,9 @@ ProgramNode(0...293)( IntegerNode(57...58)() ), GlobalVariableReadNode(60...64)(), - InstanceVariableReadNode(66...70)(:abc), + InstanceVariableReadNode(66...70)(:@abc), InstanceVariableWriteNode(72...80)( - :abc, + :@abc, (72...76), IntegerNode(79...80)(), (77...78) @@ -63,15 +63,15 @@ ProgramNode(0...293)( ) ), MultiWriteNode(123...137)( - [InstanceVariableTargetNode(123...127)(:foo), - InstanceVariableTargetNode(129...133)(:bar)], + [InstanceVariableTargetNode(123...127)(:@foo), + InstanceVariableTargetNode(129...133)(:@bar)], (134...135), IntegerNode(136...137)(), nil, nil ), InstanceVariableWriteNode(139...150)( - :foo, + :@foo, (139...143), ArrayNode(146...150)( [IntegerNode(146...147)(), IntegerNode(149...150)()], diff --git a/test/yarp/snapshots/whitequark/array_words_interp.txt b/test/yarp/snapshots/whitequark/array_words_interp.txt index 963a83dd480833..d4bc9292e6ea64 100644 --- a/test/yarp/snapshots/whitequark/array_words_interp.txt +++ b/test/yarp/snapshots/whitequark/array_words_interp.txt @@ -51,7 +51,7 @@ ProgramNode(0...38)( StringNode(29...32)(nil, (29...32), nil, "foo"), EmbeddedVariableNode(32...37)( (32...33), - InstanceVariableReadNode(33...37)(:baz) + InstanceVariableReadNode(33...37)(:@baz) )], nil )], diff --git a/test/yarp/snapshots/whitequark/defined.txt b/test/yarp/snapshots/whitequark/defined.txt index 9d8d997e2bab16..3ddfdd84bbf5ea 100644 --- a/test/yarp/snapshots/whitequark/defined.txt +++ b/test/yarp/snapshots/whitequark/defined.txt @@ -3,7 +3,7 @@ ProgramNode(0...42)( StatementsNode(0...42)( [DefinedNode(0...13)( nil, - InstanceVariableReadNode(9...13)(:foo), + InstanceVariableReadNode(9...13)(:@foo), nil, (0...8) ), diff --git a/test/yarp/snapshots/whitequark/ivar.txt b/test/yarp/snapshots/whitequark/ivar.txt index cf233251f2835d..ae2c8c34b48306 100644 --- a/test/yarp/snapshots/whitequark/ivar.txt +++ b/test/yarp/snapshots/whitequark/ivar.txt @@ -1,4 +1,4 @@ ProgramNode(0...4)( [], - StatementsNode(0...4)([InstanceVariableReadNode(0...4)(:foo)]) + StatementsNode(0...4)([InstanceVariableReadNode(0...4)(:@foo)]) ) diff --git a/test/yarp/snapshots/whitequark/ivasgn.txt b/test/yarp/snapshots/whitequark/ivasgn.txt index 587dcfaa9df007..8e952de2465dd1 100644 --- a/test/yarp/snapshots/whitequark/ivasgn.txt +++ b/test/yarp/snapshots/whitequark/ivasgn.txt @@ -2,7 +2,7 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( [InstanceVariableWriteNode(0...9)( - :var, + :@var, (0...4), IntegerNode(7...9)(), (5...6) diff --git a/test/yarp/snapshots/whitequark/masgn_splat.txt b/test/yarp/snapshots/whitequark/masgn_splat.txt index c6265c652e02f8..b093187ef62849 100644 --- a/test/yarp/snapshots/whitequark/masgn_splat.txt +++ b/test/yarp/snapshots/whitequark/masgn_splat.txt @@ -51,7 +51,7 @@ ProgramNode(0...139)( nil ), MultiWriteNode(47...65)( - [InstanceVariableTargetNode(47...51)(:foo), + [InstanceVariableTargetNode(47...51)(:@foo), ClassVariableTargetNode(53...58)()], (59...60), ArrayNode(61...65)( diff --git a/test/yarp/snapshots/whitequark/parser_bug_272.txt b/test/yarp/snapshots/whitequark/parser_bug_272.txt index 50d753980713e9..0cb194d1ae4126 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_272.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_272.txt @@ -6,7 +6,7 @@ ProgramNode(0...15)( nil, (0...1), nil, - ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:b)]), + ArgumentsNode(2...4)([InstanceVariableReadNode(2...4)(:@b)]), nil, BlockNode(5...15)( [:c], diff --git a/test/yarp/snapshots/whitequark/resbody_var.txt b/test/yarp/snapshots/whitequark/resbody_var.txt index 4530e6cdbd442a..e88a7612dc1b2a 100644 --- a/test/yarp/snapshots/whitequark/resbody_var.txt +++ b/test/yarp/snapshots/whitequark/resbody_var.txt @@ -10,7 +10,7 @@ ProgramNode(0...73)( (13...19), [], (20...22), - InstanceVariableTargetNode(23...26)(:ex), + InstanceVariableTargetNode(23...26)(:@ex), StatementsNode(28...31)( [CallNode(28...31)( nil, diff --git a/test/yarp/snapshots/whitequark/string_concat.txt b/test/yarp/snapshots/whitequark/string_concat.txt index fde498b39fecef..a51362805e4997 100644 --- a/test/yarp/snapshots/whitequark/string_concat.txt +++ b/test/yarp/snapshots/whitequark/string_concat.txt @@ -7,7 +7,7 @@ ProgramNode(0...14)( [StringNode(1...4)(nil, (1...4), nil, "foo"), EmbeddedVariableNode(4...7)( (4...5), - InstanceVariableReadNode(5...7)(:a) + InstanceVariableReadNode(5...7)(:@a) )], (7...8) ), diff --git a/test/yarp/snapshots/whitequark/string_dvar.txt b/test/yarp/snapshots/whitequark/string_dvar.txt index 2378e8fdcbc3c3..981ea318c4b356 100644 --- a/test/yarp/snapshots/whitequark/string_dvar.txt +++ b/test/yarp/snapshots/whitequark/string_dvar.txt @@ -5,7 +5,7 @@ ProgramNode(0...14)( (0...1), [EmbeddedVariableNode(1...4)( (1...2), - InstanceVariableReadNode(2...4)(:a) + InstanceVariableReadNode(2...4)(:@a) ), StringNode(4...5)(nil, (4...5), nil, " "), EmbeddedVariableNode(5...9)((5...6), ClassVariableReadNode(6...9)()), diff --git a/test/yarp/snapshots/whitequark/var_op_asgn.txt b/test/yarp/snapshots/whitequark/var_op_asgn.txt index 3aff7a6a60267a..5618d0944c3291 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn.txt @@ -8,7 +8,7 @@ ProgramNode(0...53)( :| ), InstanceVariableOperatorWriteNode(13...20)( - :a, + :@a, (13...15), (16...18), IntegerNode(19...20)(), diff --git a/yarp/yarp.c b/yarp/yarp.c index c9d7cd08b42ab7..46fe7259a45eb0 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2702,7 +2702,7 @@ yp_instance_variable_read_node_create(yp_parser_t *parser, const yp_token_t *tok .type = YP_NODE_INSTANCE_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) }, - .name = yp_parser_constant_id_location(parser, token->start + 1, token->end) + .name = yp_parser_constant_id_location(parser, token->start, token->end) }; return node; From 00dbee94ac71527cffbfa959a99f17457eb440fc Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 15:28:10 -0400 Subject: [PATCH 151/208] [ruby/yarp] Add class variables to the constant pool https://github.com/ruby/yarp/commit/be5cb60c83 --- lib/yarp/desugar_visitor.rb | 14 +++--- test/yarp/snapshots/methods.txt | 2 +- test/yarp/snapshots/patterns.txt | 2 +- test/yarp/snapshots/seattlerb/case_in.txt | 2 +- test/yarp/snapshots/strings.txt | 2 +- .../unparser/corpus/literal/assignment.txt | 5 ++- .../unparser/corpus/literal/dstr.txt | 2 +- .../unparser/corpus/literal/literal.txt | 2 +- .../unparser/corpus/literal/variables.txt | 2 +- .../unparser/corpus/semantic/dstr.txt | 2 +- test/yarp/snapshots/variables.txt | 8 ++-- test/yarp/snapshots/whitequark/cvar.txt | 5 ++- test/yarp/snapshots/whitequark/cvasgn.txt | 7 ++- .../yarp/snapshots/whitequark/masgn_splat.txt | 2 +- .../yarp/snapshots/whitequark/string_dvar.txt | 5 ++- .../yarp/snapshots/whitequark/var_op_asgn.txt | 2 + yarp/config.yml | 14 ++++++ yarp/yarp.c | 44 ++++++++++++------- 18 files changed, 81 insertions(+), 41 deletions(-) diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb index 8fe488368e9079..d0c0494e0c013e 100644 --- a/lib/yarp/desugar_visitor.rb +++ b/lib/yarp/desugar_visitor.rb @@ -8,7 +8,7 @@ class DesugarVisitor < MutationVisitor # # @@foo && @@foo = bar def visit_class_variable_and_write_node(node) - desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode) + desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, arguments: [node.name]) end # @@foo ||= bar @@ -17,7 +17,7 @@ def visit_class_variable_and_write_node(node) # # defined?(@@foo) ? @@foo : @@foo = bar def visit_class_variable_or_write_node(node) - desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode) + desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode, arguments: [node.name]) end # @@foo += bar @@ -26,7 +26,7 @@ def visit_class_variable_or_write_node(node) # # @@foo = @@foo + bar def visit_class_variable_operator_write_node(node) - desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode) + desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, arguments: [node.name]) end # Foo &&= bar @@ -245,15 +245,15 @@ def desugar_or_write_node(node, read_class, write_class, arguments: []) end # Don't desugar `x ||= y` to `defined?(x) ? x : x = y` - def desugar_or_write_defined_node(node, read_class, write_class) + def desugar_or_write_defined_node(node, read_class, write_class, arguments: []) IfNode.new( node.operator_loc, - DefinedNode.new(nil, read_class.new(node.name_loc), nil, node.operator_loc, node.name_loc), - StatementsNode.new([read_class.new(node.name_loc)], node.location), + DefinedNode.new(nil, read_class.new(*arguments, node.name_loc), nil, node.operator_loc, node.name_loc), + StatementsNode.new([read_class.new(*arguments, node.name_loc)], node.location), ElseNode.new( node.operator_loc, StatementsNode.new( - [write_class.new(node.name_loc, node.value, node.operator_loc, node.location)], + [write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location)], node.location ), node.operator_loc, diff --git a/test/yarp/snapshots/methods.txt b/test/yarp/snapshots/methods.txt index 70e030811c4e1c..b5ec66aae3d615 100644 --- a/test/yarp/snapshots/methods.txt +++ b/test/yarp/snapshots/methods.txt @@ -910,7 +910,7 @@ ProgramNode(0...1194)( ), DefNode(811...826)( (821...822), - ClassVariableReadNode(815...820)(), + ClassVariableReadNode(815...820)(:@@var), nil, nil, [], diff --git a/test/yarp/snapshots/patterns.txt b/test/yarp/snapshots/patterns.txt index 63764bca0beff5..01292cbf3caabd 100644 --- a/test/yarp/snapshots/patterns.txt +++ b/test/yarp/snapshots/patterns.txt @@ -941,7 +941,7 @@ ProgramNode(0...3743)( "foo" ), PinnedVariableNode(974...980)( - ClassVariableReadNode(975...980)(), + ClassVariableReadNode(975...980)(:@@bar), (974...975) ), (971...973) diff --git a/test/yarp/snapshots/seattlerb/case_in.txt b/test/yarp/snapshots/seattlerb/case_in.txt index 2e508d8c9ae135..aaee410ef325bd 100644 --- a/test/yarp/snapshots/seattlerb/case_in.txt +++ b/test/yarp/snapshots/seattlerb/case_in.txt @@ -502,7 +502,7 @@ ProgramNode(0...747)( (633...634) ), PinnedVariableNode(638...642)( - ClassVariableReadNode(639...642)(), + ClassVariableReadNode(639...642)(:@@c), (638...639) )], nil, diff --git a/test/yarp/snapshots/strings.txt b/test/yarp/snapshots/strings.txt index ee8bc79f17eeaf..86a732d85d5fbe 100644 --- a/test/yarp/snapshots/strings.txt +++ b/test/yarp/snapshots/strings.txt @@ -20,7 +20,7 @@ ProgramNode(0...498)( (122...123), [EmbeddedVariableNode(123...129)( (123...124), - ClassVariableReadNode(124...129)() + ClassVariableReadNode(124...129)(:@@foo) )], (129...130) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 7e193e0e87dd25..78ab4d3d78d1a8 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -54,8 +54,8 @@ ProgramNode(0...704)( (54...55) ), MultiWriteNode(65...84)( - [ClassVariableTargetNode(66...69)(), - ClassVariableTargetNode(71...74)()], + [ClassVariableTargetNode(66...69)(:@@a), + ClassVariableTargetNode(71...74)(:@@b)], (76...77), ArrayNode(78...84)( [IntegerNode(79...80)(), IntegerNode(82...83)()], @@ -291,6 +291,7 @@ ProgramNode(0...704)( ) ), ClassVariableWriteNode(302...309)( + :@@a, (302...305), IntegerNode(308...309)(), (306...307) diff --git a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt index 1abbed681f80bd..facf0171e790ed 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt @@ -84,7 +84,7 @@ ProgramNode(0...299)( [StringNode(167...168)(nil, (167...168), nil, "a"), EmbeddedVariableNode(168...172)( (168...169), - ClassVariableReadNode(169...172)() + ClassVariableReadNode(169...172)(:@@a) )], (172...173) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/literal.txt b/test/yarp/snapshots/unparser/corpus/literal/literal.txt index 03f9aecbd56ec0..3b85d3e95fb34f 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/literal.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/literal.txt @@ -158,7 +158,7 @@ ProgramNode(0...916)( StringNode(210...211)(nil, (210...211), nil, " "), EmbeddedVariableNode(211...215)( (211...212), - ClassVariableReadNode(212...215)() + ClassVariableReadNode(212...215)(:@@a) ), StringNode(215...216)(nil, (215...216), nil, " "), EmbeddedVariableNode(216...219)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/variables.txt b/test/yarp/snapshots/unparser/corpus/literal/variables.txt index 051fb2f498941a..dfb85ec01370ad 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/variables.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/variables.txt @@ -3,7 +3,7 @@ ProgramNode(0...66)( StatementsNode(0...66)( [CallNode(0...1)(nil, nil, (0...1), nil, nil, nil, nil, 2, "a"), InstanceVariableReadNode(2...4)(:@a), - ClassVariableReadNode(5...8)(), + ClassVariableReadNode(5...8)(:@@a), GlobalVariableReadNode(9...11)(), NumberedReferenceReadNode(12...14)(), BackReferenceReadNode(15...17)(), diff --git a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt index 47d37b1c38c43e..8822eb53337c33 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/dstr.txt @@ -246,7 +246,7 @@ ProgramNode(0...608)( [StringNode(598...599)(nil, (598...599), nil, "a"), EmbeddedVariableNode(599...603)( (599...600), - ClassVariableReadNode(600...603)() + ClassVariableReadNode(600...603)(:@@a) )], (603...604) ), diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index 2dd87366c5674e..419942d1eb997e 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -1,21 +1,23 @@ ProgramNode(0...293)( [:abc, :foo, :bar, :baz], StatementsNode(0...293)( - [ClassVariableReadNode(0...5)(), + [ClassVariableReadNode(0...5)(:@@abc), ClassVariableWriteNode(7...16)( + :@@abc, (7...12), IntegerNode(15...16)(), (13...14) ), MultiWriteNode(18...34)( - [ClassVariableTargetNode(18...23)(), - ClassVariableTargetNode(25...30)()], + [ClassVariableTargetNode(18...23)(:@@foo), + ClassVariableTargetNode(25...30)(:@@bar)], (31...32), IntegerNode(33...34)(), nil, nil ), ClassVariableWriteNode(36...48)( + :@@foo, (36...41), ArrayNode(44...48)( [IntegerNode(44...45)(), IntegerNode(47...48)()], diff --git a/test/yarp/snapshots/whitequark/cvar.txt b/test/yarp/snapshots/whitequark/cvar.txt index 069ff24b17ea3a..6a1f2e50f48566 100644 --- a/test/yarp/snapshots/whitequark/cvar.txt +++ b/test/yarp/snapshots/whitequark/cvar.txt @@ -1 +1,4 @@ -ProgramNode(0...5)([], StatementsNode(0...5)([ClassVariableReadNode(0...5)()])) +ProgramNode(0...5)( + [], + StatementsNode(0...5)([ClassVariableReadNode(0...5)(:@@foo)]) +) diff --git a/test/yarp/snapshots/whitequark/cvasgn.txt b/test/yarp/snapshots/whitequark/cvasgn.txt index b972caaab79f0e..7e07810eb6cc0f 100644 --- a/test/yarp/snapshots/whitequark/cvasgn.txt +++ b/test/yarp/snapshots/whitequark/cvasgn.txt @@ -1,6 +1,11 @@ ProgramNode(0...10)( [], StatementsNode(0...10)( - [ClassVariableWriteNode(0...10)((0...5), IntegerNode(8...10)(), (6...7))] + [ClassVariableWriteNode(0...10)( + :@@var, + (0...5), + IntegerNode(8...10)(), + (6...7) + )] ) ) diff --git a/test/yarp/snapshots/whitequark/masgn_splat.txt b/test/yarp/snapshots/whitequark/masgn_splat.txt index b093187ef62849..d21760a65b3921 100644 --- a/test/yarp/snapshots/whitequark/masgn_splat.txt +++ b/test/yarp/snapshots/whitequark/masgn_splat.txt @@ -52,7 +52,7 @@ ProgramNode(0...139)( ), MultiWriteNode(47...65)( [InstanceVariableTargetNode(47...51)(:@foo), - ClassVariableTargetNode(53...58)()], + ClassVariableTargetNode(53...58)(:@@bar)], (59...60), ArrayNode(61...65)( [SplatNode(61...65)( diff --git a/test/yarp/snapshots/whitequark/string_dvar.txt b/test/yarp/snapshots/whitequark/string_dvar.txt index 981ea318c4b356..12f083660bb8d7 100644 --- a/test/yarp/snapshots/whitequark/string_dvar.txt +++ b/test/yarp/snapshots/whitequark/string_dvar.txt @@ -8,7 +8,10 @@ ProgramNode(0...14)( InstanceVariableReadNode(2...4)(:@a) ), StringNode(4...5)(nil, (4...5), nil, " "), - EmbeddedVariableNode(5...9)((5...6), ClassVariableReadNode(6...9)()), + EmbeddedVariableNode(5...9)( + (5...6), + ClassVariableReadNode(6...9)(:@@a) + ), StringNode(9...10)(nil, (9...10), nil, " "), EmbeddedVariableNode(10...13)( (10...11), diff --git a/test/yarp/snapshots/whitequark/var_op_asgn.txt b/test/yarp/snapshots/whitequark/var_op_asgn.txt index 5618d0944c3291..0927e8b8f3aebc 100644 --- a/test/yarp/snapshots/whitequark/var_op_asgn.txt +++ b/test/yarp/snapshots/whitequark/var_op_asgn.txt @@ -2,6 +2,7 @@ ProgramNode(0...53)( [:a], StatementsNode(0...53)( [ClassVariableOperatorWriteNode(0...11)( + :@@var, (0...5), (6...8), IntegerNode(9...11)(), @@ -28,6 +29,7 @@ ProgramNode(0...53)( nil, StatementsNode(37...48)( [ClassVariableOperatorWriteNode(37...48)( + :@@var, (37...42), (43...45), IntegerNode(46...48)(), diff --git a/yarp/config.yml b/yarp/config.yml index b4e8b479137581..72c4aeb8af6c66 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -727,6 +727,8 @@ nodes: ^^^^^^^^^^^^^ - name: ClassVariableAndWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: operator_loc @@ -740,6 +742,8 @@ nodes: ^^^^^^^^^^^^^^^^ - name: ClassVariableOperatorWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: operator_loc @@ -755,6 +759,8 @@ nodes: ^^^^^^^^^^^^^^^^^ - name: ClassVariableOrWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: operator_loc @@ -767,12 +773,18 @@ nodes: @@target ||= value ^^^^^^^^^^^^^^^^^^ - name: ClassVariableReadNode + child_nodes: + - name: name + type: constant comment: | Represents referencing a class variable. @@foo ^^^^^ - name: ClassVariableTargetNode + child_nodes: + - name: name + type: constant comment: | Represents writing to a class variable in a context that doesn't have an explicit value. @@ -780,6 +792,8 @@ nodes: ^^^^^ ^^^^^ - name: ClassVariableWriteNode child_nodes: + - name: name + type: constant - name: name_loc type: location - name: value diff --git a/yarp/yarp.c b/yarp/yarp.c index 46fe7259a45eb0..6fd16e16fcdd57 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -1558,8 +1558,7 @@ yp_class_node_create(yp_parser_t *parser, yp_constant_id_list_t *locals, const y // Allocate and initialize a new ClassVariableAndWriteNode node. static yp_class_variable_and_write_node_t * -yp_class_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_CLASS_VARIABLE_READ_NODE)); +yp_class_variable_and_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { assert(operator->type == YP_TOKEN_AMPERSAND_AMPERSAND_EQUAL); yp_class_variable_and_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_and_write_node_t); @@ -1567,11 +1566,12 @@ yp_class_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, { .type = YP_NODE_CLASS_VARIABLE_AND_WRITE_NODE, .location = { - .start = target->location.start, + .start = target->base.location.start, .end = value->location.end } }, - .name_loc = target->location, + .name = target->name, + .name_loc = target->base.location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -1581,18 +1581,19 @@ yp_class_variable_and_write_node_create(yp_parser_t *parser, yp_node_t *target, // Allocate and initialize a new ClassVariableOperatorWriteNode node. static yp_class_variable_operator_write_node_t * -yp_class_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { +yp_class_variable_operator_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { yp_class_variable_operator_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_operator_write_node_t); *node = (yp_class_variable_operator_write_node_t) { { .type = YP_NODE_CLASS_VARIABLE_OPERATOR_WRITE_NODE, .location = { - .start = target->location.start, + .start = target->base.location.start, .end = value->location.end } }, - .name_loc = target->location, + .name = target->name, + .name_loc = target->base.location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value, .operator = yp_parser_constant_id_location(parser, operator->start, operator->end - 1) @@ -1603,8 +1604,7 @@ yp_class_variable_operator_write_node_create(yp_parser_t *parser, yp_node_t *tar // Allocate and initialize a new ClassVariableOrWriteNode node. static yp_class_variable_or_write_node_t * -yp_class_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, const yp_token_t *operator, yp_node_t *value) { - assert(YP_NODE_TYPE_P(target, YP_NODE_CLASS_VARIABLE_READ_NODE)); +yp_class_variable_or_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *target, const yp_token_t *operator, yp_node_t *value) { assert(operator->type == YP_TOKEN_PIPE_PIPE_EQUAL); yp_class_variable_or_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_or_write_node_t); @@ -1612,11 +1612,12 @@ yp_class_variable_or_write_node_create(yp_parser_t *parser, yp_node_t *target, c { .type = YP_NODE_CLASS_VARIABLE_OR_WRITE_NODE, .location = { - .start = target->location.start, + .start = target->base.location.start, .end = value->location.end } }, - .name_loc = target->location, + .name = target->name, + .name_loc = target->base.location, .operator_loc = YP_LOCATION_TOKEN_VALUE(operator), .value = value }; @@ -1629,13 +1630,21 @@ static yp_class_variable_read_node_t * yp_class_variable_read_node_create(yp_parser_t *parser, const yp_token_t *token) { assert(token->type == YP_TOKEN_CLASS_VARIABLE); yp_class_variable_read_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_read_node_t); - *node = (yp_class_variable_read_node_t) {{ .type = YP_NODE_CLASS_VARIABLE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(token) }}; + + *node = (yp_class_variable_read_node_t) { + { + .type = YP_NODE_CLASS_VARIABLE_READ_NODE, + .location = YP_LOCATION_TOKEN_VALUE(token) + }, + .name = yp_parser_constant_id_location(parser, token->start, token->end) + }; + return node; } // Initialize a new ClassVariableWriteNode node from a ClassVariableRead node. static yp_class_variable_write_node_t * -yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp_class_variable_read_node_t *read_node, yp_token_t *operator, yp_node_t *value) { +yp_class_variable_write_node_create(yp_parser_t *parser, yp_class_variable_read_node_t *read_node, yp_token_t *operator, yp_node_t *value) { yp_class_variable_write_node_t *node = YP_ALLOC_NODE(parser, yp_class_variable_write_node_t); *node = (yp_class_variable_write_node_t) { @@ -1646,6 +1655,7 @@ yp_class_variable_read_node_to_class_variable_write_node(yp_parser_t *parser, yp .end = value->location.end }, }, + .name = read_node->name, .name_loc = YP_LOCATION_NODE_VALUE((yp_node_t *) read_node), .operator_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator), .value = value @@ -8007,7 +8017,7 @@ parse_write(yp_parser_t *parser, yp_node_t *target, yp_token_t *operator, yp_nod case YP_NODE_MISSING_NODE: return target; case YP_NODE_CLASS_VARIABLE_READ_NODE: { - yp_class_variable_write_node_t *write_node = yp_class_variable_read_node_to_class_variable_write_node(parser, (yp_class_variable_read_node_t *) target, operator, value); + yp_class_variable_write_node_t *write_node = yp_class_variable_write_node_create(parser, (yp_class_variable_read_node_t *) target, operator, value); yp_node_destroy(parser, target); return (yp_node_t *) write_node; } @@ -12837,7 +12847,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after &&="); - yp_node_t *result = (yp_node_t *) yp_class_variable_and_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_class_variable_and_write_node_create(parser, (yp_class_variable_read_node_t *) node, &token, value); yp_node_destroy(parser, node); return result; @@ -12938,7 +12948,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after ||="); - yp_node_t *result = (yp_node_t *) yp_class_variable_or_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_class_variable_or_write_node_create(parser, (yp_class_variable_read_node_t *) node, &token, value); yp_node_destroy(parser, node); return result; @@ -13049,7 +13059,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t parser_lex(parser); yp_node_t *value = parse_expression(parser, binding_power, "Expected a value after the operator."); - yp_node_t *result = (yp_node_t *) yp_class_variable_operator_write_node_create(parser, node, &token, value); + yp_node_t *result = (yp_node_t *) yp_class_variable_operator_write_node_create(parser, (yp_class_variable_read_node_t *) node, &token, value); yp_node_destroy(parser, node); return result; From ae609a995e344877a990f4c16eca88b02dab5eba Mon Sep 17 00:00:00 2001 From: Jeremy Evans Date: Thu, 22 Jun 2023 16:23:44 -0700 Subject: [PATCH 152/208] Document that Kernel#p is for debugging and may be uninterruptible [ci skip] Fixes [Bug #18810] --- io.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/io.c b/io.c index bd1db9aa5d675d..433ec75b27f5e3 100644 --- a/io.c +++ b/io.c @@ -8944,6 +8944,10 @@ rb_p_result(int argc, const VALUE *argv) * 0..4 * [0..4, 0..4, 0..4] * + * Kernel#p is designed for debugging purposes. + * Ruby implementations may define Kernel#p to be uninterruptible + * in whole or in part. + * On CRuby, Kernel#p's writing of data is uninterruptible. */ static VALUE From f83070816dbe281e8cd2993494704c4a2af49a47 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 15:35:35 -0400 Subject: [PATCH 153/208] [ruby/yarp] Move templating logic until YARP https://github.com/ruby/yarp/commit/8b7430dbc7 --- yarp/templates/template.rb | 141 +++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 67 deletions(-) diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index e34d5b1a7be6ea..a5e25b61b126e0 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -254,81 +254,88 @@ def initialize(config) end end -# This templates out a file using ERB with the given locals. The locals are -# derived from the config.yml file. -def template(name, locals, write_to: nil) - filepath = "templates/#{name}.erb" - template = File.expand_path("../#{filepath}", __dir__) - write_to ||= File.expand_path("../#{name}", __dir__) - - if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ - erb = ERB.new(File.read(template), trim_mode: "-") - else - erb = ERB.new(File.read(template), nil, "-") - end - erb.filename = template - - non_ruby_heading = <<~HEADING - /******************************************************************************/ - /* This file is generated by the templates/template.rb script and should not */ - /* be modified manually. See */ - /* #{filepath + " " * (74 - filepath.size) } */ - /* if you are looking to modify the */ - /* template */ - /******************************************************************************/ - HEADING - - ruby_heading = <<~HEADING - # frozen_string_literal: true - =begin - This file is generated by the templates/template.rb script and should not be - modified manually. See #{filepath} - if you are looking to modify the template - =end - - HEADING - - heading = if File.extname(filepath.gsub(".erb", "")) == ".rb" - ruby_heading - else - non_ruby_heading +module YARP + class << self + # This templates out a file using ERB with the given locals. The locals are + # derived from the config.yml file. + def template(name, write_to: nil) + filepath = "templates/#{name}.erb" + template = File.expand_path("../#{filepath}", __dir__) + write_to ||= File.expand_path("../#{name}", __dir__) + + if ERB.instance_method(:initialize).parameters.assoc(:key) # Ruby 2.6+ + erb = ERB.new(File.read(template), trim_mode: "-") + else + erb = ERB.new(File.read(template), nil, "-") + end + erb.filename = template + + non_ruby_heading = <<~HEADING + /******************************************************************************/ + /* This file is generated by the templates/template.rb script and should not */ + /* be modified manually. See */ + /* #{filepath + " " * (74 - filepath.size) } */ + /* if you are looking to modify the */ + /* template */ + /******************************************************************************/ + HEADING + + ruby_heading = <<~HEADING + # frozen_string_literal: true + =begin + This file is generated by the templates/template.rb script and should not be + modified manually. See #{filepath} + if you are looking to modify the template + =end + + HEADING + + heading = if File.extname(filepath.gsub(".erb", "")) == ".rb" + ruby_heading + else + non_ruby_heading + end + + contents = heading + erb.result_with_hash(locals) + FileUtils.mkdir_p(File.dirname(write_to)) + File.write(write_to, contents) end - contents = heading + erb.result_with_hash(locals) - FileUtils.mkdir_p(File.dirname(write_to)) - File.write(write_to, contents) -end - -def locals - config = YAML.load_file(File.expand_path("../config.yml", __dir__)) + private + + def locals + @locals ||= + YAML.load_file(File.expand_path("../config.yml", __dir__)).then do |config| + { + nodes: config.fetch("nodes").map { |node| NodeType.new(node) }.sort_by(&:name), + tokens: config.fetch("tokens").map { |token| Token.new(token) }, + flags: config.fetch("flags").map { |flags| Flags.new(flags) } + } + end + end + end - { - nodes: config.fetch("nodes").map { |node| NodeType.new(node) }.sort_by(&:name), - tokens: config.fetch("tokens").map { |token| Token.new(token) }, - flags: config.fetch("flags").map { |flags| Flags.new(flags) } - } + TEMPLATES = [ + "ext/yarp/api_node.c", + "include/yarp/ast.h", + "java/org/yarp/Loader.java", + "java/org/yarp/Nodes.java", + "java/org/yarp/AbstractNodeVisitor.java", + "lib/yarp/mutation_visitor.rb", + "lib/yarp/node.rb", + "lib/yarp/serialize.rb", + "src/node.c", + "src/prettyprint.c", + "src/serialize.c", + "src/token_type.c" + ] end -TEMPLATES = [ - "ext/yarp/api_node.c", - "include/yarp/ast.h", - "java/org/yarp/Loader.java", - "java/org/yarp/Nodes.java", - "java/org/yarp/AbstractNodeVisitor.java", - "lib/yarp/mutation_visitor.rb", - "lib/yarp/node.rb", - "lib/yarp/serialize.rb", - "src/node.c", - "src/prettyprint.c", - "src/serialize.c", - "src/token_type.c" -] - if __FILE__ == $0 if ARGV.empty? - TEMPLATES.each { |f| template(f, locals) } + YARP::TEMPLATES.each { |filepath| YARP.template(filepath) } else name, write_to = ARGV - template(name, locals, write_to: write_to) + YARP.template(name, write_to: write_to) end end From ff024150e766a0db5ee9517ab1b0e7c84beca14b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 16:09:51 -0400 Subject: [PATCH 154/208] [ruby/yarp] Rename child_nodes to fields https://github.com/ruby/yarp/commit/715fce9264 --- yarp/config.yml | 226 +++++----- yarp/templates/ext/yarp/api_node.c.erb | 78 ++-- yarp/templates/include/yarp/ast.h.erb | 24 +- yarp/templates/java/org/yarp/Loader.java.erb | 26 +- yarp/templates/java/org/yarp/Nodes.java.erb | 72 ++-- .../lib/yarp/mutation_visitor.rb.erb | 6 +- yarp/templates/lib/yarp/node.rb.erb | 80 ++-- yarp/templates/lib/yarp/serialize.rb.erb | 24 +- yarp/templates/src/node.c.erb | 66 +-- yarp/templates/src/prettyprint.c.erb | 74 ++-- yarp/templates/src/serialize.c.erb | 70 ++-- yarp/templates/template.rb | 391 +++++++++--------- 12 files changed, 570 insertions(+), 567 deletions(-) diff --git a/yarp/config.yml b/yarp/config.yml index 72c4aeb8af6c66..b1e21edf9ef2e3 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -361,7 +361,7 @@ flags: comment: "o - only interpolates values into the regular expression once" nodes: - name: AliasNode - child_nodes: + fields: - name: new_name type: node - name: old_name @@ -374,7 +374,7 @@ nodes: alias foo bar ^^^^^^^^^^^^^ - name: AlternationPatternNode - child_nodes: + fields: - name: left type: node - name: right @@ -387,7 +387,7 @@ nodes: foo => bar | baz ^^^^^^^^^ - name: AndNode - child_nodes: + fields: - name: left type: node - name: right @@ -400,7 +400,7 @@ nodes: left and right ^^^^^^^^^^^^^^ - name: ArgumentsNode - child_nodes: + fields: - name: arguments type: node[] comment: | @@ -409,7 +409,7 @@ nodes: return foo, bar, baz ^^^^^^^^^^^^^ - name: ArrayNode - child_nodes: + fields: - name: elements type: node[] - name: opening_loc @@ -423,7 +423,7 @@ nodes: [1, 2, 3] ^^^^^^^^^ - name: ArrayPatternNode - child_nodes: + fields: - name: constant type: node? - name: requireds @@ -454,7 +454,7 @@ nodes: foo in Bar[1, 2, 3] ^^^^^^^^^^^^^^^^^^^ - name: AssocNode - child_nodes: + fields: - name: key type: node - name: value @@ -467,7 +467,7 @@ nodes: { a => b } ^^^^^^ - name: AssocSplatNode - child_nodes: + fields: - name: value type: node? - name: operator_loc @@ -484,7 +484,7 @@ nodes: $' ^^ - name: BeginNode - child_nodes: + fields: - name: begin_keyword_loc type: location? - name: statements @@ -510,7 +510,7 @@ nodes: end ^^^^^ - name: BlockArgumentNode - child_nodes: + fields: - name: expression type: node? - name: operator_loc @@ -521,7 +521,7 @@ nodes: bar(&args) ^^^^^^^^^^ - name: BlockNode - child_nodes: + fields: - name: locals type: constant[] - name: parameters @@ -539,7 +539,7 @@ nodes: [1, 2, 3].each { |i| puts x } ^^^^^^^^^^^^^^ - name: BlockParameterNode - child_nodes: + fields: - name: name_loc type: location? - name: operator_loc @@ -551,7 +551,7 @@ nodes: ^^ end - name: BlockParametersNode - child_nodes: + fields: - name: parameters type: node? kind: ParametersNode @@ -571,7 +571,7 @@ nodes: ^^^^^^^^^^^^^^^^^ end - name: BreakNode - child_nodes: + fields: - name: arguments type: node? kind: ArgumentsNode @@ -583,7 +583,7 @@ nodes: break foo ^^^^^^^^^ - name: CallNode - child_nodes: + fields: - name: receiver type: node? - name: operator_loc @@ -626,7 +626,7 @@ nodes: foo&.bar ^^^^^^^^ - name: CallOperatorAndWriteNode - child_nodes: + fields: - name: target type: node kind: CallNode @@ -640,7 +640,7 @@ nodes: foo.bar &&= value ^^^^^^^^^^^^^^^^^ - name: CallOperatorOrWriteNode - child_nodes: + fields: - name: target type: node kind: CallNode @@ -654,7 +654,7 @@ nodes: foo.bar ||= value ^^^^^^^^^^^^^^^^^ - name: CallOperatorWriteNode - child_nodes: + fields: - name: target type: node kind: CallNode @@ -670,7 +670,7 @@ nodes: foo.bar += baz ^^^^^^^^^^^^^^ - name: CapturePatternNode - child_nodes: + fields: - name: value type: node - name: target @@ -683,7 +683,7 @@ nodes: foo => [bar => baz] ^^^^^^^^^^^^ - name: CaseNode - child_nodes: + fields: - name: predicate type: node? - name: conditions @@ -703,7 +703,7 @@ nodes: when false end - name: ClassNode - child_nodes: + fields: - name: locals type: constant[] - name: class_keyword_loc @@ -726,7 +726,7 @@ nodes: class Foo end ^^^^^^^^^^^^^ - name: ClassVariableAndWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -741,7 +741,7 @@ nodes: @@target &&= value ^^^^^^^^^^^^^^^^ - name: ClassVariableOperatorWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -758,7 +758,7 @@ nodes: @@target += value ^^^^^^^^^^^^^^^^^ - name: ClassVariableOrWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -791,7 +791,7 @@ nodes: @@foo, @@bar = baz ^^^^^ ^^^^^ - name: ClassVariableWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -806,7 +806,7 @@ nodes: @@foo = 1 ^^^^^^^^^ - name: ConstantAndWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -819,7 +819,7 @@ nodes: Target &&= value ^^^^^^^^^^^^^^^^ - name: ConstantOperatorWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -834,7 +834,7 @@ nodes: Target += value ^^^^^^^^^^^^^^^ - name: ConstantOrWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -847,7 +847,7 @@ nodes: Target ||= value ^^^^^^^^^^^^^^^^ - name: ConstantPathAndWriteNode - child_nodes: + fields: - name: target type: node kind: ConstantPathNode @@ -861,7 +861,7 @@ nodes: Parent::Child &&= value ^^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathNode - child_nodes: + fields: - name: parent type: node? - name: child @@ -874,7 +874,7 @@ nodes: Foo::Bar ^^^^^^^^ - name: ConstantPathOperatorWriteNode - child_nodes: + fields: - name: target type: node kind: ConstantPathNode @@ -890,7 +890,7 @@ nodes: Parent::Child += value ^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathOrWriteNode - child_nodes: + fields: - name: target type: node kind: ConstantPathNode @@ -904,7 +904,7 @@ nodes: Parent::Child ||= value ^^^^^^^^^^^^^^^^^^^^^^^ - name: ConstantPathTargetNode - child_nodes: + fields: - name: parent type: node? - name: child @@ -917,7 +917,7 @@ nodes: Foo::Foo, Bar::Bar = baz ^^^^^^^^ ^^^^^^^^ - name: ConstantPathWriteNode - child_nodes: + fields: - name: target type: node kind: ConstantPathNode @@ -949,7 +949,7 @@ nodes: Foo, Bar = baz ^^^ ^^^ - name: ConstantWriteNode - child_nodes: + fields: - name: name_loc type: location - name: value @@ -962,7 +962,7 @@ nodes: Foo = 1 ^^^^^^^ - name: DefNode - child_nodes: + fields: - name: name_loc type: location - name: receiver @@ -993,7 +993,7 @@ nodes: end ^^^^^^^^^^ - name: DefinedNode - child_nodes: + fields: - name: lparen_loc type: location? - name: value @@ -1008,7 +1008,7 @@ nodes: defined?(a) ^^^^^^^^^^^ - name: ElseNode - child_nodes: + fields: - name: else_keyword_loc type: location - name: statements @@ -1022,7 +1022,7 @@ nodes: if a then b else c end ^^^^^^^^^^ - name: EmbeddedStatementsNode - child_nodes: + fields: - name: opening_loc type: location - name: statements @@ -1036,7 +1036,7 @@ nodes: "foo #{bar}" ^^^^^^ - name: EmbeddedVariableNode - child_nodes: + fields: - name: operator_loc type: location - name: variable @@ -1047,7 +1047,7 @@ nodes: "foo #@bar" ^^^^^ - name: EnsureNode - child_nodes: + fields: - name: ensure_keyword_loc type: location - name: statements @@ -1071,7 +1071,7 @@ nodes: false ^^^^^ - name: FindPatternNode - child_nodes: + fields: - name: constant type: node? - name: left @@ -1096,7 +1096,7 @@ nodes: foo in Foo(*bar, baz, *qux) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - name: FlipFlopNode - child_nodes: + fields: - name: left type: node? - name: right @@ -1118,7 +1118,7 @@ nodes: 1.0 ^^^ - name: ForNode - child_nodes: + fields: - name: index type: node - name: collection @@ -1155,7 +1155,7 @@ nodes: ^^^ end - name: ForwardingSuperNode - child_nodes: + fields: - name: block type: node? kind: BlockNode @@ -1165,7 +1165,7 @@ nodes: super ^^^^^ - name: GlobalVariableAndWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1178,7 +1178,7 @@ nodes: $target &&= value ^^^^^^^^^^^^^^^^^ - name: GlobalVariableOperatorWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1193,7 +1193,7 @@ nodes: $target += value ^^^^^^^^^^^^^^^^ - name: GlobalVariableOrWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1218,7 +1218,7 @@ nodes: $foo, $bar = baz ^^^^ ^^^^ - name: GlobalVariableWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1231,7 +1231,7 @@ nodes: $foo = 1 ^^^^^^^^ - name: HashNode - child_nodes: + fields: - name: opening_loc type: location - name: elements @@ -1244,7 +1244,7 @@ nodes: { a => b } ^^^^^^^^^^ - name: HashPatternNode - child_nodes: + fields: - name: constant type: node? - name: assocs @@ -1264,7 +1264,7 @@ nodes: foo => { a: 1, b: 2, **c } ^^^^^^^^^^^^^^^^^^^ - name: IfNode - child_nodes: + fields: - name: if_keyword_loc type: location? - name: predicate @@ -1286,7 +1286,7 @@ nodes: if foo then bar end ^^^^^^^^^^^^^^^^^^^ - name: ImaginaryNode - child_nodes: + fields: - name: numeric type: node comment: | @@ -1295,7 +1295,7 @@ nodes: 1.0i ^^^^ - name: InNode - child_nodes: + fields: - name: pattern type: node - name: statements @@ -1311,7 +1311,7 @@ nodes: case a; in b then c end ^^^^^^^^^^^ - name: InstanceVariableAndWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -1326,7 +1326,7 @@ nodes: @target &&= value ^^^^^^^^^^^^^^^^^ - name: InstanceVariableOperatorWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -1343,7 +1343,7 @@ nodes: @target += value ^^^^^^^^^^^^^^^^ - name: InstanceVariableOrWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -1358,7 +1358,7 @@ nodes: @target ||= value ^^^^^^^^^^^^^^^^^ - name: InstanceVariableReadNode - child_nodes: + fields: - name: name type: constant comment: | @@ -1367,7 +1367,7 @@ nodes: @foo ^^^^ - name: InstanceVariableTargetNode - child_nodes: + fields: - name: name type: constant comment: | @@ -1376,7 +1376,7 @@ nodes: @foo, @bar = baz ^^^^ ^^^^ - name: InstanceVariableWriteNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -1397,7 +1397,7 @@ nodes: 1 ^ - name: InterpolatedRegularExpressionNode - child_nodes: + fields: - name: opening_loc type: location - name: parts @@ -1414,7 +1414,7 @@ nodes: /foo #{bar} baz/ ^^^^^^^^^^^^^^^^ - name: InterpolatedStringNode - child_nodes: + fields: - name: opening_loc type: location? - name: parts @@ -1428,7 +1428,7 @@ nodes: "foo #{bar} baz" ^^^^^^^^^^^^^^^^ - name: InterpolatedSymbolNode - child_nodes: + fields: - name: opening_loc type: location? - name: parts @@ -1442,7 +1442,7 @@ nodes: :"foo #{bar} baz" ^^^^^^^^^^^^^^^^^ - name: InterpolatedXStringNode - child_nodes: + fields: - name: opening_loc type: location - name: parts @@ -1456,7 +1456,7 @@ nodes: `foo #{bar} baz` ^^^^^^^^^^^^^^^^ - name: KeywordHashNode - child_nodes: + fields: - name: elements type: node[] comment: | @@ -1465,7 +1465,7 @@ nodes: foo(a: b) ^^^^ - name: KeywordParameterNode - child_nodes: + fields: - name: name_loc type: location - name: value @@ -1481,7 +1481,7 @@ nodes: ^^^^ end - name: KeywordRestParameterNode - child_nodes: + fields: - name: operator_loc type: location - name: name_loc @@ -1493,7 +1493,7 @@ nodes: ^^^ end - name: LambdaNode - child_nodes: + fields: - name: locals type: constant[] - name: operator_loc @@ -1513,7 +1513,7 @@ nodes: ->(value) { value * 2 } ^^^^^^^^^^^^^^^^^^^^^^^ - name: LocalVariableAndWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1530,7 +1530,7 @@ nodes: target &&= value ^^^^^^^^^^^^^^^^ - name: LocalVariableOperatorWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1549,7 +1549,7 @@ nodes: target += value ^^^^^^^^^^^^^^^ - name: LocalVariableOrWriteNode - child_nodes: + fields: - name: name_loc type: location - name: operator_loc @@ -1566,7 +1566,7 @@ nodes: target ||= value ^^^^^^^^^^^^^^^^ - name: LocalVariableReadNode - child_nodes: + fields: - name: name type: constant - name: depth @@ -1579,7 +1579,7 @@ nodes: foo ^^^ - name: LocalVariableTargetNode - child_nodes: + fields: - name: name type: constant - name: depth @@ -1590,7 +1590,7 @@ nodes: foo, bar = baz ^^^ ^^^ - name: LocalVariableWriteNode - child_nodes: + fields: - name: name type: constant - name: depth @@ -1607,7 +1607,7 @@ nodes: foo = 1 ^^^^^^^ - name: MatchPredicateNode - child_nodes: + fields: - name: value type: node - name: pattern @@ -1620,7 +1620,7 @@ nodes: foo in bar ^^^^^^^^^^ - name: MatchRequiredNode - child_nodes: + fields: - name: value type: node - name: pattern @@ -1637,7 +1637,7 @@ nodes: Represents a node that is missing from the source and results in a syntax error. - name: ModuleNode - child_nodes: + fields: - name: locals type: constant[] - name: module_keyword_loc @@ -1656,7 +1656,7 @@ nodes: module Foo end ^^^^^^^^^^^^^^ - name: MultiWriteNode - child_nodes: + fields: - name: targets type: node[] - name: operator_loc @@ -1673,7 +1673,7 @@ nodes: a, b, c = 1, 2, 3 ^^^^^^^^^^^^^^^^^ - name: NextNode - child_nodes: + fields: - name: arguments type: node? kind: ArgumentsNode @@ -1691,7 +1691,7 @@ nodes: nil ^^^ - name: NoKeywordsParameterNode - child_nodes: + fields: - name: operator_loc type: location - name: keyword_loc @@ -1709,7 +1709,7 @@ nodes: $1 ^^ - name: OptionalParameterNode - child_nodes: + fields: - name: name type: constant - name: name_loc @@ -1725,7 +1725,7 @@ nodes: ^^^^^ end - name: OrNode - child_nodes: + fields: - name: left type: node - name: right @@ -1738,7 +1738,7 @@ nodes: left or right ^^^^^^^^^^^^^ - name: ParametersNode - child_nodes: + fields: - name: requireds type: node[] - name: optionals @@ -1762,7 +1762,7 @@ nodes: ^^^^^^^ end - name: ParenthesesNode - child_nodes: + fields: - name: body type: node? - name: opening_loc @@ -1776,7 +1776,7 @@ nodes: (10 + 34) ^^^^^^^^^ - name: PinnedExpressionNode - child_nodes: + fields: - name: expression type: node - name: operator_loc @@ -1792,7 +1792,7 @@ nodes: foo in ^(bar) ^^^^^^ - name: PinnedVariableNode - child_nodes: + fields: - name: variable type: node - name: operator_loc @@ -1804,7 +1804,7 @@ nodes: foo in ^bar ^^^^ - name: PostExecutionNode - child_nodes: + fields: - name: statements type: node? kind: StatementsNode @@ -1820,7 +1820,7 @@ nodes: END { foo } ^^^^^^^^^^^ - name: PreExecutionNode - child_nodes: + fields: - name: statements type: node? kind: StatementsNode @@ -1836,7 +1836,7 @@ nodes: BEGIN { foo } ^^^^^^^^^^^^^ - name: ProgramNode - child_nodes: + fields: - name: locals type: constant[] - name: statements @@ -1844,7 +1844,7 @@ nodes: kind: StatementsNode comment: The top level node of any parse tree. - name: RangeNode - child_nodes: + fields: - name: left type: node? - name: right @@ -1863,7 +1863,7 @@ nodes: c if a =~ /left/ ... b =~ /right/ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - name: RationalNode - child_nodes: + fields: - name: numeric type: node comment: | @@ -1878,7 +1878,7 @@ nodes: redo ^^^^ - name: RegularExpressionNode - child_nodes: + fields: - name: opening_loc type: location - name: content_loc @@ -1896,7 +1896,7 @@ nodes: /foo/i ^^^^^^ - name: RequiredDestructuredParameterNode - child_nodes: + fields: - name: parameters type: node[] - name: opening_loc @@ -1910,7 +1910,7 @@ nodes: ^^^^^^^^^^ end - name: RequiredParameterNode - child_nodes: + fields: - name: name type: constant comment: | @@ -1920,7 +1920,7 @@ nodes: ^ end - name: RescueModifierNode - child_nodes: + fields: - name: expression type: node - name: keyword_loc @@ -1934,7 +1934,7 @@ nodes: foo rescue nil ^^^^^^^^^^^^^^ - name: RescueNode - child_nodes: + fields: - name: keyword_loc type: location - name: exceptions @@ -1961,7 +1961,7 @@ nodes: `Foo, *splat, Bar` are in the `exceptions` field. `ex` is in the `exception` field. - name: RestParameterNode - child_nodes: + fields: - name: operator_loc type: location - name: name_loc @@ -1979,7 +1979,7 @@ nodes: retry ^^^^^ - name: ReturnNode - child_nodes: + fields: - name: keyword_loc type: location - name: arguments @@ -1997,7 +1997,7 @@ nodes: self ^^^^ - name: SingletonClassNode - child_nodes: + fields: - name: locals type: constant[] - name: class_keyword_loc @@ -2022,7 +2022,7 @@ nodes: __ENCODING__ ^^^^^^^^^^^^ - name: SourceFileNode - child_nodes: + fields: - name: filepath type: string comment: | @@ -2037,7 +2037,7 @@ nodes: __LINE__ ^^^^^^^^ - name: SplatNode - child_nodes: + fields: - name: operator_loc type: location - name: expression @@ -2048,7 +2048,7 @@ nodes: [*a] ^^ - name: StatementsNode - child_nodes: + fields: - name: body type: node[] comment: | @@ -2057,7 +2057,7 @@ nodes: foo; bar; baz ^^^^^^^^^^^^^ - name: StringConcatNode - child_nodes: + fields: - name: left type: node - name: right @@ -2068,7 +2068,7 @@ nodes: "foo" "bar" ^^^^^^^^^^^ - name: StringNode - child_nodes: + fields: - name: opening_loc type: location? - name: content_loc @@ -2090,7 +2090,7 @@ nodes: "foo #{bar} baz" ^^^^ ^^^^ - name: SuperNode - child_nodes: + fields: - name: keyword_loc type: location - name: lparen_loc @@ -2112,7 +2112,7 @@ nodes: super foo, bar ^^^^^^^^^^^^^^ - name: SymbolNode - child_nodes: + fields: - name: opening_loc type: location? - name: value_loc @@ -2136,7 +2136,7 @@ nodes: true ^^^^ - name: UndefNode - child_nodes: + fields: - name: names type: node[] - name: keyword_loc @@ -2147,7 +2147,7 @@ nodes: undef :foo, :bar, :baz ^^^^^^^^^^^^^^^^^^^^^^ - name: UnlessNode - child_nodes: + fields: - name: keyword_loc type: location - name: predicate @@ -2170,7 +2170,7 @@ nodes: unless foo then bar end ^^^^^^^^^^^^^^^^^^^^^^^ - name: UntilNode - child_nodes: + fields: - name: keyword_loc type: location - name: closing_loc @@ -2193,7 +2193,7 @@ nodes: until foo do bar end ^^^^^^^^^^^^^^^^^^^^ - name: WhenNode - child_nodes: + fields: - name: keyword_loc type: location - name: conditions @@ -2209,7 +2209,7 @@ nodes: ^^^^^^^^^ end - name: WhileNode - child_nodes: + fields: - name: keyword_loc type: location - name: closing_loc @@ -2232,7 +2232,7 @@ nodes: while foo do bar end ^^^^^^^^^^^^^^^^^^^^ - name: XStringNode - child_nodes: + fields: - name: opening_loc type: location - name: content_loc @@ -2247,7 +2247,7 @@ nodes: `foo` ^^^^^ - name: YieldNode - child_nodes: + fields: - name: keyword_loc type: location - name: lparen_loc diff --git a/yarp/templates/ext/yarp/api_node.c.erb b/yarp/templates/ext/yarp/api_node.c.erb index fb25919ca38a62..b8407350589711 100644 --- a/yarp/templates/ext/yarp/api_node.c.erb +++ b/yarp/templates/ext/yarp/api_node.c.erb @@ -106,17 +106,17 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) { switch (YP_NODE_TYPE(node)) { <%- nodes.each do |node| -%> - <%- if node.params.any? { |param| [NodeParam, OptionalNodeParam, NodeListParam].include?(param.class) } -%> + <%- if node.fields.any? { |field| [YARP::NodeField, YARP::OptionalNodeField, YARP::NodeListField].include?(field.class) } -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when NodeParam, OptionalNodeParam -%> - yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= param.name %>); - <%- when NodeListParam -%> - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { - yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= param.name %>.nodes[index]); + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> + yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= field.name %>); + <%- when YARP::NodeListField -%> + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { + yp_node_stack_push(&node_stack, (yp_node_t *) cast-><%= field.name %>.nodes[index]); } <%- end -%> <%- end -%> @@ -135,53 +135,53 @@ yp_ast_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { - <%- if node.params.any? { |param| ![NodeParam, OptionalNodeParam].include?(param.class) } -%> + <%- if node.fields.any? { |field| ![YARP::NodeField, YARP::OptionalNodeField].include?(field.class) } -%> yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; <%- end -%> - VALUE argv[<%= node.params.length + 1 %>]; - <%- node.params.each_with_index do |param, index| -%> + VALUE argv[<%= node.fields.length + 1 %>]; + <%- node.fields.each_with_index do |field, index| -%> - // <%= param.name %> - <%- case param -%> - <%- when NodeParam, OptionalNodeParam -%> + // <%= field.name %> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> argv[<%= index %>] = rb_ary_pop(value_stack); - <%- when NodeListParam -%> - argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size); - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { + <%- when YARP::NodeListField -%> + argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size); + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { rb_ary_push(argv[<%= index %>], rb_ary_pop(value_stack)); } - <%- when StringParam -%> - argv[<%= index %>] = yp_string_new(&cast-><%= param.name %>, encoding); - <%- when LocationListParam -%> - argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size); - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { - yp_location_t location = cast-><%= param.name %>.locations[index]; + <%- when YARP::StringField -%> + argv[<%= index %>] = yp_string_new(&cast-><%= field.name %>, encoding); + <%- when YARP::LocationListField -%> + argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size); + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { + yp_location_t location = cast-><%= field.name %>.locations[index]; rb_ary_push(argv[<%= index %>], yp_location_new(parser, location.start, location.end, source)); } - <%- when ConstantParam -%> - argv[<%= index %>] = rb_id2sym(constants[cast-><%= param.name %> - 1]); - <%- when ConstantListParam -%> - argv[<%= index %>] = rb_ary_new_capa(cast-><%= param.name %>.size); - for (size_t index = 0; index < cast-><%= param.name %>.size; index++) { - rb_ary_push(argv[<%= index %>], rb_id2sym(constants[cast-><%= param.name %>.ids[index] - 1])); + <%- when YARP::ConstantField -%> + argv[<%= index %>] = rb_id2sym(constants[cast-><%= field.name %> - 1]); + <%- when YARP::ConstantListField -%> + argv[<%= index %>] = rb_ary_new_capa(cast-><%= field.name %>.size); + for (size_t index = 0; index < cast-><%= field.name %>.size; index++) { + rb_ary_push(argv[<%= index %>], rb_id2sym(constants[cast-><%= field.name %>.ids[index] - 1])); } - <%- when LocationParam -%> - argv[<%= index %>] = yp_location_new(parser, cast-><%= param.name %>.start, cast-><%= param.name %>.end, source); - <%- when OptionalLocationParam -%> - argv[<%= index %>] = cast-><%= param.name %>.start == NULL ? Qnil : yp_location_new(parser, cast-><%= param.name %>.start, cast-><%= param.name %>.end, source); - <%- when UInt32Param -%> - argv[<%= index %>] = ULONG2NUM(cast-><%= param.name %>); - <%- when FlagsParam -%> - argv[<%= index %>] = ULONG2NUM(node->flags >> <%= COMMON_FLAGS %>); + <%- when YARP::LocationField -%> + argv[<%= index %>] = yp_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source); + <%- when YARP::OptionalLocationField -%> + argv[<%= index %>] = cast-><%= field.name %>.start == NULL ? Qnil : yp_location_new(parser, cast-><%= field.name %>.start, cast-><%= field.name %>.end, source); + <%- when YARP::UInt32Field -%> + argv[<%= index %>] = ULONG2NUM(cast-><%= field.name %>); + <%- when YARP::FlagsField -%> + argv[<%= index %>] = ULONG2NUM(node->flags >> <%= YARP::COMMON_FLAGS %>); <%- else -%> <%- raise -%> <%- end -%> <%- end -%> // location - argv[<%= node.params.length %>] = yp_location_new(parser, node->location.start, node->location.end, source); + argv[<%= node.fields.length %>] = yp_location_new(parser, node->location.start, node->location.end, source); - rb_ary_push(value_stack, rb_class_new_instance(<%= node.params.length + 1 %>, argv, rb_cYARP<%= node.name %>)); + rb_ary_push(value_stack, rb_class_new_instance(<%= node.fields.length + 1 %>, argv, rb_cYARP<%= node.name %>)); break; } <%- end -%> diff --git a/yarp/templates/include/yarp/ast.h.erb b/yarp/templates/include/yarp/ast.h.erb index 6eeadb49d292d4..4325d403778da6 100644 --- a/yarp/templates/include/yarp/ast.h.erb +++ b/yarp/templates/include/yarp/ast.h.erb @@ -83,17 +83,17 @@ typedef struct yp_node { // <%= node.name %> typedef struct yp_<%= node.human %> { yp_node_t base; -<%- node.params.grep_v(FlagsParam).each do |param| -%> - <%= case param - when NodeParam, OptionalNodeParam then "struct #{param.c_type} *#{param.name}" - when NodeListParam then "struct yp_node_list #{param.name}" - when LocationListParam then "yp_location_list_t #{param.name}" - when ConstantParam then "yp_constant_id_t #{param.name}" - when ConstantListParam then "yp_constant_id_list_t #{param.name}" - when StringParam then "yp_string_t #{param.name}" - when LocationParam, OptionalLocationParam then "yp_location_t #{param.name}" - when UInt32Param then "uint32_t #{param.name}" - else raise param.class.name +<%- node.fields.grep_v(YARP::FlagsField).each do |field| -%> + <%= case field + when YARP::NodeField, YARP::OptionalNodeField then "struct #{field.c_type} *#{field.name}" + when YARP::NodeListField then "struct yp_node_list #{field.name}" + when YARP::LocationListField then "yp_location_list_t #{field.name}" + when YARP::ConstantField then "yp_constant_id_t #{field.name}" + when YARP::ConstantListField then "yp_constant_id_list_t #{field.name}" + when YARP::StringField then "yp_string_t #{field.name}" + when YARP::LocationField, YARP::OptionalLocationField then "yp_location_t #{field.name}" + when YARP::UInt32Field then "uint32_t #{field.name}" + else raise field.class.name end %>; <%- end -%> @@ -103,7 +103,7 @@ typedef struct yp_<%= node.human %> { // <%= flag.name %> typedef enum { - <%- flag.values.each.with_index(COMMON_FLAGS) do |value, index| -%> + <%- flag.values.each.with_index(YARP::COMMON_FLAGS) do |value, index| -%> YP_<%= flag.human.upcase %>_<%= value.name %> = 1 << <%= index %>, <%- end -%> } yp_<%= flag.human %>_t; diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb index 1c3a64354c0c11..4cab15be2082be 100644 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ b/yarp/templates/java/org/yarp/Loader.java.erb @@ -257,19 +257,19 @@ public class Loader { case <%= index + 1 %>: <%- params = node.needs_serialized_length? ? ["buffer.getInt()"] : [] - params.concat node.params.map { |param| - case param - when NodeParam then "#{param.java_cast}loadNode()" - when OptionalNodeParam then "#{param.java_cast}loadOptionalNode()" - when StringParam then "loadString()" - when NodeListParam then "loadNodes()" - when LocationListParam then "loadLocations()" - when ConstantParam then "loadConstant()" - when ConstantListParam then "loadConstants()" - when LocationParam then "loadLocation()" - when OptionalLocationParam then "loadOptionalLocation()" - when UInt32Param then "loadVarInt()" - when FlagsParam then "loadFlags()" + params.concat node.fields.map { |field| + case field + when YARP::NodeField then "#{field.java_cast}loadNode()" + when YARP::OptionalNodeField then "#{field.java_cast}loadOptionalNode()" + when YARP::StringField then "loadString()" + when YARP::NodeListField then "loadNodes()" + when YARP::LocationListField then "loadLocations()" + when YARP::ConstantField then "loadConstant()" + when YARP::ConstantListField then "loadConstants()" + when YARP::LocationField then "loadLocation()" + when YARP::OptionalLocationField then "loadOptionalLocation()" + when YARP::UInt32Field then "loadVarInt()" + when YARP::FlagsField then "loadFlags()" else raise end } diff --git a/yarp/templates/java/org/yarp/Nodes.java.erb b/yarp/templates/java/org/yarp/Nodes.java.erb index aa16f4729d3c1e..772e1e21e4d8d3 100644 --- a/yarp/templates/java/org/yarp/Nodes.java.erb +++ b/yarp/templates/java/org/yarp/Nodes.java.erb @@ -189,13 +189,13 @@ public abstract class Nodes { <%- if node.needs_serialized_length? -%> public final int serializedLength; <%- end -%> - <%- node.params.each do |param| -%> - public final <%= param.java_type %> <%= param.name %>;<%= ' // optional' if param.class.name.start_with?('Optional') %> + <%- node.fields.each do |field| -%> + public final <%= field.java_type %> <%= field.name %>;<%= ' // optional' if field.class.name.start_with?('Optional') %> <%- end -%> <%- params = node.needs_serialized_length? ? ["int serializedLength"] : [] - params.concat node.params.map { "#{_1.java_type} #{_1.name}" } + params.concat node.fields.map { "#{_1.java_type} #{_1.name}" } params.concat ["int startOffset", "int length"] -%> public <%=node.name -%>(<%= params.join(", ") %>) { @@ -203,17 +203,17 @@ public abstract class Nodes { <%- if node.needs_serialized_length? -%> this.serializedLength = serializedLength; <%- end -%> - <%- node.params.each do |param| -%> - this.<%= param.name %> = <%= param.name %>; + <%- node.fields.each do |field| -%> + this.<%= field.name %> = <%= field.name %>; <%- end -%> } <%# methods for flags -%> - <%- node.params.each do |param| -%> - <%- if param.is_a?(FlagsParam) -%> - <%- flags.find { |flag| flag.name == param.kind }.tap { raise "Expected to find #{param.kind}" unless _1 }.values.each do |value| -%> + <%- node.fields.each do |field| -%> + <%- if field.is_a?(YARP::FlagsField) -%> + <%- flags.find { |flag| flag.name == field.kind }.tap { raise "Expected to find #{field.kind}" unless _1 }.values.each do |value| -%> public boolean is<%= value.camelcase %>() { - return <%= param.kind %>.is<%= value.camelcase %>(this.<%= param.name %>); + return <%= field.kind %>.is<%= value.camelcase %>(this.<%= field.name %>); } <%- end -%> <%- end -%> @@ -229,52 +229,52 @@ public abstract class Nodes { @Override public void setNewLineFlag(Source source, boolean[] newlineMarked) { - <%- param = node.params.find { |p| p.name == node.newline } or raise node.newline -%> - <%- case param -%> - <%- when SingleNodeParam -%> - this.<%= param.name %>.setNewLineFlag(source, newlineMarked); - <%- when NodeListParam -%> - Node first = this.<%= param.name %>.length > 0 ? this.<%= param.name %>[0] : null; + <%- field = node.fields.find { |f| f.name == node.newline } or raise node.newline -%> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> + this.<%= field.name %>.setNewLineFlag(source, newlineMarked); + <%- when YARP::NodeListField -%> + Node first = this.<%= field.name %>.length > 0 ? this.<%= field.name %>[0] : null; if (first != null) { first.setNewLineFlag(source, newlineMarked); } - <%- else raise param.class.name -%> + <%- else raise field.class.name -%> <%- end -%> } <%- end -%> public void visitChildNodes(AbstractNodeVisitor visitor) { - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when NodeListParam -%> - for (Nodes.Node child : this.<%= param.name %>) { + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::NodeListField -%> + for (Nodes.Node child : this.<%= field.name %>) { child.accept(visitor); } - <%- when NodeParam -%> - this.<%= param.name %>.accept(visitor); - <%- when OptionalNodeParam -%> - if (this.<%= param.name %> != null) { - this.<%= param.name %>.accept(visitor); + <%- when YARP::NodeField -%> + this.<%= field.name %>.accept(visitor); + <%- when YARP::OptionalNodeField -%> + if (this.<%= field.name %> != null) { + this.<%= field.name %>.accept(visitor); } <%- end -%> <%- end -%> } public Node[] childNodes() { - <%- if node.params.none?(NodeListParam) and node.params.none?(SingleNodeParam) -%> + <%- if node.fields.none?(YARP::NodeListField) and node.fields.none?(YARP::NodeKindField) -%> return EMPTY_ARRAY; - <%- elsif node.params.one?(NodeListParam) and node.params.none?(SingleNodeParam) -%> - return this.<%= node.params.grep(NodeListParam).first.name %>; - <%- elsif node.params.none?(NodeListParam) -%> - return new Node[] { <%= node.params.grep(SingleNodeParam).map { "this.#{_1.name}" }.join(', ') %> }; + <%- elsif node.fields.one?(YARP::NodeListField) and node.fields.none?(YARP::NodeKindField) -%> + return this.<%= node.fields.grep(YARP::NodeListField).first.name %>; + <%- elsif node.fields.none?(YARP::NodeListField) -%> + return new Node[] { <%= node.fields.grep(YARP::NodeKindField).map { "this.#{_1.name}" }.join(', ') %> }; <%- else -%> ArrayList childNodes = new ArrayList<>(); - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when SingleNodeParam -%> - childNodes.add(this.<%= param.name %>); - <%- when NodeListParam -%> - childNodes.addAll(Arrays.asList(this.<%= param.name %>)); + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> + childNodes.add(this.<%= field.name %>); + <%- when YARP::NodeListField -%> + childNodes.addAll(Arrays.asList(this.<%= field.name %>)); <%- end -%> <%- end -%> return childNodes.toArray(EMPTY_ARRAY); diff --git a/yarp/templates/lib/yarp/mutation_visitor.rb.erb b/yarp/templates/lib/yarp/mutation_visitor.rb.erb index 82e6bc32c07f31..3bb2dfb4b27616 100644 --- a/yarp/templates/lib/yarp/mutation_visitor.rb.erb +++ b/yarp/templates/lib/yarp/mutation_visitor.rb.erb @@ -7,9 +7,9 @@ module YARP <%= "\n" if index != 0 -%> # Copy a <%= node.name %> node def visit_<%= node.human %>(node) - <%- params = node.params.select { |param| [NodeParam, OptionalNodeParam, NodeListParam].include?(param.class) } -%> - <%- if params.any? -%> - node.copy(<%= params.map { |param| "#{param.name}: #{param.is_a?(NodeListParam) ? "visit_all" : "visit"}(node.#{param.name})" }.join(", ") %>) + <%- fields = node.fields.select { |field| [YARP::NodeField, YARP::OptionalNodeField, YARP::NodeListField].include?(field.class) } -%> + <%- if fields.any? -%> + node.copy(<%= fields.map { |field| "#{field.name}: #{field.is_a?(YARP::NodeListField) ? "visit_all" : "visit"}(node.#{field.name})" }.join(", ") %>) <%- else -%> node.copy <%- end -%> diff --git a/yarp/templates/lib/yarp/node.rb.erb b/yarp/templates/lib/yarp/node.rb.erb index 1a86350da889ac..6241eabd12bd0a 100644 --- a/yarp/templates/lib/yarp/node.rb.erb +++ b/yarp/templates/lib/yarp/node.rb.erb @@ -1,15 +1,15 @@ module YARP <%- nodes.each do |node| -%> <%= "#{node.comment.split("\n").map { |line| line.empty? ? "#" : "# #{line}" }.join("\n ")}\n " if node.comment %>class <%= node.name -%> < Node - <%- node.params.each do |param| -%> - # attr_reader <%= param.name %>: <%= param.rbs_class %> - attr_reader :<%= param.name %> + <%- node.fields.each do |field| -%> + # attr_reader <%= field.name %>: <%= field.rbs_class %> + attr_reader :<%= field.name %> <%- end -%> - # def initialize: (<%= (node.params.map { |param| "#{param.name}: #{param.rbs_class}" } + ["location: Location"]).join(", ") %>) -> void - def initialize(<%= (node.params.map(&:name) + ["location"]).join(", ") %>) - <%- node.params.each do |param| -%> - @<%= param.name %> = <%= param.name %> + # def initialize: (<%= (node.fields.map { |field| "#{field.name}: #{field.rbs_class}" } + ["location: Location"]).join(", ") %>) -> void + def initialize(<%= (node.fields.map(&:name) + ["location"]).join(", ") %>) + <%- node.fields.each do |field| -%> + @<%= field.name %> = <%= field.name %> <%- end -%> @location = location end @@ -26,24 +26,24 @@ module YARP <%- elsif node.newline.is_a?(String) -%> def set_newline_flag(newline_marked) - <%- param = node.params.find { |p| p.name == node.newline } or raise node.newline -%> - <%- case param -%> - <%- when SingleNodeParam -%> - <%= param.name %>.set_newline_flag(newline_marked) - <%- when NodeListParam -%> - first = <%= param.name %>.first + <%- field = node.fields.find { |f| f.name == node.newline } or raise node.newline -%> + <%- case field -%> + <%- when YARP::NodeField, YARP::OptionalNodeField -%> + <%= field.name %>.set_newline_flag(newline_marked) + <%- when YARP::NodeListField -%> + first = <%= field.name %>.first first.set_newline_flag(newline_marked) if first - <%- else raise param.class.name -%> + <%- else raise field.class.name -%> <%- end -%> end <%- end -%> # def child_nodes: () -> Array[nil | Node] def child_nodes - [<%= node.params.map { |param| - case param - when SingleNodeParam then param.name - when NodeListParam then "*#{param.name}" + [<%= node.fields.map { |field| + case field + when YARP::NodeField, YARP::OptionalNodeField then field.name + when YARP::NodeListField then "*#{field.name}" end }.compact.join(", ") %>] end @@ -51,7 +51,7 @@ module YARP # def copy: (**params) -> <%= node.name %> def copy(**params) <%= node.name %>.new( - <%- (node.params.map(&:name) + ["location"]).map do |name| -%> + <%- (node.fields.map(&:name) + ["location"]).map do |name| -%> params.fetch(:<%= name %>) { <%= name %> }, <%- end -%> ) @@ -62,32 +62,32 @@ module YARP # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location] def deconstruct_keys(keys) - { <%= (node.params.map { |param| "#{param.name}: #{param.name}" } + ["location: location"]).join(", ") %> } + { <%= (node.fields.map { |field| "#{field.name}: #{field.name}" } + ["location: location"]).join(", ") %> } end - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when LocationParam -%> - <%- raise unless param.name.end_with?("_loc") -%> - <%- next if node.params.any? { |other| other.name == param.name.delete_suffix("_loc") } -%> - - # def <%= param.name.delete_suffix("_loc") %>: () -> String - def <%= param.name.delete_suffix("_loc") %> - <%= param.name %>.slice + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::LocationField -%> + <%- raise unless field.name.end_with?("_loc") -%> + <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> + + # def <%= field.name.delete_suffix("_loc") %>: () -> String + def <%= field.name.delete_suffix("_loc") %> + <%= field.name %>.slice end - <%- when OptionalLocationParam -%> - <%- raise unless param.name.end_with?("_loc") -%> - <%- next if node.params.any? { |other| other.name == param.name.delete_suffix("_loc") } -%> + <%- when YARP::OptionalLocationField -%> + <%- raise unless field.name.end_with?("_loc") -%> + <%- next if node.fields.any? { |other| other.name == field.name.delete_suffix("_loc") } -%> - # def <%= param.name.delete_suffix("_loc") %>: () -> String? - def <%= param.name.delete_suffix("_loc") %> - <%= param.name %>&.slice + # def <%= field.name.delete_suffix("_loc") %>: () -> String? + def <%= field.name.delete_suffix("_loc") %> + <%= field.name %>&.slice end - <%- when FlagsParam -%> - <%- flags.find { |flag| flag.name == param.kind }.tap { |flag| raise "Expected to find #{param.kind}" unless flag }.values.each do |value| -%> + <%- when YARP::FlagsField -%> + <%- flags.find { |flag| flag.name == field.kind }.tap { |flag| raise "Expected to find #{field.kind}" unless flag }.values.each do |value| -%> # def <%= value.name.downcase %>?: () -> bool def <%= value.name.downcase %>? - <%= param.name %>.anybits?(<%= param.kind %>::<%= value.name %>) + <%= field.name %>.anybits?(<%= field.kind %>::<%= value.name %>) end <%- end -%> <%- end -%> @@ -123,8 +123,8 @@ module YARP <%- nodes.each do |node| -%> # Create a new <%= node.name %> node - def <%= node.name %>(<%= (node.params.map(&:name) + ["location = Location()"]).join(", ") %>) - <%= node.name %>.new(<%= (node.params.map(&:name) + ["location"]).join(", ") %>) + def <%= node.name %>(<%= (node.fields.map(&:name) + ["location = Location()"]).join(", ") %>) + <%= node.name %>.new(<%= (node.fields.map(&:name) + ["location"]).join(", ") %>) end <%- end -%> end diff --git a/yarp/templates/lib/yarp/serialize.rb.erb b/yarp/templates/lib/yarp/serialize.rb.erb index 1959a71b62d19c..d54c3119543d99 100644 --- a/yarp/templates/lib/yarp/serialize.rb.erb +++ b/yarp/templates/lib/yarp/serialize.rb.erb @@ -179,18 +179,18 @@ module YARP <%- if node.needs_serialized_length? -%> load_serialized_length <%- end -%> - <%= node.name %>.new(<%= (node.params.map { |param| - case param - when NodeParam then "load_node" - when OptionalNodeParam then "load_optional_node" - when StringParam then "load_string" - when NodeListParam then "Array.new(load_varint) { load_node }" - when LocationListParam then "Array.new(load_varint) { load_location }" - when ConstantParam then "load_constant" - when ConstantListParam then "Array.new(load_varint) { load_constant }" - when LocationParam then "load_location" - when OptionalLocationParam then "load_optional_location" - when UInt32Param, FlagsParam then "load_varint" + <%= node.name %>.new(<%= (node.fields.map { |field| + case field + when YARP::NodeField then "load_node" + when YARP::OptionalNodeField then "load_optional_node" + when YARP::StringField then "load_string" + when YARP::NodeListField then "Array.new(load_varint) { load_node }" + when YARP::LocationListField then "Array.new(load_varint) { load_location }" + when YARP::ConstantField then "load_constant" + when YARP::ConstantListField then "Array.new(load_varint) { load_constant }" + when YARP::LocationField then "load_location" + when YARP::OptionalLocationField then "load_optional_location" + when YARP::UInt32Field, YARP::FlagsField then "load_varint" else raise end } + ["location"]).join(", ") -%>) diff --git a/yarp/templates/src/node.c.erb b/yarp/templates/src/node.c.erb index c173f677601612..aa756ed4f9a81f 100644 --- a/yarp/templates/src/node.c.erb +++ b/yarp/templates/src/node.c.erb @@ -79,26 +79,26 @@ yp_node_destroy(yp_parser_t *parser, yp_node_t *node) { <%- nodes.each do |node| -%> #line <%= __LINE__ + 1 %> "<%= File.basename(__FILE__) %>" case <%= node.type %>: { - <%- if node.params.any? { |param| ![LocationParam, OptionalLocationParam, UInt32Param, FlagsParam, ConstantParam].include?(param.class) } -%> + <%- if node.fields.any? { |field| ![YARP::LocationField, YARP::OptionalLocationField, YARP::UInt32Field, YARP::FlagsField, YARP::ConstantField].include?(field.class) } -%> yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; <%- end -%> - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when LocationParam, OptionalLocationParam, UInt32Param, FlagsParam, ConstantParam -%> - <%- when NodeParam -%> - yp_node_destroy(parser, (yp_node_t *)cast-><%= param.name %>); - <%- when OptionalNodeParam -%> - if (cast-><%= param.name %> != NULL) { - yp_node_destroy(parser, (yp_node_t *)cast-><%= param.name %>); + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::LocationField, YARP::OptionalLocationField, YARP::UInt32Field, YARP::FlagsField, YARP::ConstantField -%> + <%- when YARP::NodeField -%> + yp_node_destroy(parser, (yp_node_t *)cast-><%= field.name %>); + <%- when YARP::OptionalNodeField -%> + if (cast-><%= field.name %> != NULL) { + yp_node_destroy(parser, (yp_node_t *)cast-><%= field.name %>); } - <%- when StringParam -%> - yp_string_free(&cast-><%= param.name %>); - <%- when NodeListParam -%> - yp_node_list_free(parser, &cast-><%= param.name %>); - <%- when LocationListParam -%> - yp_location_list_free(&cast-><%= param.name %>); - <%- when ConstantListParam -%> - yp_constant_id_list_free(&cast-><%= param.name %>); + <%- when YARP::StringField -%> + yp_string_free(&cast-><%= field.name %>); + <%- when YARP::NodeListField -%> + yp_node_list_free(parser, &cast-><%= field.name %>); + <%- when YARP::LocationListField -%> + yp_location_list_free(&cast-><%= field.name %>); + <%- when YARP::ConstantListField -%> + yp_constant_id_list_free(&cast-><%= field.name %>); <%- else -%> <%- raise -%> <%- end -%> @@ -128,23 +128,23 @@ yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) { case <%= node.type %>: { yp_<%= node.human %>_t *cast = (yp_<%= node.human %>_t *) node; memsize->memsize += sizeof(*cast); - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when ConstantParam, UInt32Param, FlagsParam, LocationParam, OptionalLocationParam -%> - <%- when NodeParam -%> - yp_node_memsize_node((yp_node_t *)cast-><%= param.name %>, memsize); - <%- when OptionalNodeParam -%> - if (cast-><%= param.name %> != NULL) { - yp_node_memsize_node((yp_node_t *)cast-><%= param.name %>, memsize); + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::ConstantField, YARP::UInt32Field, YARP::FlagsField, YARP::LocationField, YARP::OptionalLocationField -%> + <%- when YARP::NodeField -%> + yp_node_memsize_node((yp_node_t *)cast-><%= field.name %>, memsize); + <%- when YARP::OptionalNodeField -%> + if (cast-><%= field.name %> != NULL) { + yp_node_memsize_node((yp_node_t *)cast-><%= field.name %>, memsize); } - <%- when StringParam -%> - memsize->memsize += yp_string_memsize(&cast-><%= param.name %>); - <%- when NodeListParam -%> - yp_node_list_memsize(&cast-><%= param.name %>, memsize); - <%- when LocationListParam -%> - memsize->memsize += yp_location_list_memsize(&cast-><%= param.name %>); - <%- when ConstantListParam -%> - memsize->memsize += yp_constant_id_list_memsize(&cast-><%= param.name %>); + <%- when YARP::StringField -%> + memsize->memsize += yp_string_memsize(&cast-><%= field.name %>); + <%- when YARP::NodeListField -%> + yp_node_list_memsize(&cast-><%= field.name %>, memsize); + <%- when YARP::LocationListField -%> + memsize->memsize += yp_location_list_memsize(&cast-><%= field.name %>); + <%- when YARP::ConstantListField -%> + memsize->memsize += yp_constant_id_list_memsize(&cast-><%= field.name %>); <%- else -%> <%- raise -%> <%- end -%> diff --git a/yarp/templates/src/prettyprint.c.erb b/yarp/templates/src/prettyprint.c.erb index ded483c04aed38..4c1f49fdd2f017 100644 --- a/yarp/templates/src/prettyprint.c.erb +++ b/yarp/templates/src/prettyprint.c.erb @@ -23,64 +23,64 @@ prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) { <%- nodes.each do |node| -%> case <%= node.type %>: { yp_buffer_append_str(buffer, "<%= node.name %>(", <%= node.name.length + 1 %>); - <%- node.params.each_with_index do |param, index| -%> + <%- node.fields.each_with_index do |field, index| -%> <%= "yp_buffer_append_str(buffer, \", \", 2);" if index != 0 -%> - <%- case param -%> - <%- when NodeParam -%> - prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> == NULL) { + <%- case field -%> + <%- when YARP::NodeField -%> + prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>); + <%- when YARP::OptionalNodeField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) { yp_buffer_append_str(buffer, "nil", 3); } else { - prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>); + prettyprint_node(buffer, parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>); } - <%- when StringParam -%> + <%- when YARP::StringField -%> yp_buffer_append_str(buffer, "\"", 1); - yp_buffer_append_bytes(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= param.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= param.name %>)); + yp_buffer_append_bytes(buffer, yp_string_source(&((yp_<%= node.human %>_t *)node)-><%= field.name %>), yp_string_length(&((yp_<%= node.human %>_t *)node)-><%= field.name %>)); yp_buffer_append_str(buffer, "\"", 1); - <%- when NodeListParam -%> + <%- when YARP::NodeListField -%> yp_buffer_append_str(buffer, "[", 1); - for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) { + for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) { if (index != 0) yp_buffer_append_str(buffer, ", ", 2); - prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= param.name %>.nodes[index]); + prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= field.name %>.nodes[index]); } yp_buffer_append_str(buffer, "]", 1); - <%- when LocationListParam -%> + <%- when YARP::LocationListField -%> yp_buffer_append_str(buffer, "[", 1); - for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) { + for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) { if (index != 0) yp_buffer_append_str(buffer, ", ", 2); - prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index]); + prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>.locations[index]); } yp_buffer_append_str(buffer, "]", 1); - <%- when ConstantParam -%> - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= param.name %>); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); - <%- when ConstantListParam -%> + <%- when YARP::ConstantField -%> + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); + <%- when YARP::ConstantListField -%> yp_buffer_append_str(buffer, "[", 1); - for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) { + for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= field.name %>.size; index++) { if (index != 0) yp_buffer_append_str(buffer, ", ", 2); - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index]); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "%u", ((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index]); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); } yp_buffer_append_str(buffer, "]", 1); - <%- when LocationParam -%> - prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when OptionalLocationParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) { + <%- when YARP::LocationField -%> + prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>); + <%- when YARP::OptionalLocationField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { yp_buffer_append_str(buffer, "nil", 3); } else { - prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>); + prettyprint_location(buffer, parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>); } - <%- when UInt32Param -%> - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "+%d", ((yp_<%= node.human %>_t *)node)-><%= param.name %>); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); - <%- when FlagsParam -%> - char <%= param.name %>_buffer[12]; - snprintf(<%= param.name %>_buffer, sizeof(<%= param.name %>_buffer), "+%d", node->flags >> <%= COMMON_FLAGS %>); - yp_buffer_append_str(buffer, <%= param.name %>_buffer, strlen(<%= param.name %>_buffer)); + <%- when YARP::UInt32Field -%> + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", ((yp_<%= node.human %>_t *)node)-><%= field.name %>); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); + <%- when YARP::FlagsField -%> + char <%= field.name %>_buffer[12]; + snprintf(<%= field.name %>_buffer, sizeof(<%= field.name %>_buffer), "+%d", node->flags >> <%= YARP::COMMON_FLAGS %>); + yp_buffer_append_str(buffer, <%= field.name %>_buffer, strlen(<%= field.name %>_buffer)); <%- else -%> <%- raise -%> <%- end -%> diff --git a/yarp/templates/src/serialize.c.erb b/yarp/templates/src/serialize.c.erb index 635757b982cee1..b1049ba11668bf 100644 --- a/yarp/templates/src/serialize.c.erb +++ b/yarp/templates/src/serialize.c.erb @@ -68,51 +68,51 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) { size_t length_offset = buffer->length; yp_buffer_append_str(buffer, "\0\0\0\0", 4); /* consume 4 bytes, updated below */ <%- end -%> - <%- node.params.each do |param| -%> - <%- case param -%> - <%- when NodeParam -%> - yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); - <%- when OptionalNodeParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %> == NULL) { + <%- node.fields.each do |field| -%> + <%- case field -%> + <%- when YARP::NodeField -%> + yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- when YARP::OptionalNodeField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %> == NULL) { yp_buffer_append_u8(buffer, 0); } else { - yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); + yp_serialize_node(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } - <%- when StringParam -%> - yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); - <%- when NodeListParam -%> - uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); - yp_buffer_append_u32(buffer, <%= param.name %>_size); - for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= param.name %>.nodes[index], buffer); + <%- when YARP::StringField -%> + yp_serialize_string(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- when YARP::NodeListField -%> + uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size); + yp_buffer_append_u32(buffer, <%= field.name %>_size); + for (uint32_t index = 0; index < <%= field.name %>_size; index++) { + yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= field.name %>.nodes[index], buffer); } - <%- when LocationListParam -%> - uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); - yp_buffer_append_u32(buffer, <%= param.name %>_size); - for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>.locations[index], buffer); + <%- when YARP::LocationListField -%> + uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size); + yp_buffer_append_u32(buffer, <%= field.name %>_size); + for (uint32_t index = 0; index < <%= field.name %>_size; index++) { + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>.locations[index], buffer); } - <%- when ConstantParam -%> - yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>)); - <%- when ConstantListParam -%> - uint32_t <%= param.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size); - yp_buffer_append_u32(buffer, <%= param.name %>_size); - for (uint32_t index = 0; index < <%= param.name %>_size; index++) { - yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.ids[index])); + <%- when YARP::ConstantField -%> + yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>)); + <%- when YARP::ConstantListField -%> + uint32_t <%= field.name %>_size = yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.size); + yp_buffer_append_u32(buffer, <%= field.name %>_size); + for (uint32_t index = 0; index < <%= field.name %>_size; index++) { + yp_buffer_append_u32(buffer, yp_sizet_to_u32(((yp_<%= node.human %>_t *)node)-><%= field.name %>.ids[index])); } - <%- when LocationParam -%> - yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); - <%- when OptionalLocationParam -%> - if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.start == NULL) { + <%- when YARP::LocationField -%> + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); + <%- when YARP::OptionalLocationField -%> + if (((yp_<%= node.human %>_t *)node)-><%= field.name %>.start == NULL) { yp_buffer_append_u8(buffer, 0); } else { yp_buffer_append_u8(buffer, 1); - yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer); + yp_serialize_location(parser, &((yp_<%= node.human %>_t *)node)-><%= field.name %>, buffer); } - <%- when UInt32Param -%> - yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= param.name %>); - <%- when FlagsParam -%> - yp_buffer_append_u32(buffer, node->flags >> <%= COMMON_FLAGS %>); + <%- when YARP::UInt32Field -%> + yp_buffer_append_u32(buffer, ((yp_<%= node.human %>_t *)node)-><%= field.name %>); + <%- when YARP::FlagsField -%> + yp_buffer_append_u32(buffer, node->flags >> <%= YARP::COMMON_FLAGS %>); <%- else -%> <%- raise -%> <%- end -%> diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index a5e25b61b126e0..c72fdbc2c9fd7d 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -4,257 +4,260 @@ require "fileutils" require "yaml" -COMMON_FLAGS = 1 +module YARP + COMMON_FLAGS = 1 -class Param - attr_reader :name, :options + # This represents a field on a node. It contains all of the necessary + # information to template out the code for that field. + class Field + attr_reader :name, :options - def initialize(name:, type:, **options) - @name, @type, @options = name, type, options + def initialize(name:, type:, **options) + @name, @type, @options = name, type, options + end end -end -module KindTypes - def c_type - if options[:kind] - "yp_#{options[:kind].gsub(/(?<=.)[A-Z]/, "_\\0").downcase}" - else - "yp_node" + # Some node fields can be specialized if they point to a specific kind of + # node and not just a generic node. + class NodeKindField < Field + def c_type + if options[:kind] + "yp_#{options[:kind].gsub(/(?<=.)[A-Z]/, "_\\0").downcase}" + else + "yp_node" + end end - end - def ruby_type - options[:kind] || "Node" - end + def ruby_type + options[:kind] || "Node" + end - def java_type - options[:kind] || "Node" - end + def java_type + options[:kind] || "Node" + end - def java_cast - if options[:kind] - "(Nodes.#{options[:kind]}) " - else - "" + def java_cast + if options[:kind] + "(Nodes.#{options[:kind]}) " + else + "" + end end end -end -# This represents a parameter to a node that is itself a node. We pass them as -# references and store them as references. -class NodeParam < Param - include KindTypes + # This represents a field on a node that is itself a node. We pass them as + # references and store them as references. + class NodeField < NodeKindField + def rbs_class + ruby_type + end + end - def rbs_class - ruby_type + # This represents a field on a node that is itself a node and can be + # optionally null. We pass them as references and store them as references. + class OptionalNodeField < NodeKindField + def rbs_class + "#{ruby_type}?" + end end -end -# This represents a parameter to a node that is itself a node and can be -# optionally null. We pass them as references and store them as references. -class OptionalNodeParam < Param - include KindTypes + # This represents a field on a node that is a list of nodes. We pass them as + # references and store them directly on the struct. + class NodeListField < Field + def rbs_class + "Array[Node]" + end - def rbs_class - "#{ruby_type}?" + def java_type + "Node[]" + end end -end -SingleNodeParam = -> (node) { NodeParam === node or OptionalNodeParam === node } + # This represents a field on a node that is a list of locations. + class LocationListField < Field + def rbs_class + "Array[Location]" + end -# This represents a parameter to a node that is a list of nodes. We pass them as -# references and store them as references. -class NodeListParam < Param - def rbs_class - "Array[Node]" + def java_type + "Location[]" + end end - def java_type - "Node[]" - end -end + # This represents a field on a node that is the ID of a string interned + # through the parser's constant pool. + class ConstantField < Field + def rbs_class + "Symbol" + end -# This represents a parameter to a node that is a list of locations. -class LocationListParam < Param - def rbs_class - "Array[Location]" + def java_type + "byte[]" + end end - def java_type - "Location[]" - end -end + # This represents a field on a node that is a list of IDs that are associated + # with strings interned through the parser's constant pool. + class ConstantListField < Field + def rbs_class + "Array[Symbol]" + end -# This represents a parameter to a node that is the ID of a string interned -# through the parser's constant pool. -class ConstantParam < Param - def rbs_class - "Symbol" + def java_type + "byte[][]" + end end - def java_type - "byte[]" - end -end + # This represents a field on a node that is a string. + class StringField < Field + def rbs_class + "String" + end -# This represents a parameter to a node that is a list of IDs that are -# associated with strings interned through the parser's constant pool. -class ConstantListParam < Param - def rbs_class - "Array[Symbol]" + def java_type + "byte[]" + end end - def java_type - "byte[][]" - end -end + # This represents a field on a node that is a location. + class LocationField < Field + def rbs_class + "Location" + end -# This represents a parameter to a node that is a string. -class StringParam < Param - def rbs_class - "String" + def java_type + "Location" + end end - def java_type - "byte[]" - end -end + # This represents a field on a node that is a location that is optional. + class OptionalLocationField < Field + def rbs_class + "Location?" + end -# This represents a parameter to a node that is a location. -class LocationParam < Param - def rbs_class - "Location" + def java_type + "Location" + end end - def java_type - "Location" - end -end + # This represents an integer field. + class UInt32Field < Field + def rbs_class + "Integer" + end -# This represents a parameter to a node that is a location that is optional. -class OptionalLocationParam < Param - def rbs_class - "Location?" + def java_type + "int" + end end - def java_type - "Location" - end -end + # This represents a set of flags. It is very similar to the UInt32Field, but + # can be directly embedded into the flags field on the struct and provides + # convenient methods for checking if a flag is set. + class FlagsField < Field + def rbs_class + "Integer" + end -# This represents an integer parameter. -class UInt32Param < Param - def rbs_class - "Integer" - end + def java_type + "short" + end - def java_type - "int" + def kind + options.fetch(:kind) + end end -end -# This represents a set of flags. It is very similar to the UInt32Param, but can -# be directly embedded into the flags field on the struct and provides -# convenient methods for checking if a flag is set. -class FlagsParam < Param - def rbs_class - "Integer" - end + # This class represents a node in the tree, configured by the config.yml file in + # YAML format. It contains information about the name of the node and the + # various child nodes it contains. + class NodeType + attr_reader :name, :type, :human, :fields, :newline, :comment - def java_type - "short" - end + def initialize(config) + @name = config.fetch("name") - def kind - options.fetch(:kind) - end -end + type = @name.gsub(/(?<=.)[A-Z]/, "_\\0") + @type = "YP_NODE_#{type.upcase}" + @human = type.downcase -PARAM_TYPES = { - "node" => NodeParam, - "node?" => OptionalNodeParam, - "node[]" => NodeListParam, - "string" => StringParam, - "location[]" => LocationListParam, - "constant" => ConstantParam, - "constant[]" => ConstantListParam, - "location" => LocationParam, - "location?" => OptionalLocationParam, - "uint32" => UInt32Param, - "flags" => FlagsParam -} - -# This class represents a node in the tree, configured by the config.yml file in -# YAML format. It contains information about the name of the node and the -# various child nodes it contains. -class NodeType - attr_reader :name, :type, :human, :params, :newline, :comment - - def initialize(config) - @name = config.fetch("name") - - type = @name.gsub(/(?<=.)[A-Z]/, "_\\0") - @type = "YP_NODE_#{type.upcase}" - @human = type.downcase - @params = config.fetch("child_nodes", []).map do |param| - param_type = PARAM_TYPES[param.fetch("type")] || - raise("Unknown param type: #{param["type"].inspect}") - param_type.new(**param.transform_keys(&:to_sym)) - end - @newline = config.fetch("newline", true) - @comment = config.fetch("comment") - end + @fields = + config.fetch("fields", []).map do |field| + field_type_for(field.fetch("type")).new(**field.transform_keys(&:to_sym)) + end - # Should emit serialized length of node so implementations can skip - # the node to enable lazy parsing. - def needs_serialized_length? - @name == "DefNode" - end -end + @newline = config.fetch("newline", true) + @comment = config.fetch("comment") + end -# This represents a token in the lexer. They are configured through the -# config.yml file for now, but this will probably change as we transition to -# storing semantic strings instead of the lexer tokens. -class Token - attr_reader :name, :value, :comment + # Should emit serialized length of node so implementations can skip + # the node to enable lazy parsing. + def needs_serialized_length? + name == "DefNode" + end - def initialize(config) - @name = config.fetch("name") - @value = config["value"] - @comment = config.fetch("comment") - end + private - def declaration - output = [] - output << "YP_TOKEN_#{name}" - output << " = #{value}" if value - output << ", // #{comment}" - output.join + def field_type_for(name) + case name + when "node" then NodeField + when "node?" then OptionalNodeField + when "node[]" then NodeListField + when "string" then StringField + when "location[]" then LocationListField + when "constant" then ConstantField + when "constant[]" then ConstantListField + when "location" then LocationField + when "location?" then OptionalLocationField + when "uint32" then UInt32Field + when "flags" then FlagsField + else raise("Unknown field type: #{name.inspect}") + end + end end -end -# Represents a set of flags that should be internally represented with an enum. -class Flags - attr_reader :name, :human, :values + # This represents a token in the lexer. + class Token + attr_reader :name, :value, :comment - def initialize(config) - @name = config.fetch("name") - @human = @name.gsub(/(?<=.)[A-Z]/, "_\\0").downcase - @values = config.fetch("values").map { |flag| Flag.new(flag) } + def initialize(config) + @name = config.fetch("name") + @value = config["value"] + @comment = config.fetch("comment") + end + + def declaration + output = [] + output << "YP_TOKEN_#{name}" + output << " = #{value}" if value + output << ", // #{comment}" + output.join + end end -end -class Flag - attr_reader :name, :camelcase, :comment + # Represents a set of flags that should be internally represented with an enum. + class Flags + # Represents an individual flag within a set of flags. + class Flag + attr_reader :name, :camelcase, :comment + + def initialize(config) + @name = config.fetch("name") + @camelcase = @name.split("_").map(&:capitalize).join + @comment = config.fetch("comment") + end + end - def initialize(config) - @name = config.fetch("name") - @camelcase = @name.split("_").map(&:capitalize).join - @comment = config.fetch("comment") + attr_reader :name, :human, :values + + def initialize(config) + @name = config.fetch("name") + @human = @name.gsub(/(?<=.)[A-Z]/, "_\\0").downcase + @values = config.fetch("values").map { |flag| Flag.new(flag) } + end end -end -module YARP class << self # This templates out a file using ERB with the given locals. The locals are # derived from the config.yml file. From c0cb9efb16993b0898979844c54207f979b2bd44 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 16:12:50 -0400 Subject: [PATCH 155/208] [ruby/yarp] Fix up Rust config.yml reading https://github.com/ruby/yarp/commit/087f367847 --- yarp/config.yml | 4 ++-- yarp/templates/template.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/yarp/config.yml b/yarp/config.yml index b1e21edf9ef2e3..32a605c61d9050 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -773,7 +773,7 @@ nodes: @@target ||= value ^^^^^^^^^^^^^^^^^^ - name: ClassVariableReadNode - child_nodes: + fields: - name: name type: constant comment: | @@ -782,7 +782,7 @@ nodes: @@foo ^^^^^ - name: ClassVariableTargetNode - child_nodes: + fields: - name: name type: constant comment: | diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index c72fdbc2c9fd7d..ec731c3e2fa0c8 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -241,7 +241,7 @@ class Flags # Represents an individual flag within a set of flags. class Flag attr_reader :name, :camelcase, :comment - + def initialize(config) @name = config.fetch("name") @camelcase = @name.split("_").map(&:capitalize).join From c46858fd6aa360e2165fa415f05b1112531128b8 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 29 Aug 2023 16:18:14 -0400 Subject: [PATCH 156/208] [ruby/yarp] Nest all of the YARP tests under the YARP namespace https://github.com/ruby/yarp/commit/440cf93a70 --- test/yarp/bom_test.rb | 96 +- test/yarp/comments_test.rb | 96 +- test/yarp/desugar_visitor_test.rb | 98 +- test/yarp/encoding_test.rb | 166 +-- test/yarp/errors_test.rb | 2088 ++++++++++++++--------------- test/yarp/heredoc_dedent_test.rb | 2 +- test/yarp/library_symbols_test.rb | 6 +- test/yarp/locals_test.rb | 178 +-- test/yarp/location_test.rb | 2 +- test/yarp/memsize_test.rb | 14 +- test/yarp/newline_test.rb | 138 +- test/yarp/parse_serialize_test.rb | 28 +- test/yarp/parse_test.rb | 324 ++--- test/yarp/regexp_test.rb | 338 ++--- test/yarp/ripper_compat_test.rb | 2 +- test/yarp/ruby_api_test.rb | 88 +- test/yarp/test_helper.rb | 30 +- test/yarp/unescape_test.rb | 14 +- test/yarp/version_test.rb | 8 +- 19 files changed, 1869 insertions(+), 1847 deletions(-) diff --git a/test/yarp/bom_test.rb b/test/yarp/bom_test.rb index b386a5d9a33b20..3a4e04a900afd2 100644 --- a/test/yarp/bom_test.rb +++ b/test/yarp/bom_test.rb @@ -6,52 +6,54 @@ require_relative "test_helper" -class BOMTest < Test::Unit::TestCase - def test_ident - assert_bom("foo") - end - - def test_back_reference - assert_bom("$+") - end - - def test_instance_variable - assert_bom("@foo") - end - - def test_class_variable - assert_bom("@@foo") - end - - def test_global_variable - assert_bom("$foo") - end - - def test_numbered_reference - assert_bom("$1") - end - - def test_percents - assert_bom("%i[]") - assert_bom("%r[]") - assert_bom("%s[]") - assert_bom("%q{}") - assert_bom("%w[]") - assert_bom("%x[]") - assert_bom("%I[]") - assert_bom("%W[]") - assert_bom("%Q{}") - end - - def test_string - assert_bom("\"\"") - assert_bom("''") - end - - private - - def assert_bom(source) - bommed = "\xEF\xBB\xBF#{source}" - assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value +module YARP + class BOMTest < TestCase + def test_ident + assert_bom("foo") + end + + def test_back_reference + assert_bom("$+") + end + + def test_instance_variable + assert_bom("@foo") + end + + def test_class_variable + assert_bom("@@foo") + end + + def test_global_variable + assert_bom("$foo") + end + + def test_numbered_reference + assert_bom("$1") + end + + def test_percents + assert_bom("%i[]") + assert_bom("%r[]") + assert_bom("%s[]") + assert_bom("%q{}") + assert_bom("%w[]") + assert_bom("%x[]") + assert_bom("%I[]") + assert_bom("%W[]") + assert_bom("%Q{}") + end + + def test_string + assert_bom("\"\"") + assert_bom("''") + end + + private + + def assert_bom(source) + bommed = "\xEF\xBB\xBF#{source}" + assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value + end end end diff --git a/test/yarp/comments_test.rb b/test/yarp/comments_test.rb index 13bf819a4eb75d..ac91eab4ac5188 100644 --- a/test/yarp/comments_test.rb +++ b/test/yarp/comments_test.rb @@ -2,67 +2,67 @@ require_relative "test_helper" -class CommentsTest < Test::Unit::TestCase - include ::YARP::DSL +module YARP + class CommentsTest < TestCase + def test_comment_inline + source = "# comment" - def test_comment_inline - source = "# comment" + assert_comment source, :inline, 0..9 + assert_equal [0], Debug.newlines(source) + end - assert_comment source, :inline, 0..9 - assert_equal [0], YARP.const_get(:Debug).newlines(source) - end + def test_comment_inline_def + source = <<~RUBY + def foo + # a comment + end + RUBY - def test_comment_inline_def - source = <<~RUBY - def foo - # a comment + assert_comment source, :inline, 10..22 end - RUBY - - assert_comment source, :inline, 10..22 - end - def test_comment___END__ - source = <<~RUBY - __END__ - comment - RUBY + def test_comment___END__ + source = <<~RUBY + __END__ + comment + RUBY - assert_comment source, :__END__, 0..16 - end + assert_comment source, :__END__, 0..16 + end - def test_comment___END__crlf - source = "__END__\r\ncomment\r\n" + def test_comment___END__crlf + source = "__END__\r\ncomment\r\n" - assert_comment source, :__END__, 0..18 - end + assert_comment source, :__END__, 0..18 + end - def test_comment_embedded_document - source = <<~RUBY - =begin - comment - =end - RUBY + def test_comment_embedded_document + source = <<~RUBY + =begin + comment + =end + RUBY - assert_comment source, :embdoc, 0..20 - end + assert_comment source, :embdoc, 0..20 + end - def test_comment_embedded_document_with_content_on_same_line - source = <<~RUBY - =begin other stuff - =end - RUBY + def test_comment_embedded_document_with_content_on_same_line + source = <<~RUBY + =begin other stuff + =end + RUBY - assert_comment source, :embdoc, 0..24 - end + assert_comment source, :embdoc, 0..24 + end - private + private - def assert_comment(source, type, location) - result = YARP.parse(source) - assert result.errors.empty?, result.errors.map(&:message).join("\n") - assert_equal result.comments.first.type, type - assert_equal result.comments.first.location.start_offset, location.begin - assert_equal result.comments.first.location.end_offset, location.end + def assert_comment(source, type, location) + result = YARP.parse(source) + assert result.errors.empty?, result.errors.map(&:message).join("\n") + assert_equal result.comments.first.type, type + assert_equal result.comments.first.location.start_offset, location.begin + assert_equal result.comments.first.location.end_offset, location.end + end end end diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index de635966a1dfb3..4dee182ef1baa9 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -2,56 +2,58 @@ require_relative "test_helper" -class DesugarVisitorTest < Test::Unit::TestCase - def test_and_write - assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") - assert_desugars("(AndNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar &&= baz") - assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") - assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") - assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") - assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") - assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") - end - - def test_or_write - assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") - assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") - assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") - assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") - assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") - assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") - assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") - end - - def test_operator_write - assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") - assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") - assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") - assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") - assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") - assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") - assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") - end - - private - - def ast_inspect(node) - parts = [node.class.name.split("::").last] - - node.deconstruct_keys(nil).each do |_, value| - case value - when YARP::Node - parts << ast_inspect(value) - when Array - parts.concat(value.map { |element| ast_inspect(element) }) - end +module YARP + class DesugarVisitorTest < TestCase + def test_and_write + assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") + assert_desugars("(AndNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar &&= baz") + assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") + assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") + assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") + assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") + end + + def test_or_write + assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") + assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") + assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") + assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") + assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") + assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") end + + def test_operator_write + assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") + assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") + assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") + assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") + assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") + assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") + end + + private + + def ast_inspect(node) + parts = [node.class.name.split("::").last] + + node.deconstruct_keys(nil).each do |_, value| + case value + when Node + parts << ast_inspect(value) + when Array + parts.concat(value.map { |element| ast_inspect(element) }) + end + end - "(#{parts.join(" ")})" - end + "(#{parts.join(" ")})" + end - def assert_desugars(expected, source) - ast = YARP.parse(source).value.accept(YARP::DesugarVisitor.new) - assert_equal expected, ast_inspect(ast.statements.body.last) + def assert_desugars(expected, source) + ast = YARP.parse(source).value.accept(DesugarVisitor.new) + assert_equal expected, ast_inspect(ast.statements.body.last) + end end end diff --git a/test/yarp/encoding_test.rb b/test/yarp/encoding_test.rb index 2ee084cd3803ca..828b45be7302c3 100644 --- a/test/yarp/encoding_test.rb +++ b/test/yarp/encoding_test.rb @@ -2,97 +2,99 @@ require_relative "test_helper" -class EncodingTest < Test::Unit::TestCase - %w[ - ascii - ascii-8bit - big5 - binary - euc-jp - gbk - iso-8859-1 - iso-8859-2 - iso-8859-3 - iso-8859-4 - iso-8859-5 - iso-8859-6 - iso-8859-7 - iso-8859-8 - iso-8859-9 - iso-8859-10 - iso-8859-11 - iso-8859-13 - iso-8859-14 - iso-8859-15 - iso-8859-16 - koi8-r - shift_jis - sjis - us-ascii - utf-8 - utf8-mac - windows-31j - windows-1251 - windows-1252 - CP1251 - CP1252 - ].each do |encoding| - define_method "test_encoding_#{encoding}" do - result = YARP.parse("# encoding: #{encoding}\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find(encoding), actual +module YARP + class EncodingTest < TestCase + %w[ + ascii + ascii-8bit + big5 + binary + euc-jp + gbk + iso-8859-1 + iso-8859-2 + iso-8859-3 + iso-8859-4 + iso-8859-5 + iso-8859-6 + iso-8859-7 + iso-8859-8 + iso-8859-9 + iso-8859-10 + iso-8859-11 + iso-8859-13 + iso-8859-14 + iso-8859-15 + iso-8859-16 + koi8-r + shift_jis + sjis + us-ascii + utf-8 + utf8-mac + windows-31j + windows-1251 + windows-1252 + CP1251 + CP1252 + ].each do |encoding| + define_method "test_encoding_#{encoding}" do + result = YARP.parse("# encoding: #{encoding}\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find(encoding), actual + end end - end - def test_coding - result = YARP.parse("# coding: utf-8\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual - end + def test_coding + result = YARP.parse("# coding: utf-8\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end - def test_coding_with_whitespace - result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("ascii-8bit"), actual - end + def test_coding_with_whitespace + result = YARP.parse("# coding \t \r \v : \t \v \r ascii-8bit \nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("ascii-8bit"), actual + end - def test_emacs_style - result = YARP.parse("# -*- coding: utf-8 -*-\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual - end + def test_emacs_style + result = YARP.parse("# -*- coding: utf-8 -*-\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end - # This test may be a little confusing. Basically when we use our strpbrk, it - # takes into account the encoding of the file. - def test_strpbrk_multibyte - result = YARP.parse(<<~RUBY) - # encoding: Shift_JIS - %w[\x81\x5c] - RUBY + # This test may be a little confusing. Basically when we use our strpbrk, it + # takes into account the encoding of the file. + def test_strpbrk_multibyte + result = YARP.parse(<<~RUBY) + # encoding: Shift_JIS + %w[\x81\x5c] + RUBY - assert(result.errors.empty?) - assert_equal( - (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), - result.value.statements.body.first.elements.first.unescaped - ) - end + assert(result.errors.empty?) + assert_equal( + (+"\x81\x5c").force_encoding(Encoding::Shift_JIS), + result.value.statements.body.first.elements.first.unescaped + ) + end - def test_utf_8_variations - %w[ - utf-8-unix - utf-8-dos - utf-8-mac - utf-8-* - ].each do |encoding| - result = YARP.parse("# coding: #{encoding}\nident") - actual = result.value.statements.body.first.name.encoding - assert_equal Encoding.find("utf-8"), actual + def test_utf_8_variations + %w[ + utf-8-unix + utf-8-dos + utf-8-mac + utf-8-* + ].each do |encoding| + result = YARP.parse("# coding: #{encoding}\nident") + actual = result.value.statements.body.first.name.encoding + assert_equal Encoding.find("utf-8"), actual + end end - end - def test_first_lexed_token - encoding = YARP.lex("# encoding: ascii-8bit").value[0][0].value.encoding - assert_equal Encoding.find("ascii-8bit"), encoding + def test_first_lexed_token + encoding = YARP.lex("# encoding: ascii-8bit").value[0][0].value.encoding + assert_equal Encoding.find("ascii-8bit"), encoding + end end end diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index e461afbbd26fc1..0e1931ec6ac68d 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -2,1136 +2,1136 @@ require_relative "test_helper" -class ErrorsTest < Test::Unit::TestCase - include ::YARP::DSL - - def test_constant_path_with_invalid_token_after - assert_error_messages "A::$b", [ - "Expected identifier or constant after '::'", - "Expected a newline or semicolon after statement." - ] - end - - def test_module_name_recoverable - expected = ModuleNode( - [], - Location(), - ConstantReadNode(), - StatementsNode( - [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] - ), - Location(), - "Parent" - ) - - assert_errors expected, "module Parent module end", [ - ["Expected to find a module name after `module`.", 20..20] - ] - end - - def test_for_loops_index_missing - expected = ForNode( - MissingNode(), - expression("1..10"), - StatementsNode([expression("i")]), - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "for in 1..10\ni\nend", [ - ["Expected index after for.", 0..0] - ] - end - - def test_for_loops_only_end - expected = ForNode( - MissingNode(), - MissingNode(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "for end", [ - ["Expected index after for.", 0..0], - ["Expected keyword in.", 3..3], - ["Expected collection.", 3..3] - ] - end - - def test_pre_execution_missing_brace - expected = PreExecutionNode( - StatementsNode([expression("1")]), - Location(), - Location(), - Location() - ) - - assert_errors expected, "BEGIN 1 }", [ - ["Expected '{' after 'BEGIN'.", 5..5] - ] - end +module YARP + class ErrorsTest < TestCase + include DSL + + def test_constant_path_with_invalid_token_after + assert_error_messages "A::$b", [ + "Expected identifier or constant after '::'", + "Expected a newline or semicolon after statement." + ] + end - def test_pre_execution_context - expected = PreExecutionNode( - StatementsNode([ - CallNode( - expression("1"), - nil, - Location(), - nil, - ArgumentsNode([MissingNode()]), - nil, - nil, - 0, - "+" - ) - ]), - Location(), - Location(), - Location() - ) - - assert_errors expected, "BEGIN { 1 + }", [ - ["Expected a value after the operator.", 11..11] - ] - end + def test_module_name_recoverable + expected = ModuleNode( + [], + Location(), + ConstantReadNode(), + StatementsNode( + [ModuleNode([], Location(), MissingNode(), nil, Location(), "")] + ), + Location(), + "Parent" + ) - def test_unterminated_embdoc - assert_errors expression("1"), "1\n=begin\n", [ - ["Unterminated embdoc", 2..9] - ] - end + assert_errors expected, "module Parent module end", [ + ["Expected to find a module name after `module`.", 20..20] + ] + end - def test_unterminated_i_list - assert_errors expression("%i["), "%i[", [ - ["Expected a closing delimiter for a `%i` list.", 3..3] - ] - end + def test_for_loops_index_missing + expected = ForNode( + MissingNode(), + expression("1..10"), + StatementsNode([expression("i")]), + Location(), + Location(), + nil, + Location() + ) - def test_unterminated_w_list - assert_errors expression("%w["), "%w[", [ - ["Expected a closing delimiter for a `%w` list.", 3..3] - ] - end + assert_errors expected, "for in 1..10\ni\nend", [ + ["Expected index after for.", 0..0] + ] + end - def test_unterminated_W_list - assert_errors expression("%W["), "%W[", [ - ["Expected a closing delimiter for a `%W` list.", 3..3] - ] - end + def test_for_loops_only_end + expected = ForNode( + MissingNode(), + MissingNode(), + nil, + Location(), + Location(), + nil, + Location() + ) - def test_unterminated_regular_expression - assert_errors expression("/hello"), "/hello", [ - ["Expected a closing delimiter for a regular expression.", 1..1] - ] - end + assert_errors expected, "for end", [ + ["Expected index after for.", 0..0], + ["Expected keyword in.", 3..3], + ["Expected collection.", 3..3] + ] + end - def test_unterminated_regular_expression_with_heredoc - source = "<<-END + /b\nEND\n" + def test_pre_execution_missing_brace + expected = PreExecutionNode( + StatementsNode([expression("1")]), + Location(), + Location(), + Location() + ) - assert_errors expression(source), source, [ - ["Expected a closing delimiter for a regular expression.", 10..10] - ] - end + assert_errors expected, "BEGIN 1 }", [ + ["Expected '{' after 'BEGIN'.", 5..5] + ] + end - def test_unterminated_xstring - assert_errors expression("`hello"), "`hello", [ - ["Expected a closing delimiter for an xstring.", 1..1] - ] - end + def test_pre_execution_context + expected = PreExecutionNode( + StatementsNode([ + CallNode( + expression("1"), + nil, + Location(), + nil, + ArgumentsNode([MissingNode()]), + nil, + nil, + 0, + "+" + ) + ]), + Location(), + Location(), + Location() + ) - def test_unterminated_string - assert_errors expression('"hello'), '"hello', [ - ["Expected a closing delimiter for an interpolated string.", 1..1] - ] - end + assert_errors expected, "BEGIN { 1 + }", [ + ["Expected a value after the operator.", 11..11] + ] + end - def test_unterminated_s_symbol - assert_errors expression("%s[abc"), "%s[abc", [ - ["Expected a closing delimiter for a dynamic symbol.", 3..3] - ] - end + def test_unterminated_embdoc + assert_errors expression("1"), "1\n=begin\n", [ + ["Unterminated embdoc", 2..9] + ] + end - def test_unterminated_parenthesized_expression - assert_errors expression('(1 + 2'), '(1 + 2', [ - ["Expected to be able to parse an expression.", 6..6], - ["Expected a closing parenthesis.", 6..6] - ] - end + def test_unterminated_i_list + assert_errors expression("%i["), "%i[", [ + ["Expected a closing delimiter for a `%i` list.", 3..3] + ] + end - def test_unterminated_argument_expression - assert_errors expression('a %'), 'a %', [ - ["Unexpected end of input", 2..3], - ["Expected a value after the operator.", 3..3], - ] - end + def test_unterminated_w_list + assert_errors expression("%w["), "%w[", [ + ["Expected a closing delimiter for a `%w` list.", 3..3] + ] + end - def test_cr_without_lf_in_percent_expression - assert_errors expression("%\r"), "%\r", [ - ["Invalid %% token", 0..2], - ] - end + def test_unterminated_W_list + assert_errors expression("%W["), "%W[", [ + ["Expected a closing delimiter for a `%W` list.", 3..3] + ] + end - def test_1_2_3 - assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 2..2], - ["Expected a closing parenthesis.", 2..2], - ["Expected a newline or semicolon after statement.", 2..2], - ["Expected to be able to parse an expression.", 2..2], - ["Expected a newline or semicolon after statement.", 5..5], - ["Expected to be able to parse an expression.", 5..5], - ["Expected a newline or semicolon after statement.", 8..8], - ["Expected to be able to parse an expression.", 8..8], - ] - end + def test_unterminated_regular_expression + assert_errors expression("/hello"), "/hello", [ + ["Expected a closing delimiter for a regular expression.", 1..1] + ] + end - def test_return_1_2_3 - assert_error_messages "return(1, 2, 3)", [ - "Expected to be able to parse an expression.", - "Expected a closing parenthesis.", - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression." - ] - end + def test_unterminated_regular_expression_with_heredoc + source = "<<-END + /b\nEND\n" - def test_return_1 - assert_errors expression("return 1,;"), "return 1,;", [ - ["Expected to be able to parse an argument.", 9..9] - ] - end + assert_errors expression(source), source, [ + ["Expected a closing delimiter for a regular expression.", 10..10] + ] + end - def test_next_1_2_3 - assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 6..6], - ["Expected a closing parenthesis.", 6..6], - ["Expected a newline or semicolon after statement.", 12..12], - ["Expected to be able to parse an expression.", 12..12] - ] - end + def test_unterminated_xstring + assert_errors expression("`hello"), "`hello", [ + ["Expected a closing delimiter for an xstring.", 1..1] + ] + end - def test_next_1 - assert_errors expression("next 1,;"), "next 1,;", [ - ["Expected to be able to parse an argument.", 7..7] - ] - end + def test_unterminated_string + assert_errors expression('"hello'), '"hello', [ + ["Expected a closing delimiter for an interpolated string.", 1..1] + ] + end - def test_break_1_2_3 - assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ - ["Expected to be able to parse an expression.", 7..7], - ["Expected a closing parenthesis.", 7..7], - ["Expected a newline or semicolon after statement.", 13..13], - ["Expected to be able to parse an expression.", 13..13], - ] - end + def test_unterminated_s_symbol + assert_errors expression("%s[abc"), "%s[abc", [ + ["Expected a closing delimiter for a dynamic symbol.", 3..3] + ] + end - def test_break_1 - assert_errors expression("break 1,;"), "break 1,;", [ - ["Expected to be able to parse an argument.", 8..8] - ] - end + def test_unterminated_parenthesized_expression + assert_errors expression('(1 + 2'), '(1 + 2', [ + ["Expected to be able to parse an expression.", 6..6], + ["Expected a closing parenthesis.", 6..6] + ] + end - def test_argument_forwarding_when_parent_is_not_forwarding - assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ - ["unexpected ... when parent method is not forwarding.", 18..21] - ] - end + def test_unterminated_argument_expression + assert_errors expression('a %'), 'a %', [ + ["Unexpected end of input", 2..3], + ["Expected a value after the operator.", 3..3], + ] + end - def test_argument_forwarding_only_effects_its_own_internals - assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), - 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ - ["unexpected ... when parent method is not forwarding.", 43..46] + def test_cr_without_lf_in_percent_expression + assert_errors expression("%\r"), "%\r", [ + ["Invalid %% token", 0..2], ] - end + end - def test_top_level_constant_with_downcased_identifier - assert_error_messages "::foo", [ - "Expected a constant after ::.", - "Expected a newline or semicolon after statement." - ] - end + def test_1_2_3 + assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 2..2], + ["Expected a closing parenthesis.", 2..2], + ["Expected a newline or semicolon after statement.", 2..2], + ["Expected to be able to parse an expression.", 2..2], + ["Expected a newline or semicolon after statement.", 5..5], + ["Expected to be able to parse an expression.", 5..5], + ["Expected a newline or semicolon after statement.", 8..8], + ["Expected to be able to parse an expression.", 8..8], + ] + end - def test_top_level_constant_starting_with_downcased_identifier - assert_error_messages "::foo::A", [ - "Expected a constant after ::.", - "Expected a newline or semicolon after statement." - ] - end + def test_return_1_2_3 + assert_error_messages "return(1, 2, 3)", [ + "Expected to be able to parse an expression.", + "Expected a closing parenthesis.", + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression." + ] + end - def test_aliasing_global_variable_with_non_global_variable - assert_errors expression("alias $a b"), "alias $a b", [ - ["Expected a global variable.", 9..10] - ] - end + def test_return_1 + assert_errors expression("return 1,;"), "return 1,;", [ + ["Expected to be able to parse an argument.", 9..9] + ] + end - def test_aliasing_non_global_variable_with_global_variable - assert_errors expression("alias a $b"), "alias a $b", [ - ["Expected a bare word or symbol argument.", 8..10] - ] - end + def test_next_1_2_3 + assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 6..6], + ["Expected a closing parenthesis.", 6..6], + ["Expected a newline or semicolon after statement.", 12..12], + ["Expected to be able to parse an expression.", 12..12] + ] + end - def test_aliasing_global_variable_with_global_number_variable - assert_errors expression("alias $a $1"), "alias $a $1", [ - ["Can't make alias for number variables.", 9..11] - ] - end + def test_next_1 + assert_errors expression("next 1,;"), "next 1,;", [ + ["Expected to be able to parse an argument.", 7..7] + ] + end - def test_def_with_expression_receiver_and_no_identifier - assert_errors expression("def (a); end"), "def (a); end", [ - ["Expected '.' or '::' after receiver", 7..7] - ] - end + def test_break_1_2_3 + assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ + ["Expected to be able to parse an expression.", 7..7], + ["Expected a closing parenthesis.", 7..7], + ["Expected a newline or semicolon after statement.", 13..13], + ["Expected to be able to parse an expression.", 13..13], + ] + end - def test_def_with_multiple_statements_receiver - assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ - ["Expected closing ')' for receiver.", 7..7], - ["Expected '.' or '::' after receiver", 7..7], - ["Expected to be able to parse an expression.", 10..10], - ["Expected to be able to parse an expression.", 11..11] - ] - end + def test_break_1 + assert_errors expression("break 1,;"), "break 1,;", [ + ["Expected to be able to parse an argument.", 8..8] + ] + end - def test_def_with_empty_expression_receiver - assert_errors expression("def ().a; end"), "def ().a; end", [ - ["Expected to be able to parse receiver.", 5..5] - ] - end + def test_argument_forwarding_when_parent_is_not_forwarding + assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ + ["unexpected ... when parent method is not forwarding.", 18..21] + ] + end - def test_block_beginning_with_brace_and_ending_with_end - assert_error_messages "x.each { x end", [ - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression.", - "Expected to be able to parse an expression.", - "Expected block beginning with '{' to end with '}'." - ] - end + def test_argument_forwarding_only_effects_its_own_internals + assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), + 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ + ["unexpected ... when parent method is not forwarding.", 43..46] + ] + end - def test_double_splat_followed_by_splat_argument - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - KeywordHashNode([AssocSplatNode(expression("kwargs"), Location())]), - SplatNode(Location(), expression("args")) - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(**kwargs, *args)", [ - ["Unexpected splat argument after double splat.", 12..17] - ] - end + def test_top_level_constant_with_downcased_identifier + assert_error_messages "::foo", [ + "Expected a constant after ::.", + "Expected a newline or semicolon after statement." + ] + end - def test_arguments_after_block - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - BlockArgumentNode(expression("block"), Location()), - expression("foo") - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(&block, foo)", [ - ["Unexpected argument after block argument.", 10..13] - ] - end + def test_top_level_constant_starting_with_downcased_identifier + assert_error_messages "::foo::A", [ + "Expected a constant after ::.", + "Expected a newline or semicolon after statement." + ] + end - def test_arguments_binding_power_for_and - assert_error_messages "foo(*bar and baz)", [ - "Expected a ')' to close the argument list.", - "Expected a newline or semicolon after statement.", - "Expected to be able to parse an expression." - ] - end + def test_aliasing_global_variable_with_non_global_variable + assert_errors expression("alias $a b"), "alias $a b", [ + ["Expected a global variable.", 9..10] + ] + end - def test_splat_argument_after_keyword_argument - expected = CallNode( - nil, - nil, - Location(), - Location(), - ArgumentsNode([ - KeywordHashNode( - [AssocNode( - SymbolNode(nil, Location(), Location(), "foo"), - expression("bar"), - nil - )] - ), - SplatNode(Location(), expression("args")) - ]), - Location(), - nil, - 0, - "a" - ) - - assert_errors expected, "a(foo: bar, *args)", [ - ["Unexpected splat argument after double splat.", 12..17] - ] - end + def test_aliasing_non_global_variable_with_global_variable + assert_errors expression("alias a $b"), "alias a $b", [ + ["Expected a bare word or symbol argument.", 8..10] + ] + end - def test_module_definition_in_method_body - expected = DefNode( - Location(), - nil, - nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "def foo;module A;end;end", [ - ["Module definition in method body", 8..14] - ] - end + def test_aliasing_global_variable_with_global_number_variable + assert_errors expression("alias $a $1"), "alias $a $1", [ + ["Can't make alias for number variables.", 9..11] + ] + end - def test_module_definition_in_method_body_within_block - expected = DefNode( - Location(), - nil, - nil, - StatementsNode( - [CallNode( - nil, - nil, - Location(), - nil, - nil, - nil, - BlockNode( - [], - nil, - StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), - Location(), - Location() - ), - 0, - "bar" - )] - ), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, " - def foo - bar do - module Foo;end - end - end - ", [ - ["Module definition in method body", 40..46] - ] - end + def test_def_with_expression_receiver_and_no_identifier + assert_errors expression("def (a); end"), "def (a); end", [ + ["Expected '.' or '::' after receiver", 7..7] + ] + end - def test_class_definition_in_method_body - expected = DefNode( - Location(), - nil, - nil, - StatementsNode( - [ClassNode( - [], - Location(), - ConstantReadNode(), - nil, - nil, - nil, - Location(), - "A" - )] - ), - [], - Location(), - nil, - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "def foo;class A;end;end", [ - ["Class definition in method body", 8..13] - ] - end + def test_def_with_multiple_statements_receiver + assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ + ["Expected closing ')' for receiver.", 7..7], + ["Expected '.' or '::' after receiver", 7..7], + ["Expected to be able to parse an expression.", 10..10], + ["Expected to be able to parse an expression.", 11..11] + ] + end - def test_bad_arguments - expected = DefNode( - Location(), - nil, - ParametersNode([ - RequiredParameterNode(:A), - RequiredParameterNode(:@a), - RequiredParameterNode(:$A), - RequiredParameterNode(:@@a), - ], [], [], nil, [], nil, nil), - nil, - [:A, :@a, :$A, :@@a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(A, @a, $A, @@a);end", [ - ["Formal argument cannot be a constant", 8..9], - ["Formal argument cannot be an instance variable", 11..13], - ["Formal argument cannot be a global variable", 15..17], - ["Formal argument cannot be a class variable", 19..22], - ] - end + def test_def_with_empty_expression_receiver + assert_errors expression("def ().a; end"), "def ().a; end", [ + ["Expected to be able to parse receiver.", 5..5] + ] + end - def test_cannot_assign_to_a_reserved_numbered_parameter - expected = BeginNode( - Location(), - StatementsNode([ - LocalVariableWriteNode(:_1, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_2, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_3, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_4, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_5, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_6, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_7, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_8, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_9, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_10, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()) - ]), - nil, - nil, - nil, - Location() - ) - source = <<~RUBY - begin - _1=:a;_2=:a;_3=:a;_4=:a;_5=:a - _6=:a;_7=:a;_8=:a;_9=:a;_10=:a - end - RUBY - assert_errors expected, source, [ - ["reserved for numbered parameter", 8..10], - ["reserved for numbered parameter", 14..16], - ["reserved for numbered parameter", 20..22], - ["reserved for numbered parameter", 26..28], - ["reserved for numbered parameter", 32..34], - ["reserved for numbered parameter", 40..42], - ["reserved for numbered parameter", 46..48], - ["reserved for numbered parameter", 52..54], - ["reserved for numbered parameter", 58..60], - ] - end + def test_block_beginning_with_brace_and_ending_with_end + assert_error_messages "x.each { x end", [ + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression.", + "Expected to be able to parse an expression.", + "Expected block beginning with '{' to end with '}'." + ] + end - def test_do_not_allow_trailing_commas_in_method_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:c)], - [], - [], + def test_double_splat_followed_by_splat_argument + expected = CallNode( nil, - [], nil, - nil - ), - nil, - [:a, :b, :c], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,c,);end", [ - ["Unexpected ','.", 13..14] - ] - end - - def test_do_not_allow_trailing_commas_in_lambda_parameters - expected = LambdaNode( - [:a, :b], - Location(), - Location(), - Location(), - BlockParametersNode( - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), - [], Location(), - Location() - ), - nil - ) - assert_errors expected, "-> (a, b, ) {}", [ - ["Unexpected ','.", 8..9] - ] - end - - def test_do_not_allow_multiple_codepoints_in_a_single_character_literal - expected = StringNode(Location(), Location(), nil, "\u0001\u0002") - - assert_errors expected, '?\u{0001 0002}', [ - ["Multiple codepoints at single character literal", 9..12] - ] - end - - def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation - expected = StringNode(Location(), Location(), Location(), "\u0001") - - assert_errors expected, '"\u{0000001}"', [ - ["invalid Unicode escape.", 4..11], - ["invalid Unicode escape.", 4..11] - ] - end - - def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_character_notation - expected = StringNode(Location(), Location(), Location(), "\u0000z}") + Location(), + ArgumentsNode([ + KeywordHashNode([AssocSplatNode(expression("kwargs"), Location())]), + SplatNode(Location(), expression("args")) + ]), + Location(), + nil, + 0, + "a" + ) - assert_errors expected, '"\u{000z}"', [ - ["unterminated Unicode escape", 7..7], - ["unterminated Unicode escape", 7..7] - ] - end + assert_errors expected, "a(**kwargs, *args)", [ + ["Unexpected splat argument after double splat.", 12..17] + ] + end - def test_method_parameters_after_block - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_arguments_after_block + expected = CallNode( nil, - [], nil, - BlockParameterNode(Location(), Location()) - ), - nil, - [:block, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(&block, a)\nend", [ - ["Unexpected parameter order", 16..17] - ] - end + Location(), + Location(), + ArgumentsNode([ + BlockArgumentNode(expression("block"), Location()), + expression("foo") + ]), + Location(), + nil, + 0, + "a" + ) - def test_method_with_arguments_after_anonymous_block - expected = DefNode( - Location(), - nil, - ParametersNode([], [], [RequiredParameterNode(:a)], nil, [], nil, BlockParameterNode(nil, Location())), - nil, - [:&, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(&, a)\nend", [ - ["Unexpected parameter order", 11..12] - ] - end + assert_errors expected, "a(&block, foo)", [ + ["Unexpected argument after block argument.", 10..13] + ] + end - def test_method_parameters_after_arguments_forwarding - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], - nil, - [], - ForwardingParameterNode(), - nil - ), - nil, - [:"...", :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(..., a)\nend", [ - ["Unexpected parameter order", 13..14] - ] - end + def test_arguments_binding_power_for_and + assert_error_messages "foo(*bar and baz)", [ + "Expected a ')' to close the argument list.", + "Expected a newline or semicolon after statement.", + "Expected to be able to parse an expression." + ] + end - def test_keywords_parameters_before_required_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_splat_argument_after_keyword_argument + expected = CallNode( nil, - [KeywordParameterNode(Location(), nil)], nil, - nil - ), - nil, - [:b, :a], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(b:, a)\nend", [ - ["Unexpected parameter order", 12..13] - ] - end - - def test_rest_keywords_parameters_before_required_parameters - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [], + Location(), + Location(), + ArgumentsNode([ + KeywordHashNode( + [AssocNode( + SymbolNode(nil, Location(), Location(), "foo"), + expression("bar"), + nil + )] + ), + SplatNode(Location(), expression("args")) + ]), + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:rest, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - assert_errors expected, "def foo(**rest, b:)\nend", [ - ["Unexpected parameter order", 16..18] - ] - end + 0, + "a" + ) - def test_double_arguments_forwarding - expected = DefNode( - Location(), - nil, - ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), - nil, - [:"..."], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(..., ...)\nend", [ - ["Unexpected parameter order", 13..16] - ] - end + assert_errors expected, "a(foo: bar, *args)", [ + ["Unexpected splat argument after double splat.", 12..17] + ] + end - def test_multiple_error_in_parameters_order - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], + def test_module_definition_in_method_body + expected = DefNode( + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_switching_to_optional_arguments_twice - expected = DefNode( - Location(), - nil, - ParametersNode( - [], - [], - [RequiredParameterNode(:a)], nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_switching_to_named_arguments_twice - expected = DefNode( - Location(), - nil, - ParametersNode( - [], + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "A")]), [], - [RequiredParameterNode(:a)], + Location(), nil, - [KeywordParameterNode(Location(), nil)], - KeywordRestParameterNode(Location(), Location()), - nil - ), - nil, - [:args, :a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] - ] - end - - def test_returning_to_optional_parameters_multiple_times - expected = DefNode( - Location(), - nil, - ParametersNode( - [RequiredParameterNode(:a)], - [ - OptionalParameterNode(:b, Location(), Location(), IntegerNode()), - OptionalParameterNode(:d, Location(), Location(), IntegerNode()) - ], - [RequiredParameterNode(:c), RequiredParameterNode(:e)], nil, - [], nil, - nil - ), - nil, - [:a, :b, :c, :d, :e], - Location(), - nil, - Location(), - Location(), - nil, - Location(), - ) - - assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ - ["Unexpected parameter order", 23..24] - ] - end - - def test_case_without_when_clauses_errors_on_else_clause - expected = CaseNode( - SymbolNode(Location(), Location(), nil, "a"), - [], - ElseNode(Location(), nil, Location()), - Location(), - Location() - ) - - assert_errors expected, "case :a\nelse\nend", [ - ["Unexpected else without no when clauses in case statement.", 8..12] - ] - end - - def test_setter_method_cannot_be_defined_in_an_endless_method_definition - expected = DefNode( - Location(), - nil, - nil, - StatementsNode([IntegerNode()]), - [], - Location(), - nil, - Location(), - Location(), - Location(), - nil - ) - - assert_errors expected, "def a=() = 42", [ - ["Setter method cannot be defined in an endless method definition", 4..6] - ] - end - - def test_do_not_allow_forward_arguments_in_lambda_literals - expected = LambdaNode( - [:"..."], - Location(), - Location(), - Location(), - BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), - nil - ) - - assert_errors expected, "->(...) {}", [ - ["Unexpected ...", 3..6] - ] - end - - def test_do_not_allow_forward_arguments_in_blocks - expected = CallNode( - nil, - nil, - Location(), - nil, - nil, - nil, - BlockNode( - [:"..."], - BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), nil, - Location(), Location() - ), - 0, - "a" - ) - - assert_errors expected, "a {|...|}", [ - ["Unexpected ...", 4..7] - ] - end - - def test_dont_allow_return_inside_class_body - expected = ClassNode( - [], - Location(), - ConstantReadNode(), - nil, - nil, - StatementsNode([ReturnNode(Location(), nil)]), - Location(), - "A" - ) - - assert_errors expected, "class A; return; end", [ - ["Invalid return in class/module body", 15..16] - ] - end - - def test_dont_allow_return_inside_module_body - expected = ModuleNode( - [], - Location(), - ConstantReadNode(), - StatementsNode([ReturnNode(Location(), nil)]), - Location(), - "A" - ) - - assert_errors expected, "module A; return; end", [ - ["Invalid return in class/module body", 16..17] - ] - end + ) - def test_dont_allow_setting_to_back_and_nth_reference - expected = BeginNode( - Location(), - StatementsNode([ - GlobalVariableWriteNode(Location(), Location(), NilNode()), - GlobalVariableWriteNode(Location(), Location(), NilNode()) - ]), - nil, - nil, - nil, - Location() - ) - - assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ - ["Can't set variable", 6..8], - ["Can't set variable", 15..20] - ] - end + assert_errors expected, "def foo;module A;end;end", [ + ["Module definition in method body", 8..14] + ] + end - def test_duplicated_parameter_names - # For some reason, Ripper reports no error for Ruby 3.0 when you have - # duplicated parameter names for positional parameters. - unless RUBY_VERSION < "3.1.0" + def test_module_definition_in_method_body_within_block expected = DefNode( Location(), nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:a)], [], [], nil, [], nil, nil), nil, - [:a, :b], + StatementsNode( + [CallNode( + nil, + nil, + Location(), + nil, + nil, + nil, + BlockNode( + [], + nil, + StatementsNode([ModuleNode([], Location(), ConstantReadNode(), nil, Location(), "Foo")]), + Location(), + Location() + ), + 0, + "bar" + )] + ), + [], Location(), nil, + nil, + nil, + nil, + Location() + ) + + assert_errors expected, <<~RUBY, [["Module definition in method body", 21..27]] + def foo + bar do + module Foo;end + end + end + RUBY + end + + def test_class_definition_in_method_body + expected = DefNode( Location(), + nil, + nil, + StatementsNode( + [ClassNode( + [], + Location(), + ConstantReadNode(), + nil, + nil, + nil, + Location(), + "A" + )] + ), + [], Location(), nil, + nil, + nil, + nil, Location() ) - assert_errors expected, "def foo(a,b,a);end", [ - ["Duplicated parameter name.", 12..13] - ] - end - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], RestParameterNode(Location(), Location()), [], nil, nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,*a);end", [ - ["Duplicated parameter name.", 13..14] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], KeywordRestParameterNode(Location(), Location()), nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,**a);end", [ - ["Duplicated parameter name.", 14..15] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, BlockParameterNode(Location(), Location())), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a,b,&a);end", [ - ["Duplicated parameter name.", 13..14] - ] - - expected = DefNode( - Location(), - nil, - ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(Location(), Location()), [], nil, nil), - nil, - [:a, :b, :c], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) - - assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] - end + assert_errors expected, "def foo;class A;end;end", [ + ["Class definition in method body", 8..13] + ] + end - def test_unterminated_global_variable - assert_errors expression("$"), "$", [ - ["Invalid global variable.", 0..1] - ] - end + def test_bad_arguments + expected = DefNode( + Location(), + nil, + ParametersNode([ + RequiredParameterNode(:A), + RequiredParameterNode(:@a), + RequiredParameterNode(:$A), + RequiredParameterNode(:@@a), + ], [], [], nil, [], nil, nil), + nil, + [:A, :@a, :$A, :@@a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - private + assert_errors expected, "def foo(A, @a, $A, @@a);end", [ + ["Formal argument cannot be a constant", 8..9], + ["Formal argument cannot be an instance variable", 11..13], + ["Formal argument cannot be a global variable", 15..17], + ["Formal argument cannot be a class variable", 19..22], + ] + end - def assert_errors(expected, source, errors) - # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby - assert_nil Ripper.sexp_raw(source) if RUBY_ENGINE == "ruby" + def test_cannot_assign_to_a_reserved_numbered_parameter + expected = BeginNode( + Location(), + StatementsNode([ + LocalVariableWriteNode(:_1, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_2, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_3, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_4, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_5, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_6, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_7, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_8, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_9, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), + LocalVariableWriteNode(:_10, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()) + ]), + nil, + nil, + nil, + Location() + ) + source = <<~RUBY + begin + _1=:a;_2=:a;_3=:a;_4=:a;_5=:a + _6=:a;_7=:a;_8=:a;_9=:a;_10=:a + end + RUBY + assert_errors expected, source, [ + ["reserved for numbered parameter", 8..10], + ["reserved for numbered parameter", 14..16], + ["reserved for numbered parameter", 20..22], + ["reserved for numbered parameter", 26..28], + ["reserved for numbered parameter", 32..34], + ["reserved for numbered parameter", 40..42], + ["reserved for numbered parameter", 46..48], + ["reserved for numbered parameter", 52..54], + ["reserved for numbered parameter", 58..60], + ] + end - result = YARP.parse(source) - node = result.value.statements.body.last + def test_do_not_allow_trailing_commas_in_method_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:c)], + [], + [], + nil, + [], + nil, + nil + ), + nil, + [:a, :b, :c], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - assert_equal_nodes(expected, node, compare_location: false) - assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) - end + assert_errors expected, "def foo(a,b,c,);end", [ + ["Unexpected ','.", 13..14] + ] + end - def assert_error_messages(source, errors) - assert_nil Ripper.sexp_raw(source) - result = YARP.parse(source) - assert_equal(errors, result.errors.map(&:message)) - end + def test_do_not_allow_trailing_commas_in_lambda_parameters + expected = LambdaNode( + [:a, :b], + Location(), + Location(), + Location(), + BlockParametersNode( + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, nil), + [], + Location(), + Location() + ), + nil + ) + assert_errors expected, "-> (a, b, ) {}", [ + ["Unexpected ','.", 8..9] + ] + end - def expression(source) - YARP.parse(source).value.statements.body.last + def test_do_not_allow_multiple_codepoints_in_a_single_character_literal + expected = StringNode(Location(), Location(), nil, "\u0001\u0002") + + assert_errors expected, '?\u{0001 0002}', [ + ["Multiple codepoints at single character literal", 9..12] + ] + end + + def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation + expected = StringNode(Location(), Location(), Location(), "\u0001") + + assert_errors expected, '"\u{0000001}"', [ + ["invalid Unicode escape.", 4..11], + ["invalid Unicode escape.", 4..11] + ] + end + + def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_character_notation + expected = StringNode(Location(), Location(), Location(), "\u0000z}") + + assert_errors expected, '"\u{000z}"', [ + ["unterminated Unicode escape", 7..7], + ["unterminated Unicode escape", 7..7] + ] + end + + def test_method_parameters_after_block + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [], + nil, + BlockParameterNode(Location(), Location()) + ), + nil, + [:block, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(&block, a)\nend", [ + ["Unexpected parameter order", 16..17] + ] + end + + def test_method_with_arguments_after_anonymous_block + expected = DefNode( + Location(), + nil, + ParametersNode([], [], [RequiredParameterNode(:a)], nil, [], nil, BlockParameterNode(nil, Location())), + nil, + [:&, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(&, a)\nend", [ + ["Unexpected parameter order", 11..12] + ] + end + + def test_method_parameters_after_arguments_forwarding + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [], + ForwardingParameterNode(), + nil + ), + nil, + [:"...", :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(..., a)\nend", [ + ["Unexpected parameter order", 13..14] + ] + end + + def test_keywords_parameters_before_required_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + nil, + nil + ), + nil, + [:b, :a], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(b:, a)\nend", [ + ["Unexpected parameter order", 12..13] + ] + end + + def test_rest_keywords_parameters_before_required_parameters + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:rest, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + assert_errors expected, "def foo(**rest, b:)\nend", [ + ["Unexpected parameter order", 16..18] + ] + end + + def test_double_arguments_forwarding + expected = DefNode( + Location(), + nil, + ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), + nil, + [:"..."], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(..., ...)\nend", [ + ["Unexpected parameter order", 13..16] + ] + end + + def test_multiple_error_in_parameters_order + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_switching_to_optional_arguments_twice + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_switching_to_named_arguments_twice + expected = DefNode( + Location(), + nil, + ParametersNode( + [], + [], + [RequiredParameterNode(:a)], + nil, + [KeywordParameterNode(Location(), nil)], + KeywordRestParameterNode(Location(), Location()), + nil + ), + nil, + [:args, :a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(**args, a, b:)\nend", [ + ["Unexpected parameter order", 16..17], + ["Unexpected parameter order", 19..21] + ] + end + + def test_returning_to_optional_parameters_multiple_times + expected = DefNode( + Location(), + nil, + ParametersNode( + [RequiredParameterNode(:a)], + [ + OptionalParameterNode(:b, Location(), Location(), IntegerNode()), + OptionalParameterNode(:d, Location(), Location(), IntegerNode()) + ], + [RequiredParameterNode(:c), RequiredParameterNode(:e)], + nil, + [], + nil, + nil + ), + nil, + [:a, :b, :c, :d, :e], + Location(), + nil, + Location(), + Location(), + nil, + Location(), + ) + + assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ + ["Unexpected parameter order", 23..24] + ] + end + + def test_case_without_when_clauses_errors_on_else_clause + expected = CaseNode( + SymbolNode(Location(), Location(), nil, "a"), + [], + ElseNode(Location(), nil, Location()), + Location(), + Location() + ) + + assert_errors expected, "case :a\nelse\nend", [ + ["Unexpected else without no when clauses in case statement.", 8..12] + ] + end + + def test_setter_method_cannot_be_defined_in_an_endless_method_definition + expected = DefNode( + Location(), + nil, + nil, + StatementsNode([IntegerNode()]), + [], + Location(), + nil, + Location(), + Location(), + Location(), + nil + ) + + assert_errors expected, "def a=() = 42", [ + ["Setter method cannot be defined in an endless method definition", 4..6] + ] + end + + def test_do_not_allow_forward_arguments_in_lambda_literals + expected = LambdaNode( + [:"..."], + Location(), + Location(), + Location(), + BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), + nil + ) + + assert_errors expected, "->(...) {}", [ + ["Unexpected ...", 3..6] + ] + end + + def test_do_not_allow_forward_arguments_in_blocks + expected = CallNode( + nil, + nil, + Location(), + nil, + nil, + nil, + BlockNode( + [:"..."], + BlockParametersNode(ParametersNode([], [], [], nil, [], ForwardingParameterNode(), nil), [], Location(), Location()), + nil, + Location(), + Location() + ), + 0, + "a" + ) + + assert_errors expected, "a {|...|}", [ + ["Unexpected ...", 4..7] + ] + end + + def test_dont_allow_return_inside_class_body + expected = ClassNode( + [], + Location(), + ConstantReadNode(), + nil, + nil, + StatementsNode([ReturnNode(Location(), nil)]), + Location(), + "A" + ) + + assert_errors expected, "class A; return; end", [ + ["Invalid return in class/module body", 15..16] + ] + end + + def test_dont_allow_return_inside_module_body + expected = ModuleNode( + [], + Location(), + ConstantReadNode(), + StatementsNode([ReturnNode(Location(), nil)]), + Location(), + "A" + ) + + assert_errors expected, "module A; return; end", [ + ["Invalid return in class/module body", 16..17] + ] + end + + def test_dont_allow_setting_to_back_and_nth_reference + expected = BeginNode( + Location(), + StatementsNode([ + GlobalVariableWriteNode(Location(), Location(), NilNode()), + GlobalVariableWriteNode(Location(), Location(), NilNode()) + ]), + nil, + nil, + nil, + Location() + ) + + assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ + ["Can't set variable", 6..8], + ["Can't set variable", 15..20] + ] + end + + def test_duplicated_parameter_names + # For some reason, Ripper reports no error for Ruby 3.0 when you have + # duplicated parameter names for positional parameters. + unless RUBY_VERSION < "3.1.0" + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b), RequiredParameterNode(:a)], [], [], nil, [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,a);end", [ + ["Duplicated parameter name.", 12..13] + ] + end + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], RestParameterNode(Location(), Location()), [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,*a);end", [ + ["Duplicated parameter name.", 13..14] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], KeywordRestParameterNode(Location(), Location()), nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,**a);end", [ + ["Duplicated parameter name.", 14..15] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([RequiredParameterNode(:a), RequiredParameterNode(:b)], [], [], nil, [], nil, BlockParameterNode(Location(), Location())), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a,b,&a);end", [ + ["Duplicated parameter name.", 13..14] + ] + + expected = DefNode( + Location(), + nil, + ParametersNode([], [OptionalParameterNode(:a, Location(), Location(), IntegerNode())], [RequiredParameterNode(:b)], RestParameterNode(Location(), Location()), [], nil, nil), + nil, + [:a, :b, :c], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) + + assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter *", 16..17]] + end + + def test_unterminated_global_variable + assert_errors expression("$"), "$", [ + ["Invalid global variable.", 0..1] + ] + end + + private + + def assert_errors(expected, source, errors) + # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby + assert_nil Ripper.sexp_raw(source) if RUBY_ENGINE == "ruby" + + result = YARP.parse(source) + node = result.value.statements.body.last + + assert_equal_nodes(expected, node, compare_location: false) + assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) + end + + def assert_error_messages(source, errors) + assert_nil Ripper.sexp_raw(source) + result = YARP.parse(source) + assert_equal(errors, result.errors.map(&:message)) + end + + def expression(source) + YARP.parse(source).value.statements.body.last + end end end diff --git a/test/yarp/heredoc_dedent_test.rb b/test/yarp/heredoc_dedent_test.rb index 2744b930ed3f4c..975232889f488e 100644 --- a/test/yarp/heredoc_dedent_test.rb +++ b/test/yarp/heredoc_dedent_test.rb @@ -3,7 +3,7 @@ require_relative "test_helper" module YARP - class HeredocDedentTest < Test::Unit::TestCase + class HeredocDedentTest < TestCase filepath = File.expand_path("fixtures/tilde_heredocs.txt", __dir__) File.read(filepath).split(/(?=\n)\n(?=<)/).each_with_index do |heredoc, index| diff --git a/test/yarp/library_symbols_test.rb b/test/yarp/library_symbols_test.rb index 9a52d189baff64..53f56d9bfa6c73 100644 --- a/test/yarp/library_symbols_test.rb +++ b/test/yarp/library_symbols_test.rb @@ -2,12 +2,14 @@ require_relative "test_helper" -if RUBY_PLATFORM =~ /linux/ +return if RUBY_PLATFORM !~ /linux/ + +module YARP # # examine a yarp dll or static archive for expected external symbols. # these tests only work on a linux system right now. # - class LibrarySymbolsTest < Test::Unit::TestCase + class LibrarySymbolsTest < TestCase def setup super diff --git a/test/yarp/locals_test.rb b/test/yarp/locals_test.rb index 42fb72df789b88..45aecdcaf7ef2e 100644 --- a/test/yarp/locals_test.rb +++ b/test/yarp/locals_test.rb @@ -15,93 +15,95 @@ require_relative "test_helper" -class LocalsTest < Test::Unit::TestCase - invalid = [] - todos = [] - - # Invalid break - invalid << "break.txt" - invalid << "if.txt" - invalid << "rescue.txt" - invalid << "seattlerb/block_break.txt" - invalid << "unless.txt" - invalid << "whitequark/break.txt" - invalid << "whitequark/break_block.txt" - - # Invalid next - invalid << "next.txt" - invalid << "seattlerb/block_next.txt" - invalid << "unparser/corpus/literal/control.txt" - invalid << "whitequark/next.txt" - invalid << "whitequark/next_block.txt" - - # Invalid redo - invalid << "keywords.txt" - invalid << "whitequark/redo.txt" - - # Invalid retry - invalid << "whitequark/retry.txt" - - # Invalid yield - invalid << "seattlerb/dasgn_icky2.txt" - invalid << "seattlerb/yield_arg.txt" - invalid << "seattlerb/yield_call_assocs.txt" - invalid << "seattlerb/yield_empty_parens.txt" - invalid << "unparser/corpus/literal/yield.txt" - invalid << "whitequark/args_assocs.txt" - invalid << "whitequark/args_assocs_legacy.txt" - invalid << "whitequark/yield.txt" - invalid << "yield.txt" - - # Dead code eliminated - invalid << "whitequark/ruby_bug_10653.txt" - - # case :a - # in Symbol(*lhs, x, *rhs) - # end - todos << "seattlerb/case_in.txt" - - # <<~HERE - # #{<<~THERE} - # THERE - # HERE - todos << "seattlerb/heredoc_nested.txt" - - base = File.join(__dir__, "fixtures") - skips = invalid | todos - - Dir["**/*.txt", base: base].each do |relative| - next if skips.include?(relative) - - filepath = File.join(base, relative) - define_method("test_#{relative}") { assert_locals(filepath) } - end - - def setup - @previous_default_external = Encoding.default_external - ignore_warnings { Encoding.default_external = Encoding::UTF_8 } - end - - def teardown - ignore_warnings { Encoding.default_external = @previous_default_external } - end - - private - - def assert_locals(filepath) - source = File.read(filepath) - - expected = YARP.const_get(:Debug).cruby_locals(source) - actual = YARP.const_get(:Debug).yarp_locals(source) - - assert_equal(expected, actual) - end - - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity +module YARP + class LocalsTest < TestCase + invalid = [] + todos = [] + + # Invalid break + invalid << "break.txt" + invalid << "if.txt" + invalid << "rescue.txt" + invalid << "seattlerb/block_break.txt" + invalid << "unless.txt" + invalid << "whitequark/break.txt" + invalid << "whitequark/break_block.txt" + + # Invalid next + invalid << "next.txt" + invalid << "seattlerb/block_next.txt" + invalid << "unparser/corpus/literal/control.txt" + invalid << "whitequark/next.txt" + invalid << "whitequark/next_block.txt" + + # Invalid redo + invalid << "keywords.txt" + invalid << "whitequark/redo.txt" + + # Invalid retry + invalid << "whitequark/retry.txt" + + # Invalid yield + invalid << "seattlerb/dasgn_icky2.txt" + invalid << "seattlerb/yield_arg.txt" + invalid << "seattlerb/yield_call_assocs.txt" + invalid << "seattlerb/yield_empty_parens.txt" + invalid << "unparser/corpus/literal/yield.txt" + invalid << "whitequark/args_assocs.txt" + invalid << "whitequark/args_assocs_legacy.txt" + invalid << "whitequark/yield.txt" + invalid << "yield.txt" + + # Dead code eliminated + invalid << "whitequark/ruby_bug_10653.txt" + + # case :a + # in Symbol(*lhs, x, *rhs) + # end + todos << "seattlerb/case_in.txt" + + # <<~HERE + # #{<<~THERE} + # THERE + # HERE + todos << "seattlerb/heredoc_nested.txt" + + base = File.join(__dir__, "fixtures") + skips = invalid | todos + + Dir["**/*.txt", base: base].each do |relative| + next if skips.include?(relative) + + filepath = File.join(base, relative) + define_method("test_#{relative}") { assert_locals(filepath) } + end + + def setup + @previous_default_external = Encoding.default_external + ignore_warnings { Encoding.default_external = Encoding::UTF_8 } + end + + def teardown + ignore_warnings { Encoding.default_external = @previous_default_external } + end + + private + + def assert_locals(filepath) + source = File.read(filepath) + + expected = Debug.cruby_locals(source) + actual = Debug.yarp_locals(source) + + assert_equal(expected, actual) + end + + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end end end diff --git a/test/yarp/location_test.rb b/test/yarp/location_test.rb index 8e357fa193c3c1..192cfdd716cdb4 100644 --- a/test/yarp/location_test.rb +++ b/test/yarp/location_test.rb @@ -3,7 +3,7 @@ require_relative "test_helper" module YARP - class LocationTest < Test::Unit::TestCase + class LocationTest < TestCase def test_AliasNode assert_location(AliasNode, "alias foo bar") end diff --git a/test/yarp/memsize_test.rb b/test/yarp/memsize_test.rb index 9ff670c11869bd..07c85ce329504a 100644 --- a/test/yarp/memsize_test.rb +++ b/test/yarp/memsize_test.rb @@ -4,12 +4,14 @@ return if YARP::BACKEND == :FFI -class MemsizeTest < Test::Unit::TestCase - def test_memsize - result = YARP.const_get(:Debug).memsize("2 + 3") +module YARP + class MemsizeTest < TestCase + def test_memsize + result = Debug.memsize("2 + 3") - assert_equal 5, result[:length] - assert_kind_of Integer, result[:memsize] - assert_equal 6, result[:node_count] + assert_equal 5, result[:length] + assert_kind_of Integer, result[:memsize] + assert_equal 6, result[:node_count] + end end end diff --git a/test/yarp/newline_test.rb b/test/yarp/newline_test.rb index 2eaaefc61ea9aa..5a85f856f39f5f 100644 --- a/test/yarp/newline_test.rb +++ b/test/yarp/newline_test.rb @@ -4,97 +4,93 @@ return unless defined?(RubyVM::InstructionSequence) -# It is useful to have a diff even if the strings to compare are big -# However, ruby/ruby does not have a version of Test::Unit with access to -# max_diff_target_string_size -if defined?(Test::Unit::Assertions::AssertionMessage) - Test::Unit::Assertions::AssertionMessage.max_diff_target_string_size = 5000 -end - -class NewlineTest < Test::Unit::TestCase - class NewlineVisitor < YARP::Visitor - attr_reader :source, :newlines - - def initialize(source) - @source = source - @newlines = [] - end +module YARP + class NewlineTest < TestCase + class NewlineVisitor < Visitor + attr_reader :source, :newlines + + def initialize(source) + @source = source + @newlines = [] + end - def visit(node) - newlines << source.line(node.location.start_offset) if node&.newline? - super(node) + def visit(node) + newlines << source.line(node.location.start_offset) if node&.newline? + super(node) + end end - end - base = File.dirname(__dir__) - Dir["{lib,test}/**/*.rb", base: base].each do |relative| - define_method("test_newline_flags_#{relative}") do - assert_newlines(base, relative) + base = File.dirname(__dir__) + Dir["{lib,test}/**/*.rb", base: base].each do |relative| + define_method("test_newline_flags_#{relative}") do + assert_newlines(base, relative) + end end - end - private + private - def assert_newlines(base, relative) - filepath = File.join(base, relative) - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - expected = rubyvm_lines(source) + def assert_newlines(base, relative) + filepath = File.join(base, relative) + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + expected = rubyvm_lines(source) - result = YARP.parse_file(filepath) - assert_empty result.errors + result = YARP.parse_file(filepath) + assert_empty result.errors - result.mark_newlines - visitor = NewlineVisitor.new(result.source) + result.mark_newlines + visitor = NewlineVisitor.new(result.source) - result.value.accept(visitor) - actual = visitor.newlines + result.value.accept(visitor) + actual = visitor.newlines - source.each_line.with_index(1) do |line, line_number| - # Lines like `while (foo = bar)` result in two line flags in the bytecode - # but only one newline flag in the AST. We need to remove the extra line - # flag from the bytecode to make the test pass. - if line.match?(/while \(/) - index = expected.index(line_number) - expected.delete_at(index) if index - end + source.each_line.with_index(1) do |line, line_number| + # Lines like `while (foo = bar)` result in two line flags in the + # bytecode but only one newline flag in the AST. We need to remove the + # extra line flag from the bytecode to make the test pass. + if line.match?(/while \(/) + index = expected.index(line_number) + expected.delete_at(index) if index + end - # Lines like `foo =` where the value is on the next line result in another - # line flag in the bytecode but only one newline flag in the AST. - if line.match?(/^\s+\w+ =$/) - if source.lines[line_number].match?(/^\s+case/) - actual[actual.index(line_number)] += 1 - else - actual.delete_at(actual.index(line_number)) + # Lines like `foo =` where the value is on the next line result in + # another line flag in the bytecode but only one newline flag in the + # AST. + if line.match?(/^\s+\w+ =$/) + if source.lines[line_number].match?(/^\s+case/) + actual[actual.index(line_number)] += 1 + else + actual.delete_at(actual.index(line_number)) + end end - end - if line.match?(/^\s+\w+ = \[$/) - if !expected.include?(line_number) && !expected.include?(line_number + 2) - actual[actual.index(line_number)] += 1 + if line.match?(/^\s+\w+ = \[$/) + if !expected.include?(line_number) && !expected.include?(line_number + 2) + actual[actual.index(line_number)] += 1 + end end end + + assert_equal expected, actual end - assert_equal expected, actual - end + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity - end + def rubyvm_lines(source) + queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] + lines = [] - def rubyvm_lines(source) - queue = [ignore_warnings { RubyVM::InstructionSequence.compile(source) }] - lines = [] + while iseq = queue.shift + lines.concat(iseq.trace_points.filter_map { |line, event| line if event == :line }) + iseq.each_child { |insn| queue << insn unless insn.label.start_with?("ensure in ") } + end - while iseq = queue.shift - lines.concat(iseq.trace_points.filter_map { |line, event| line if event == :line }) - iseq.each_child { |insn| queue << insn unless insn.label.start_with?("ensure in ") } + lines.sort end - - lines.sort end end diff --git a/test/yarp/parse_serialize_test.rb b/test/yarp/parse_serialize_test.rb index daecbe1488d2d6..82a1c29d48a10f 100644 --- a/test/yarp/parse_serialize_test.rb +++ b/test/yarp/parse_serialize_test.rb @@ -4,23 +4,25 @@ return if YARP::BACKEND == :FFI -class ParseSerializeTest < Test::Unit::TestCase - def test_parse_serialize - dumped = YARP.const_get(:Debug).parse_serialize_file(__FILE__) - result = YARP.load(File.read(__FILE__), dumped) +module YARP + class ParseSerializeTest < TestCase + def test_parse_serialize + dumped = Debug.parse_serialize_file(__FILE__) + result = YARP.load(File.read(__FILE__), dumped) - assert_kind_of YARP::ParseResult, result, "Expected the return value to be a ParseResult" - assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" - end + assert_kind_of ParseResult, result, "Expected the return value to be a ParseResult" + assert_equal __FILE__, find_file_node(result)&.filepath, "Expected the filepath to be set correctly" + end - private + private - def find_file_node(result) - queue = [result.value] + def find_file_node(result) + queue = [result.value] - while (node = queue.shift) - return node if node.is_a?(YARP::SourceFileNode) - queue.concat(node.child_nodes.compact) + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.child_nodes.compact) + end end end end diff --git a/test/yarp/parse_test.rb b/test/yarp/parse_test.rb index b288d597b23eb9..5299cfd7b11ed4 100644 --- a/test/yarp/parse_test.rb +++ b/test/yarp/parse_test.rb @@ -2,219 +2,221 @@ require_relative "test_helper" -class ParseTest < Test::Unit::TestCase - # When we pretty-print the trees to compare against the snapshots, we want to - # be certain that we print with the same external encoding. This is because - # methods like Symbol#inspect take into account external encoding and it could - # change how the snapshot is generated. On machines with certain settings - # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're - # going to force it to be UTF-8 to keep the snapshots consistent. - def setup - @previous_default_external = Encoding.default_external - ignore_warnings { Encoding.default_external = Encoding::UTF_8 } - end +module YARP + class ParseTest < TestCase + # When we pretty-print the trees to compare against the snapshots, we want to + # be certain that we print with the same external encoding. This is because + # methods like Symbol#inspect take into account external encoding and it could + # change how the snapshot is generated. On machines with certain settings + # (like LANG=C or -Eascii-8bit) this could have been changed. So here we're + # going to force it to be UTF-8 to keep the snapshots consistent. + def setup + @previous_default_external = Encoding.default_external + ignore_warnings { Encoding.default_external = Encoding::UTF_8 } + end - def teardown - ignore_warnings { Encoding.default_external = @previous_default_external } - end + def teardown + ignore_warnings { Encoding.default_external = @previous_default_external } + end - def test_empty_string - result = YARP.parse("") - assert_equal [], result.value.statements.body - end + def test_empty_string + result = YARP.parse("") + assert_equal [], result.value.statements.body + end - def test_parse_takes_file_path - filepath = "filepath.rb" - result = YARP.parse("def foo; __FILE__; end", filepath) + def test_parse_takes_file_path + filepath = "filepath.rb" + result = YARP.parse("def foo; __FILE__; end", filepath) - assert_equal filepath, find_source_file_node(result.value).filepath - end + assert_equal filepath, find_source_file_node(result.value).filepath + end - def test_parse_lex - node, tokens = YARP.parse_lex("def foo; end").value + def test_parse_lex + node, tokens = YARP.parse_lex("def foo; end").value - assert_kind_of YARP::ProgramNode, node - assert_equal 5, tokens.length - end + assert_kind_of ProgramNode, node + assert_equal 5, tokens.length + end - def test_parse_lex_file - node, tokens = YARP.parse_lex_file(__FILE__).value + def test_parse_lex_file + node, tokens = YARP.parse_lex_file(__FILE__).value - assert_kind_of YARP::ProgramNode, node - refute_empty tokens - end + assert_kind_of ProgramNode, node + refute_empty tokens + end - # To accurately compare against Ripper, we need to make sure that we're - # running on Ruby 3.2+. - ripper_enabled = RUBY_VERSION >= "3.2.0" + # To accurately compare against Ripper, we need to make sure that we're + # running on Ruby 3.2+. + ripper_enabled = RUBY_VERSION >= "3.2.0" - # The FOCUS environment variable allows you to specify one particular fixture - # to test, instead of all of them. - base = File.join(__dir__, "fixtures") - relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] + # The FOCUS environment variable allows you to specify one particular fixture + # to test, instead of all of them. + base = File.join(__dir__, "fixtures") + relatives = ENV["FOCUS"] ? [ENV["FOCUS"]] : Dir["**/*.txt", base: base] - relatives.each do |relative| - # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" - next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) + relatives.each do |relative| + # These fail on TruffleRuby due to a difference in Symbol#inspect: :测试 vs :"测试" + next if RUBY_ENGINE == "truffleruby" and %w[seattlerb/bug202.txt seattlerb/magic_encoding_comment.txt].include?(relative) - filepath = File.join(base, relative) - snapshot = File.expand_path(File.join("snapshots", relative), __dir__) + filepath = File.join(base, relative) + snapshot = File.expand_path(File.join("snapshots", relative), __dir__) + + directory = File.dirname(snapshot) + FileUtils.mkdir_p(directory) unless File.directory?(directory) - directory = File.dirname(snapshot) - FileUtils.mkdir_p(directory) unless File.directory?(directory) + ripper_should_parse = ripper_should_match = ripper_enabled - ripper_should_parse = ripper_should_match = ripper_enabled + # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if + # we're on an earlier version. + ripper_should_match = false if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" - # This file has changed behavior in Ripper in Ruby 3.3, so we skip it if - # we're on an earlier version. - ripper_should_match = false if relative == "seattlerb/pct_w_heredoc_interp_nested.txt" && RUBY_VERSION < "3.3.0" + # It seems like there are some oddities with nested heredocs and ripper. + # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. + ripper_should_match = false if relative == "seattlerb/heredoc_nested.txt" - # It seems like there are some oddities with nested heredocs and ripper. - # Waiting for feedback on https://bugs.ruby-lang.org/issues/19838. - ripper_should_match = false if relative == "seattlerb/heredoc_nested.txt" + # Ripper seems to have a bug that the regex portions before and after the heredoc are combined + # into a single token. See https://bugs.ruby-lang.org/issues/19838. + # + # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. + ripper_should_parse = false if relative == "spanning_heredoc.txt" - # Ripper seems to have a bug that the regex portions before and after the heredoc are combined - # into a single token. See https://bugs.ruby-lang.org/issues/19838. - # - # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. - ripper_should_parse = false if relative == "spanning_heredoc.txt" + define_method "test_filepath_#{relative}" do + # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, + # and explicitly set the external encoding to UTF-8 to override the binmode default. + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - define_method "test_filepath_#{relative}" do - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture + # that is invalid Ruby. + refute_nil(Ripper.sexp_raw(source), "Ripper failed to parse") if ripper_should_parse - # Make sure that it can be correctly parsed by Ripper. If it can't, then we have a fixture - # that is invalid Ruby. - refute_nil(Ripper.sexp_raw(source), "Ripper failed to parse") if ripper_should_parse + # Next, assert that there were no errors during parsing. + result = YARP.parse(source, relative) + assert_empty result.errors - # Next, assert that there were no errors during parsing. - result = YARP.parse(source, relative) - assert_empty result.errors + # Next, pretty print the source. + printed = PP.pp(result.value, +"", 79) - # Next, pretty print the source. - printed = PP.pp(result.value, +"", 79) + if File.exist?(snapshot) + saved = File.read(snapshot) - if File.exist?(snapshot) - saved = File.read(snapshot) + # If the snapshot file exists, but the printed value does not match the + # snapshot, then update the snapshot file. + if printed != saved + File.write(snapshot, printed) + warn("Updated snapshot at #{snapshot}.") + end - # If the snapshot file exists, but the printed value does not match the - # snapshot, then update the snapshot file. - if printed != saved + # If the snapshot file exists, then assert that the printed value + # matches the snapshot. + assert_equal(saved, printed) + else + # If the snapshot file does not yet exist, then write it out now. File.write(snapshot, printed) - warn("Updated snapshot at #{snapshot}.") + warn("Created snapshot at #{snapshot}.") end - # If the snapshot file exists, then assert that the printed value - # matches the snapshot. - assert_equal(saved, printed) - else - # If the snapshot file does not yet exist, then write it out now. - File.write(snapshot, printed) - warn("Created snapshot at #{snapshot}.") - end - - # Next, assert that the value can be serialized and deserialized without - # changing the shape of the tree. - assert_equal_nodes(result.value, YARP.load(source, YARP.dump(source, relative)).value) - - # Next, check that the location ranges of each node in the tree are a - # superset of their respective child nodes. - assert_non_overlapping_locations(result.value) - - # Next, assert that the newlines are in the expected places. - expected_newlines = [0] - source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } + # Next, assert that the value can be serialized and deserialized without + # changing the shape of the tree. + assert_equal_nodes(result.value, YARP.load(source, YARP.dump(source, relative)).value) - # If there's a __END__, then we should trip out those newlines because we - # don't actually scan them during parsing (because we don't need to). - if found = result.comments.find { |comment| comment.type == :__END__ } - expected_newlines = expected_newlines[...found.location.start_line] - end + # Next, check that the location ranges of each node in the tree are a + # superset of their respective child nodes. + assert_non_overlapping_locations(result.value) - assert_equal expected_newlines, YARP.const_get(:Debug).newlines(source) + # Next, assert that the newlines are in the expected places. + expected_newlines = [0] + source.b.scan("\n") { expected_newlines << $~.offset(0)[0] + 1 } - if ripper_should_parse && ripper_should_match - # Finally, assert that we can lex the source and get the same tokens as - # Ripper. - lex_result = YARP.lex_compat(source) - assert_equal [], lex_result.errors - tokens = lex_result.value + # If there's a __END__, then we should trip out those newlines because we + # don't actually scan them during parsing (because we don't need to). + if found = result.comments.find { |comment| comment.type == :__END__ } + expected_newlines = expected_newlines[...found.location.start_line] + end - begin - YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| - assert_equal ripper, yarp + assert_equal expected_newlines, Debug.newlines(source) + + if ripper_should_parse && ripper_should_match + # Finally, assert that we can lex the source and get the same tokens as + # Ripper. + lex_result = YARP.lex_compat(source) + assert_equal [], lex_result.errors + tokens = lex_result.value + + begin + YARP.lex_ripper(source).zip(tokens).each do |(ripper, yarp)| + assert_equal ripper, yarp + end + rescue SyntaxError + raise ArgumentError, "Test file has invalid syntax #{filepath}" end - rescue SyntaxError - raise ArgumentError, "Test file has invalid syntax #{filepath}" end end end - end - Dir["*.txt", base: base].each do |relative| - next if relative == "newline_terminated.txt" + Dir["*.txt", base: base].each do |relative| + next if relative == "newline_terminated.txt" - # We test every snippet (separated by \n\n) in isolation - # to ensure the parser does not try to read bytes further than the end of each snippet - define_method "test_individual_snippets_#{relative}" do - filepath = File.join(base, relative) + # We test every snippet (separated by \n\n) in isolation + # to ensure the parser does not try to read bytes further than the end of each snippet + define_method "test_individual_snippets_#{relative}" do + filepath = File.join(base, relative) - # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, - # and explicitly set the external encoding to UTF-8 to override the binmode default. - file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) + # First, read the source from the filepath. Use binmode to avoid converting CRLF on Windows, + # and explicitly set the external encoding to UTF-8 to override the binmode default. + file_contents = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| - snippet = snippet.rstrip - result = YARP.parse(snippet, relative) - assert_empty result.errors + file_contents.split(/(?<=\S)\n\n(?=\S)/).each do |snippet| + snippet = snippet.rstrip + result = YARP.parse(snippet, relative) + assert_empty result.errors - assert_equal_nodes(result.value, YARP.load(snippet, YARP.dump(snippet, relative)).value) + assert_equal_nodes(result.value, YARP.load(snippet, YARP.dump(snippet, relative)).value) + end end end - end - private + private - # Check that the location ranges of each node in the tree are a superset of - # their respective child nodes. - def assert_non_overlapping_locations(node) - queue = [node] + # Check that the location ranges of each node in the tree are a superset of + # their respective child nodes. + def assert_non_overlapping_locations(node) + queue = [node] - while (current = queue.shift) - # We only want to compare parent/child location overlap in the case that - # we are not looking at a heredoc. That's because heredoc locations are - # special in that they only use the declaration of the heredoc. - compare = !(current.is_a?(YARP::InterpolatedStringNode) || current.is_a?(YARP::InterpolatedXStringNode)) || !current.opening&.start_with?("<<") + while (current = queue.shift) + # We only want to compare parent/child location overlap in the case that + # we are not looking at a heredoc. That's because heredoc locations are + # special in that they only use the declaration of the heredoc. + compare = !(current.is_a?(InterpolatedStringNode) || current.is_a?(InterpolatedXStringNode)) || !current.opening&.start_with?("<<") - current.child_nodes.each do |child| - # child_nodes can return nil values, so we need to skip those. - next unless child + current.child_nodes.each do |child| + # child_nodes can return nil values, so we need to skip those. + next unless child - # Now that we know we have a child node, add that to the queue. - queue << child + # Now that we know we have a child node, add that to the queue. + queue << child - if compare - assert_operator current.location.start_offset, :<=, child.location.start_offset - assert_operator current.location.end_offset, :>=, child.location.end_offset + if compare + assert_operator current.location.start_offset, :<=, child.location.start_offset + assert_operator current.location.end_offset, :>=, child.location.end_offset + end end end end - end - def find_source_file_node(program) - queue = [program] - while (node = queue.shift) - return node if node.is_a?(YARP::SourceFileNode) - queue.concat(node.child_nodes.compact) + def find_source_file_node(program) + queue = [program] + while (node = queue.shift) + return node if node.is_a?(SourceFileNode) + queue.concat(node.child_nodes.compact) + end end - end - def ignore_warnings - previous_verbosity = $VERBOSE - $VERBOSE = nil - yield - ensure - $VERBOSE = previous_verbosity + def ignore_warnings + previous_verbosity = $VERBOSE + $VERBOSE = nil + yield + ensure + $VERBOSE = previous_verbosity + end end end diff --git a/test/yarp/regexp_test.rb b/test/yarp/regexp_test.rb index 241fcc862f931d..9863a54758aa1f 100644 --- a/test/yarp/regexp_test.rb +++ b/test/yarp/regexp_test.rb @@ -4,196 +4,198 @@ return if YARP::BACKEND == :FFI -class RegexpTest < Test::Unit::TestCase - ############################################################################## - # These tests test the actual use case of extracting named capture groups - ############################################################################## +module YARP + class RegexpTest < TestCase + ############################################################################## + # These tests test the actual use case of extracting named capture groups + ############################################################################## + + def test_named_captures_with_arrows + assert_equal(["foo"], named_captures("(?bar)")) + end + + def test_named_captures_with_single_quotes + assert_equal(["foo"], named_captures("(?'foo'bar)")) + end + + def test_nested_named_captures_with_arrows + assert_equal(["foo", "bar"], named_captures("(?(?baz))")) + end + + def test_nested_named_captures_with_single_quotes + assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) + end + + def test_allows_duplicate_named_captures + assert_equal(["foo", "foo"], named_captures("(?bar)(?baz)")) + end + + def test_named_capture_inside_fake_range_quantifier + assert_equal(["foo"], named_captures("foo{1, (?2)}")) + end + + ############################################################################## + # These tests test the rest of the AST. They are not exhaustive, but they + # should cover the most common cases. We test these to make sure we don't + # accidentally regress and stop being able to extract named captures. + ############################################################################## + + def test_alternation + refute_nil(named_captures("foo|bar")) + end + + def test_anchors + refute_nil(named_captures("^foo$")) + end + + def test_any + refute_nil(named_captures(".")) + end + + def test_posix_character_classes + refute_nil(named_captures("[[:digit:]]")) + end + + def test_negated_posix_character_classes + refute_nil(named_captures("[[:^digit:]]")) + end + + def test_invalid_posix_character_classes_should_fall_back_to_regular_classes + refute_nil(named_captures("[[:foo]]")) + end + + def test_character_sets + refute_nil(named_captures("[abc]")) + end + + def test_nested_character_sets + refute_nil(named_captures("[[abc]]")) + end + + def test_nested_character_sets_with_operators + refute_nil(named_captures("[[abc] && [def]]")) + end + + def test_named_capture_inside_nested_character_set + assert_equal([], named_captures("[foo (?bar)]")) + end + + def test_negated_character_sets + refute_nil(named_captures("[^abc]")) + end + + def test_character_ranges + refute_nil(named_captures("[a-z]")) + end + + def test_negated_character_ranges + refute_nil(named_captures("[^a-z]")) + end + + def test_fake_named_captures_inside_character_sets + assert_equal([], named_captures("[a-z(?)]")) + end + + def test_fake_named_capture_inside_character_set_with_escaped_ending + assert_equal([], named_captures("[a-z\\](?)]")) + end + + def test_comments + refute_nil(named_captures("(?#foo)")) + end + + def test_comments_with_escaped_parentheses + refute_nil(named_captures("(?#foo\\)\\))")) + end - def test_named_captures_with_arrows - assert_equal(["foo"], named_captures("(?bar)")) - end - - def test_named_captures_with_single_quotes - assert_equal(["foo"], named_captures("(?'foo'bar)")) - end - - def test_nested_named_captures_with_arrows - assert_equal(["foo", "bar"], named_captures("(?(?baz))")) - end - - def test_nested_named_captures_with_single_quotes - assert_equal(["foo", "bar"], named_captures("(?'foo'(?'bar'baz))")) - end - - def test_allows_duplicate_named_captures - assert_equal(["foo", "foo"], named_captures("(?bar)(?baz)")) - end - - def test_named_capture_inside_fake_range_quantifier - assert_equal(["foo"], named_captures("foo{1, (?2)}")) - end - - ############################################################################## - # These tests test the rest of the AST. They are not exhaustive, but they - # should cover the most common cases. We test these to make sure we don't - # accidentally regress and stop being able to extract named captures. - ############################################################################## - - def test_alternation - refute_nil(named_captures("foo|bar")) - end - - def test_anchors - refute_nil(named_captures("^foo$")) - end - - def test_any - refute_nil(named_captures(".")) - end - - def test_posix_character_classes - refute_nil(named_captures("[[:digit:]]")) - end - - def test_negated_posix_character_classes - refute_nil(named_captures("[[:^digit:]]")) - end - - def test_invalid_posix_character_classes_should_fall_back_to_regular_classes - refute_nil(named_captures("[[:foo]]")) - end - - def test_character_sets - refute_nil(named_captures("[abc]")) - end - - def test_nested_character_sets - refute_nil(named_captures("[[abc]]")) - end - - def test_nested_character_sets_with_operators - refute_nil(named_captures("[[abc] && [def]]")) - end - - def test_named_capture_inside_nested_character_set - assert_equal([], named_captures("[foo (?bar)]")) - end - - def test_negated_character_sets - refute_nil(named_captures("[^abc]")) - end + def test_non_capturing_groups + refute_nil(named_captures("(?:foo)")) + end - def test_character_ranges - refute_nil(named_captures("[a-z]")) - end + def test_positive_lookaheads + refute_nil(named_captures("(?=foo)")) + end - def test_negated_character_ranges - refute_nil(named_captures("[^a-z]")) - end + def test_negative_lookaheads + refute_nil(named_captures("(?!foo)")) + end - def test_fake_named_captures_inside_character_sets - assert_equal([], named_captures("[a-z(?)]")) - end - - def test_fake_named_capture_inside_character_set_with_escaped_ending - assert_equal([], named_captures("[a-z\\](?)]")) - end - - def test_comments - refute_nil(named_captures("(?#foo)")) - end - - def test_comments_with_escaped_parentheses - refute_nil(named_captures("(?#foo\\)\\))")) - end - - def test_non_capturing_groups - refute_nil(named_captures("(?:foo)")) - end - - def test_positive_lookaheads - refute_nil(named_captures("(?=foo)")) - end - - def test_negative_lookaheads - refute_nil(named_captures("(?!foo)")) - end - - def test_positive_lookbehinds - refute_nil(named_captures("(?<=foo)")) - end + def test_positive_lookbehinds + refute_nil(named_captures("(?<=foo)")) + end - def test_negative_lookbehinds - refute_nil(named_captures("(?foo)")) - end + def test_atomic_groups + refute_nil(named_captures("(?>foo)")) + end - def test_absence_operator - refute_nil(named_captures("(?~foo)")) - end + def test_absence_operator + refute_nil(named_captures("(?~foo)")) + end - def test_conditional_expression_with_index - refute_nil(named_captures("(?(1)foo)")) - end + def test_conditional_expression_with_index + refute_nil(named_captures("(?(1)foo)")) + end - def test_conditional_expression_with_name - refute_nil(named_captures("(?(foo)bar)")) - end + def test_conditional_expression_with_name + refute_nil(named_captures("(?(foo)bar)")) + end - def test_conditional_expression_with_group - refute_nil(named_captures("(?()bar)")) - end + def test_conditional_expression_with_group + refute_nil(named_captures("(?()bar)")) + end - def test_options_on_groups - refute_nil(named_captures("(?imxdau:foo)")) - end + def test_options_on_groups + refute_nil(named_captures("(?imxdau:foo)")) + end - def test_options_on_groups_with_invalid_options - assert_nil(named_captures("(?z:bar)")) - end + def test_options_on_groups_with_invalid_options + assert_nil(named_captures("(?z:bar)")) + end - def test_options_on_groups_getting_turned_off - refute_nil(named_captures("(?-imx:foo)")) - end + def test_options_on_groups_getting_turned_off + refute_nil(named_captures("(?-imx:foo)")) + end - def test_options_on_groups_some_getting_turned_on_some_getting_turned_off - refute_nil(named_captures("(?im-x:foo)")) - end + def test_options_on_groups_some_getting_turned_on_some_getting_turned_off + refute_nil(named_captures("(?im-x:foo)")) + end - def test_star_quantifier - refute_nil(named_captures("foo*")) - end + def test_star_quantifier + refute_nil(named_captures("foo*")) + end - def test_plus_quantifier - refute_nil(named_captures("foo+")) - end + def test_plus_quantifier + refute_nil(named_captures("foo+")) + end - def test_question_mark_quantifier - refute_nil(named_captures("foo?")) - end + def test_question_mark_quantifier + refute_nil(named_captures("foo?")) + end - def test_endless_range_quantifier - refute_nil(named_captures("foo{1,}")) - end + def test_endless_range_quantifier + refute_nil(named_captures("foo{1,}")) + end - def test_beginless_range_quantifier - refute_nil(named_captures("foo{,1}")) - end + def test_beginless_range_quantifier + refute_nil(named_captures("foo{,1}")) + end - def test_range_quantifier - refute_nil(named_captures("foo{1,2}")) - end + def test_range_quantifier + refute_nil(named_captures("foo{1,2}")) + end - def test_fake_range_quantifier_because_of_spaces - refute_nil(named_captures("foo{1, 2}")) - end + def test_fake_range_quantifier_because_of_spaces + refute_nil(named_captures("foo{1, 2}")) + end - private + private - def named_captures(source) - YARP.const_get(:Debug).named_captures(source) + def named_captures(source) + Debug.named_captures(source) + end end end diff --git a/test/yarp/ripper_compat_test.rb b/test/yarp/ripper_compat_test.rb index e13cef08b1fc6c..9fcdfe63c685ca 100644 --- a/test/yarp/ripper_compat_test.rb +++ b/test/yarp/ripper_compat_test.rb @@ -3,7 +3,7 @@ require_relative "test_helper" module YARP - class RipperCompatTest < Test::Unit::TestCase + class RipperCompatTest < TestCase def test_1_plus_2 assert_equivalent("1 + 2") end diff --git a/test/yarp/ruby_api_test.rb b/test/yarp/ruby_api_test.rb index 31b3f0fdec3b91..dc12012f44abf2 100644 --- a/test/yarp/ruby_api_test.rb +++ b/test/yarp/ruby_api_test.rb @@ -2,61 +2,63 @@ require_relative "test_helper" -class YARPRubyAPITest < Test::Unit::TestCase - def test_ruby_api - filepath = __FILE__ - source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) +module YARP + class RubyAPITest < TestCase + def test_ruby_api + filepath = __FILE__ + source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8) - assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value - assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) + assert_equal YARP.lex(source, filepath).value, YARP.lex_file(filepath).value + assert_equal YARP.dump(source, filepath), YARP.dump_file(filepath) - serialized = YARP.dump(source, filepath) - ast1 = YARP.load(source, serialized).value - ast2 = YARP.parse(source, filepath).value - ast3 = YARP.parse_file(filepath).value + serialized = YARP.dump(source, filepath) + ast1 = YARP.load(source, serialized).value + ast2 = YARP.parse(source, filepath).value + ast3 = YARP.parse_file(filepath).value - assert_equal_nodes ast1, ast2 - assert_equal_nodes ast2, ast3 - end + assert_equal_nodes ast1, ast2 + assert_equal_nodes ast2, ast3 + end - def test_literal_value_method - assert_equal 123, parse_expression("123").value - assert_equal 3.14, parse_expression("3.14").value - assert_equal 42i, parse_expression("42i").value - assert_equal 42.1ri, parse_expression("42.1ri").value - assert_equal 3.14i, parse_expression("3.14i").value - assert_equal 42r, parse_expression("42r").value - assert_equal 0.5r, parse_expression("0.5r").value - assert_equal 42ri, parse_expression("42ri").value - assert_equal 0.5ri, parse_expression("0.5ri").value - end + def test_literal_value_method + assert_equal 123, parse_expression("123").value + assert_equal 3.14, parse_expression("3.14").value + assert_equal 42i, parse_expression("42i").value + assert_equal 42.1ri, parse_expression("42.1ri").value + assert_equal 3.14i, parse_expression("3.14i").value + assert_equal 42r, parse_expression("42r").value + assert_equal 0.5r, parse_expression("0.5r").value + assert_equal 42ri, parse_expression("42ri").value + assert_equal 0.5ri, parse_expression("0.5ri").value + end - def test_location_join - recv, args_node, _ = parse_expression("1234 + 567").child_nodes - arg = args_node.arguments[0] + def test_location_join + recv, args_node, _ = parse_expression("1234 + 567").child_nodes + arg = args_node.arguments[0] - joined = recv.location.join(arg.location) - assert_equal 0, joined.start_offset - assert_equal 10, joined.length + joined = recv.location.join(arg.location) + assert_equal 0, joined.start_offset + assert_equal 10, joined.length - assert_raise RuntimeError, "Incompatible locations" do - arg.location.join(recv.location) - end + assert_raise RuntimeError, "Incompatible locations" do + arg.location.join(recv.location) + end - other_arg = parse_expression("1234 + 567").arguments.arguments[0] + other_arg = parse_expression("1234 + 567").arguments.arguments[0] - assert_raise RuntimeError, "Incompatible sources" do - other_arg.location.join(recv.location) - end + assert_raise RuntimeError, "Incompatible sources" do + other_arg.location.join(recv.location) + end - assert_raise RuntimeError, "Incompatible sources" do - recv.location.join(other_arg.location) + assert_raise RuntimeError, "Incompatible sources" do + recv.location.join(other_arg.location) + end end - end - private + private - def parse_expression(source) - YARP.parse(source).value.statements.body.first + def parse_expression(source) + YARP.parse(source).value.statements.body.first + end end end diff --git a/test/yarp/test_helper.rb b/test/yarp/test_helper.rb index 0be0f51651f03b..49587e3b03bc71 100644 --- a/test/yarp/test_helper.rb +++ b/test/yarp/test_helper.rb @@ -8,8 +8,15 @@ puts "Using YARP backend: #{YARP::BACKEND}" if ENV["YARP_FFI_BACKEND"] +# It is useful to have a diff even if the strings to compare are big +# However, ruby/ruby does not have a version of Test::Unit with access to +# max_diff_target_string_size +if defined?(Test::Unit::Assertions::AssertionMessage) + Test::Unit::Assertions::AssertionMessage.max_diff_target_string_size = 5000 +end + module YARP - module Assertions + class TestCase < ::Test::Unit::TestCase private def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) @@ -31,18 +38,17 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) parent: actual ) end - when YARP::SourceFileNode + when SourceFileNode deconstructed_expected = expected.deconstruct_keys(nil) deconstructed_actual = actual.deconstruct_keys(nil) assert_equal deconstructed_expected.keys, deconstructed_actual.keys - # Filepaths can be different if test suites were run - # on different machines. - # We accommodate for this by comparing the basenames, - # and not the absolute filepaths + # Filepaths can be different if test suites were run on different + # machines. We accommodate for this by comparing the basenames, and not + # the absolute filepaths. assert_equal deconstructed_expected.except(:filepath), deconstructed_actual.except(:filepath) assert_equal File.basename(deconstructed_expected[:filepath]), File.basename(deconstructed_actual[:filepath]) - when YARP::Node + when Node deconstructed_expected = expected.deconstruct_keys(nil) deconstructed_actual = actual.deconstruct_keys(nil) assert_equal deconstructed_expected.keys, deconstructed_actual.keys @@ -55,10 +61,11 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) parent: actual ) end - when YARP::Location + when Location assert_operator actual.start_offset, :<=, actual.end_offset, -> { "start_offset > end_offset for #{actual.inspect}, parent is #{parent.pretty_inspect}" } + if compare_location assert_equal( expected.start_offset, @@ -71,7 +78,6 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) actual.end_offset, -> { "End locations were different. Parent: #{parent.pretty_inspect}" } ) - end else assert_equal expected, actual @@ -84,11 +90,11 @@ def assert_valid_locations(value, parent: nil) value.each do |element| assert_valid_locations(element, parent: value) end - when YARP::Node + when Node value.deconstruct_keys(nil).each_value do |field| assert_valid_locations(field, parent: value) end - when YARP::Location + when Location assert_operator value.start_offset, :<=, value.end_offset, -> { "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" } @@ -96,5 +102,3 @@ def assert_valid_locations(value, parent: nil) end end end - -Test::Unit::TestCase.include(YARP::Assertions) diff --git a/test/yarp/unescape_test.rb b/test/yarp/unescape_test.rb index eef989ad23ab13..f39bdd0e394ee3 100644 --- a/test/yarp/unescape_test.rb +++ b/test/yarp/unescape_test.rb @@ -4,8 +4,8 @@ return if YARP::BACKEND == :FFI -module UnescapeTest - class UnescapeNoneTest < Test::Unit::TestCase +module YARP + class UnescapeNoneTest < TestCase def test_backslash assert_unescape_none("\\") end @@ -17,11 +17,11 @@ def test_single_quote private def assert_unescape_none(source) - assert_equal(source, YARP.const_get(:Debug).unescape_none(source)) + assert_equal(source, Debug.unescape_none(source)) end end - class UnescapeMinimalTest < Test::Unit::TestCase + class UnescapeMinimalTest < TestCase def test_backslash assert_unescape_minimal("\\", "\\\\") end @@ -37,11 +37,11 @@ def test_single_char private def assert_unescape_minimal(expected, source) - assert_equal(expected, YARP.const_get(:Debug).unescape_minimal(source)) + assert_equal(expected, Debug.unescape_minimal(source)) end end - class UnescapeAllTest < Test::Unit::TestCase + class UnescapeAllTest < TestCase def test_backslash assert_unescape_all("\\", "\\\\") end @@ -139,7 +139,7 @@ def test_escaping_normal_characters private def unescape_all(source) - YARP.const_get(:Debug).unescape_all(source) + Debug.unescape_all(source) end def assert_unescape_all(expected, source, forced_encoding = nil) diff --git a/test/yarp/version_test.rb b/test/yarp/version_test.rb index aaace0aa89adc1..6011eb695d7267 100644 --- a/test/yarp/version_test.rb +++ b/test/yarp/version_test.rb @@ -2,8 +2,10 @@ require_relative "test_helper" -class VersionTest < Test::Unit::TestCase - def test_version_is_set - refute_nil YARP::VERSION +module YARP + class VersionTest < TestCase + def test_version_is_set + refute_nil VERSION + end end end From cc71e2344b21e6d9931e7bdd4434c08c56a8ea00 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 10:40:29 -0400 Subject: [PATCH 157/208] [ruby/yarp] Revisit lex.rake, make lex:rubygems more useable https://github.com/ruby/yarp/commit/4c76f4a0c0 --- test/yarp/desugar_visitor_test.rb | 6 +++--- test/yarp/fuzzer_test.rb | 2 +- test/yarp/test_helper.rb | 17 ----------------- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index 4dee182ef1baa9..c03b02e67a00a1 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -13,7 +13,7 @@ def test_and_write assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo &&= bar") assert_desugars("(AndNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo &&= bar") end - + def test_or_write assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") @@ -23,7 +23,7 @@ def test_or_write assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo ||= bar") assert_desugars("(OrNode (LocalVariableReadNode) (LocalVariableWriteNode (CallNode)))", "foo = 1; foo ||= bar") end - + def test_operator_write assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") @@ -33,7 +33,7 @@ def test_operator_write assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo += bar") assert_desugars("(LocalVariableWriteNode (CallNode (LocalVariableReadNode) (ArgumentsNode (CallNode))))", "foo = 1; foo += bar") end - + private def ast_inspect(node) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 61845b91f76466..384f3ff0d203a3 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -4,7 +4,7 @@ module YARP # These tests are simply to exercise snippets found by the fuzzer that caused invalid memory access. - class FuzzerTest < Test::Unit::TestCase + class FuzzerTest < TestCase def self.snippet(name, source) define_method(:"test_fuzzer_#{name}") { YARP.dump(source) } end diff --git a/test/yarp/test_helper.rb b/test/yarp/test_helper.rb index 49587e3b03bc71..b79adf4b166e91 100644 --- a/test/yarp/test_helper.rb +++ b/test/yarp/test_helper.rb @@ -83,22 +83,5 @@ def assert_equal_nodes(expected, actual, compare_location: true, parent: nil) assert_equal expected, actual end end - - def assert_valid_locations(value, parent: nil) - case value - when Array - value.each do |element| - assert_valid_locations(element, parent: value) - end - when Node - value.deconstruct_keys(nil).each_value do |field| - assert_valid_locations(field, parent: value) - end - when Location - assert_operator value.start_offset, :<=, value.end_offset, -> { - "start_offset > end_offset for #{value.inspect}, parent is #{parent.pretty_inspect}" - } - end - end end end From 209eda599a06682b0d32aa84870038e854fc49ef Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 16:44:21 -0400 Subject: [PATCH 158/208] [ruby/yarp] BASERUBY fails because of .then not existing https://github.com/ruby/yarp/commit/db925f2b88 --- yarp/templates/template.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yarp/templates/template.rb b/yarp/templates/template.rb index ec731c3e2fa0c8..3969645643e581 100755 --- a/yarp/templates/template.rb +++ b/yarp/templates/template.rb @@ -308,7 +308,9 @@ def template(name, write_to: nil) def locals @locals ||= - YAML.load_file(File.expand_path("../config.yml", __dir__)).then do |config| + begin + config = YAML.load_file(File.expand_path("../config.yml", __dir__)) + { nodes: config.fetch("nodes").map { |node| NodeType.new(node) }.sort_by(&:name), tokens: config.fetch("tokens").map { |token| Token.new(token) }, From f80582cda8fb0d994db0c1cdf1c282f3a049485c Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 09:59:09 -0400 Subject: [PATCH 159/208] [ruby/yarp] fix: StatementsNode with out-of-order body nodes The presence of the heredocs in this snippet with invalid syntax: for < StatementsNode(16...14)( [MissingNode(16...16)(), InterpolatedStringNode(10...13)((10...13), [], (16...18)), MissingNode(13...14)()] ), (0...3), (16...16), nil, (14...14) )] which failed an assertion during serialization. With this fix, the node's locations are: [ForNode(0...14)( MultiWriteNode(4...7)([InterpolatedStringNode(4...7)((4...7), [], (14...16))], nil, nil, nil, nil), MissingNode(16...16)(), > StatementsNode(10...16)( [MissingNode(16...16)(), InterpolatedStringNode(10...13)((10...13), [], (16...18)), MissingNode(13...14)()] ), (0...3), (16...16), nil, (14...14) )] Found by the fuzzer. https://github.com/ruby/yarp/commit/09bcedc05e --- test/yarp/fuzzer_test.rb | 6 ++++++ yarp/yarp.c | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 384f3ff0d203a3..d75d1422f0efc7 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -22,5 +22,11 @@ def self.snippet(name, source) snippet "incomplete escaped list", "%w[\\" snippet "incomplete escaped regex", "/a\\" snippet "unterminated heredoc with unterminated escape at end of file", "<location.start < node->base.location.start) { node->base.location.start = statement->location.start; } + if (statement->location.end > node->base.location.end) { + node->base.location.end = statement->location.end; + } yp_node_list_append(&node->body, statement); - node->base.location.end = statement->location.end; // Every statement gets marked as a place where a newline can occur. statement->flags |= YP_NODE_FLAG_NEWLINE; From 6beaf010a447ce9fb35c85f9fb2f41685e2114ba Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 13:09:02 -0400 Subject: [PATCH 160/208] [ruby/yarp] fix: binary CallNode with out-of-order arg and receiver The snippet added in this commit previously resulted in a CallNode with inverted start and end locations: > AssocNode(15...13)( > CallNode(15...13)( StringNode(15...17)((15...16), (16...16), (16...17), ""), nil, (12...13), nil, ArgumentsNode(12...13)([MissingNode(12...13)()]), nil, nil, 0, "/" ), MissingNode(13...13)(), (13...13) ), which failed an assertion during serialization. After this change, it looks better: > AssocNode(12...13)( > CallNode(12...17)( StringNode(15...17)((15...16), (16...16), (16...17), ""), nil, (12...13), nil, ArgumentsNode(12...13)([MissingNode(12...13)()]), nil, nil, 0, "/" ), MissingNode(13...13)(), (13...13) ), Found by the fuzzer. https://github.com/ruby/yarp/commit/040aa63ad6 --- test/yarp/fuzzer_test.rb | 5 +++++ yarp/yarp.c | 8 ++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index d75d1422f0efc7..4d7a0af8e7d903 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -28,5 +28,10 @@ def self.snippet(name, source) A B EOF + snippet "create a binary call node with arg before receiver", <<~EOF + <<-A.g/{/ + A + /, ""\\ + EOF end end diff --git a/yarp/yarp.c b/yarp/yarp.c index 40d8d3c9720ea7..5b8fba9f627ce7 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -161,6 +161,10 @@ debug_token(yp_token_t * token) { #endif +/* Macros for min/max. */ +#define MIN(a,b) (((a)<(b))?(a):(b)) +#define MAX(a,b) (((a)>(b))?(a):(b)) + /******************************************************************************/ /* Lex mode manipulations */ /******************************************************************************/ @@ -1240,8 +1244,8 @@ static yp_call_node_t * yp_call_node_binary_create(yp_parser_t *parser, yp_node_t *receiver, yp_token_t *operator, yp_node_t *argument) { yp_call_node_t *node = yp_call_node_create(parser); - node->base.location.start = receiver->location.start; - node->base.location.end = argument->location.end; + node->base.location.start = MIN(receiver->location.start, argument->location.start); + node->base.location.end = MAX(receiver->location.end, argument->location.end); node->receiver = receiver; node->message_loc = YP_OPTIONAL_LOCATION_TOKEN_VALUE(operator); From bbaae3681ca9b4d00f2bb2330da1c7e3ff06cd17 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 13:30:29 -0400 Subject: [PATCH 161/208] [ruby/yarp] fix: regular expression with start and end out of order Also, a similar test and fix for interpolated regular expressions. This snippet: <<-A.g//, A /{/, ''\ previously created a regular expression node with inverted start and end: RegularExpressionNode(14...13)((14...15), (15...21), (12...13), ", ''", 0), which failed an assertion during serialization. After this change: RegularExpressionNode(12...15)((14...15), (15...21), (12...13), ", ''", 0), Found by the fuzzer. https://github.com/ruby/yarp/commit/5fef572f95 --- test/yarp/fuzzer_test.rb | 11 +++++++++++ yarp/yarp.c | 11 ++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 4d7a0af8e7d903..568c2eaf08d9c5 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -33,5 +33,16 @@ def self.snippet(name, source) A /, ""\\ EOF + snippet "regular expression with start and end out of order", <<~RUBY + <<-A.g//, + A + /{/, ''\\ + RUBY + snippet "interpolated regular expression with start and end out of order", <<~RUBY + <<-A.g/{/, + A + a + /{/, ''\\ + RUBY end end diff --git a/yarp/yarp.c b/yarp/yarp.c index 5b8fba9f627ce7..5381eabe0065ca 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -2766,8 +2766,13 @@ yp_interpolated_regular_expression_node_create(yp_parser_t *parser, const yp_tok static inline void yp_interpolated_regular_expression_node_append(yp_interpolated_regular_expression_node_t *node, yp_node_t *part) { + if (node->base.location.start > part->location.start) { + node->base.location.start = part->location.start; + } + if (node->base.location.end < part->location.end) { + node->base.location.end = part->location.end; + } yp_node_list_append(&node->parts, part); - node->base.location.end = part->location.end; } static inline void @@ -3600,8 +3605,8 @@ yp_regular_expression_node_create(yp_parser_t *parser, const yp_token_t *opening .type = YP_NODE_REGULAR_EXPRESSION_NODE, .flags = yp_regular_expression_flags_create(closing), .location = { - .start = opening->start, - .end = closing->end + .start = MIN(opening->start, closing->start), + .end = MAX(opening->end, closing->end) } }, .opening_loc = YP_LOCATION_TOKEN_VALUE(opening), From 6599ca44bbaf9d1084638b392a46f1b3277212b9 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 30 Aug 2023 15:27:12 -0400 Subject: [PATCH 162/208] [ruby/yarp] simplify the calling convention for `unescape` We don't need to pass in a destination pointer _and_ a write_to_str boolean flag. https://github.com/ruby/yarp/commit/347cb29ebb --- yarp/unescape.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/yarp/unescape.c b/yarp/unescape.c index 0d7833be5dd2f5..b0aabf5e22a62d 100644 --- a/yarp/unescape.c +++ b/yarp/unescape.c @@ -185,7 +185,7 @@ unescape_char(uint8_t value, const uint8_t flags) { // Read a specific escape sequence into the given destination. static const uint8_t * -unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t *backslash, const uint8_t *end, const uint8_t flags, bool write_to_str) { +unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t *backslash, const uint8_t *end, const uint8_t flags) { switch (backslash[1]) { case 'a': case 'b': @@ -196,7 +196,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t case 's': case 't': case 'v': - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(unescape_chars[backslash[1]], flags); } return backslash + 2; @@ -206,7 +206,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t uint8_t value; const uint8_t *cursor = backslash + unescape_octal(backslash, &value); - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(value, flags); } return cursor; @@ -216,7 +216,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t uint8_t value; const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value); - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(value, flags); } return cursor; @@ -258,7 +258,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t uint32_t value; unescape_unicode(unicode_start, (size_t) (unicode_cursor - unicode_start), &value); - if (write_to_str) { + if (dest) { *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, &parser->error_list); } @@ -276,7 +276,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t uint32_t value; unescape_unicode(backslash + 2, 4, &value); - if (write_to_str) { + if (dest) { *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, &parser->error_list); } return backslash + 6; @@ -301,9 +301,9 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t switch (backslash[2]) { case '\\': - return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); + return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL); case '?': - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(0x7f, flags); } return backslash + 3; @@ -313,7 +313,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t return backslash + 2; } - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(backslash[2], flags | YP_UNESCAPE_FLAG_CONTROL); } return backslash + 3; @@ -339,9 +339,9 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t switch (backslash[3]) { case '\\': - return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL, write_to_str); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL); case '?': - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(0x7f, flags); } return backslash + 4; @@ -351,7 +351,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t return backslash + 2; } - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(backslash[3], flags | YP_UNESCAPE_FLAG_CONTROL); } return backslash + 4; @@ -376,11 +376,11 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t } if (backslash[3] == '\\') { - return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META, write_to_str); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META); } if (char_is_ascii_printable(backslash[3])) { - if (write_to_str) { + if (dest) { dest[(*dest_length)++] = unescape_char(backslash[3], flags | YP_UNESCAPE_FLAG_META); } return backslash + 4; @@ -402,7 +402,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t default: { size_t width = yp_char_width(parser, backslash + 1, end); - if (write_to_str) { + if (dest) { memcpy(dest + *dest_length, backslash + 1, width); *dest_length += width; } @@ -503,7 +503,7 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // This is the only type of unescaping left. In this case we need to // handle all of the different unescapes. assert(unescape_type == YP_UNESCAPE_ALL); - cursor = unescape(parser, dest, &dest_length, backslash, end, YP_UNESCAPE_FLAG_NONE, true); + cursor = unescape(parser, dest, &dest_length, backslash, end, YP_UNESCAPE_FLAG_NONE); break; } @@ -555,7 +555,7 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *backslash, if (expect_single_codepoint) flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; - const uint8_t *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, false); + const uint8_t *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags); assert(cursor > backslash); return (size_t) (cursor - backslash); From f1790fa4e713f364bd6983586c0ba749e7b54968 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Wed, 30 Aug 2023 17:26:22 -0400 Subject: [PATCH 163/208] [YARP] Fix variables in compilation (#8326) * [YARP] Fixed several popped instructions * [YARP] Correctly compiling class path --- test/yarp/compiler_test.rb | 20 ++++++++++---------- yarp/yarp_compiler.c | 29 +++++++++++++++++------------ 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/test/yarp/compiler_test.rb b/test/yarp/compiler_test.rb index 88d0bddec7f8df..bb1a1f47e8550f 100644 --- a/test/yarp/compiler_test.rb +++ b/test/yarp/compiler_test.rb @@ -50,7 +50,7 @@ def test_TrueNode ############################################################################ def test_ClassVariableReadNode - # assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; end; @@yct") + assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; @@yct; end") end def test_ConstantPathNode @@ -62,11 +62,11 @@ def test_ConstantReadNode end def test_GlobalVariableReadNode - # assert_equal 1, compile("$yct = 1; $yct") + assert_equal 1, compile("$yct = 1; $yct") end def test_InstanceVariableReadNode - # assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; @yct; end") + assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; @yct; end") end ############################################################################ @@ -74,7 +74,7 @@ def test_InstanceVariableReadNode ############################################################################ def test_ClassVariableWriteNode - # assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; end") + assert_equal 1, compile("class YARP::CompilerTest; @@yct = 1; end") end def test_ConstantWriteNode @@ -90,7 +90,7 @@ def test_GlobalVariableWriteNode end def test_InstanceVariableWriteNode - # assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; end") + assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; end") end ############################################################################ @@ -100,17 +100,17 @@ def test_InstanceVariableWriteNode def test_EmbeddedVariableNode # assert_equal "1", compile('class YARP::CompilerTest; @yct = 1; "#@yct"; end') # assert_equal "1", compile('class YARP::CompilerTest; @@yct = 1; "#@@yct"; end') - # assert_equal "1", compile('$yct = 1; "#$yct"') + assert_equal "1", compile('$yct = 1; "#$yct"') end def test_InterpolatedStringNode - # assert_equal "1 1 1", compile('$yct = 1; "1 #$yct 1"') - # assert_equal "1 3 1", compile('"1 #{1 + 2} 1"') + assert_equal "1 1 1", compile('$yct = 1; "1 #$yct 1"') + assert_equal "1 3 1", compile('"1 #{1 + 2} 1"') end def test_InterpolatedSymbolNode - # assert_equal :"1 1 1", compile('$yct = 1; :"1 #$yct 1"') - # assert_equal :"1 3 1", compile(':"1 #{1 + 2} 1"') + assert_equal :"1 1 1", compile('$yct = 1; :"1 #$yct 1"') + assert_equal :"1 3 1", compile(':"1 #{1 + 2} 1"') end def test_StringConcatNode diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index 0c31cd83a0fd3e..e1e8ff66699f97 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -186,7 +186,7 @@ yp_compile_branch_condition(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const yp_no DECL_ANCHOR(cond_seq); INIT_ANCHOR(cond_seq); - yp_compile_node(iseq, cond, cond_seq, src, Qfalse, compile_context); + yp_compile_node(iseq, cond, cond_seq, src, false, compile_context); ADD_SEQ(ret, cond_seq); } break; @@ -301,7 +301,7 @@ yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_nod ADD_LABEL(ret, redo_label); if (statements) { - yp_compile_node(iseq, (yp_node_t *)statements, ret, src, Qtrue, compile_context); + yp_compile_node(iseq, (yp_node_t *)statements, ret, src, true, compile_context); } ADD_LABEL(ret, next_label); @@ -355,12 +355,13 @@ yp_new_child_iseq(rb_iseq_t *iseq, yp_scope_node_t * node, yp_parser_t *parser, static int -yp_compile_class_path(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const yp_node_t *constant_path_node, const NODE *line_node) +yp_compile_class_path(LINK_ANCHOR *const ret, rb_iseq_t *iseq, const yp_node_t *constant_path_node, const NODE *line_node, const uint8_t * src, bool popped, yp_compile_context_t *compile_context) { if (constant_path_node->type == YP_NODE_CONSTANT_PATH_NODE) { - if (((yp_constant_path_node_t *)constant_path_node)->parent) { + yp_node_t *parent = ((yp_constant_path_node_t *)constant_path_node)->parent; + if (parent) { /* Bar::Foo */ - // TODO: yp_compile_node(ret, "nd_else->nd_head", cpath->nd_head)); + yp_compile_node(iseq, parent, ret, src, popped, compile_context); return VM_DEFINECLASS_FLAG_SCOPED; } else { @@ -546,7 +547,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, // TODO: Once we merge constant path nodes correctly, fix this flag const int flags = VM_DEFINECLASS_TYPE_CLASS | (class_node->superclass ? VM_DEFINECLASS_FLAG_HAS_SUPERCLASS : 0) | - yp_compile_class_path(ret, iseq, class_node->constant_path, &dummy_line_node); + yp_compile_class_path(ret, iseq, class_node->constant_path, &dummy_line_node, src, popped, compile_context); if (class_node->superclass) { yp_compile_node(iseq, class_node->superclass, ret, src, popped, compile_context); @@ -577,7 +578,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; case YP_NODE_CLASS_VARIABLE_WRITE_NODE: { yp_class_variable_write_node_t *write_node = (yp_class_variable_write_node_t *) node; - yp_compile_node(iseq, write_node->value, ret, src, popped, compile_context); + yp_compile_node(iseq, write_node->value, ret, src, false, compile_context); if (!popped) { ADD_INSN(ret, &dummy_line_node, dup); } @@ -729,7 +730,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; case YP_NODE_GLOBAL_VARIABLE_WRITE_NODE: { yp_global_variable_write_node_t *write_node = (yp_global_variable_write_node_t *) node; - yp_compile_node(iseq, write_node->value, ret, src, popped, compile_context); + yp_compile_node(iseq, write_node->value, ret, src, false, compile_context); if (!popped) { ADD_INSN(ret, &dummy_line_node, dup); } @@ -791,10 +792,12 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } case YP_NODE_INSTANCE_VARIABLE_WRITE_NODE: { yp_instance_variable_write_node_t *write_node = (yp_instance_variable_write_node_t *) node; - yp_compile_node(iseq, write_node->value, ret, src, popped, compile_context); + yp_compile_node(iseq, write_node->value, ret, src, false, compile_context); + if (!popped) { ADD_INSN(ret, &dummy_line_node, dup); } + ID ivar_name = parse_location_symbol(&write_node->name_loc); ADD_INSN2(ret, &dummy_line_node, setinstancevariable, ID2SYM(ivar_name), @@ -954,7 +957,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const rb_iseq_t *module_iseq = NEW_CHILD_ISEQ(&scope_node, module_name, ISEQ_TYPE_CLASS, lineno); const int flags = VM_DEFINECLASS_TYPE_MODULE | - yp_compile_class_path(ret, iseq, module_node->constant_path, &dummy_line_node); + yp_compile_class_path(ret, iseq, module_node->constant_path, &dummy_line_node, src, popped, compile_context); ADD_INSN (ret, &dummy_line_node, putnil); ADD_INSN3(ret, &dummy_line_node, defineclass, ID2SYM(module_id), module_iseq, INT2FIX(flags)); @@ -1257,9 +1260,11 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, for (size_t index = 0; index < node_list.size; index++) { // We only want to have popped == false for the last instruction if (!popped && (index != node_list.size - 1)) { - popped = true; + yp_compile_node(iseq, node_list.nodes[index], ret, src, true, compile_context); + } + else { + yp_compile_node(iseq, node_list.nodes[index], ret, src, false, compile_context); } - yp_compile_node(iseq, node_list.nodes[index], ret, src, popped, compile_context); } return; } From 0ec5021f3d14707d1354158b8dec26ba6e079396 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Wed, 30 Aug 2023 17:27:08 -0400 Subject: [PATCH 164/208] [YARP] Implement BreakNode, NextNode, RedoNode (#8334) --- yarp/yarp_compiler.c | 47 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index e1e8ff66699f97..e75937d6897ea7 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -268,28 +268,25 @@ yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_nod LABEL *prev_start_label = ISEQ_COMPILE_DATA(iseq)->start_label; LABEL *prev_end_label = ISEQ_COMPILE_DATA(iseq)->end_label; LABEL *prev_redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label; - int prev_loopval_popped = ISEQ_COMPILE_DATA(iseq)->loopval_popped; // TODO: Deal with ensures in here - LABEL *next_label = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(lineno); /* next */ - LABEL *redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label = NEW_LABEL(lineno); /* redo */ - LABEL *break_label = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(lineno); /* break */ + LABEL *next_label = ISEQ_COMPILE_DATA(iseq)->start_label = NEW_LABEL(lineno); /* next */ + LABEL *redo_label = ISEQ_COMPILE_DATA(iseq)->redo_label = NEW_LABEL(lineno); /* redo */ + LABEL *break_label = ISEQ_COMPILE_DATA(iseq)->end_label = NEW_LABEL(lineno); /* break */ LABEL *end_label = NEW_LABEL(lineno); LABEL *adjust_label = NEW_LABEL(lineno); LABEL *next_catch_label = NEW_LABEL(lineno); LABEL *tmp_label = NULL; - ISEQ_COMPILE_DATA(iseq)->loopval_popped = 0; - // begin; end while true if (flags & YP_LOOP_FLAGS_BEGIN_MODIFIER) { - ADD_INSNL(ret, &dummy_line_node, jump, next_label); + tmp_label = NEW_LABEL(lineno); + ADD_INSNL(ret, &dummy_line_node, jump, tmp_label); } else { // while true; end - tmp_label = NEW_LABEL(lineno); - ADD_INSNL(ret, &dummy_line_node, jump, tmp_label); + ADD_INSNL(ret, &dummy_line_node, jump, next_label); } ADD_LABEL(ret, adjust_label); @@ -334,7 +331,6 @@ yp_compile_while(rb_iseq_t *iseq, int lineno, yp_node_flags_t flags, enum yp_nod ISEQ_COMPILE_DATA(iseq)->start_label = prev_start_label; ISEQ_COMPILE_DATA(iseq)->end_label = prev_end_label; ISEQ_COMPILE_DATA(iseq)->redo_label = prev_redo_label; - ISEQ_COMPILE_DATA(iseq)->loopval_popped = prev_loopval_popped; return; } @@ -483,6 +479,19 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } return; } + case YP_NODE_BREAK_NODE: { + yp_break_node_t *break_node = (yp_break_node_t *) node; + if (break_node->arguments) { + yp_compile_node(iseq, (yp_node_t *)break_node->arguments, ret, src, Qfalse, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_INSNL(ret, &dummy_line_node, jump, ISEQ_COMPILE_DATA(iseq)->end_label); + + return; + } case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; @@ -985,6 +994,20 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; } + case YP_NODE_NEXT_NODE: { + yp_next_node_t *next_node = (yp_next_node_t *) node; + if (next_node->arguments) { + yp_compile_node(iseq, (yp_node_t *)next_node->arguments, ret, src, Qfalse, compile_context); + } + else { + ADD_INSN(ret, &dummy_line_node, putnil); + } + + ADD_INSN(ret, &dummy_line_node, pop); + ADD_INSNL(ret, &dummy_line_node, jump, ISEQ_COMPILE_DATA(iseq)->start_label); + + return; + } case YP_NODE_NIL_NODE: if (!popped) { ADD_INSN(ret, &dummy_line_node, putnil); @@ -1086,6 +1109,10 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } return; } + case YP_NODE_REDO_NODE: { + ADD_INSNL(ret, &dummy_line_node, jump, ISEQ_COMPILE_DATA(iseq)->redo_label); + return; + } case YP_NODE_RETURN_NODE: { yp_arguments_node_t *arguments = ((yp_return_node_t *)node)->arguments; From 36786cc381c118986e66d8c3184e25adbaeaf591 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Wed, 30 Aug 2023 17:30:42 -0400 Subject: [PATCH 165/208] [YARP] Compile ProgramNode as ScopeNode (#8327) * [YARP] Compile ProgramNode as ScopeNode --- test/yarp/compiler_test.rb | 8 ++++++++ yarp/yarp_compiler.c | 7 ++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/yarp/compiler_test.rb b/test/yarp/compiler_test.rb index bb1a1f47e8550f..668908d4230606 100644 --- a/test/yarp/compiler_test.rb +++ b/test/yarp/compiler_test.rb @@ -69,6 +69,10 @@ def test_InstanceVariableReadNode assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; @yct; end") end + def test_LocalVariableReadNode + assert_equal 1, compile("yct = 1; yct") + end + ############################################################################ # Writes # ############################################################################ @@ -93,6 +97,10 @@ def test_InstanceVariableWriteNode assert_equal 1, compile("class YARP::CompilerTest; @yct = 1; end") end + def test_LocalVariableWriteNode + assert_equal 1, compile("yct = 1") + end + ############################################################################ # String-likes # ############################################################################ diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index e75937d6897ea7..57638a36fb5f93 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -1064,13 +1064,15 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, case YP_NODE_PROGRAM_NODE: { yp_program_node_t *program_node = (yp_program_node_t *) node; + yp_scope_node_t scope_node; + yp_scope_node_init((yp_node_t *)node, &scope_node); if (program_node->statements->body.size == 0) { ADD_INSN(ret, &dummy_line_node, putnil); } else { - yp_compile_node(iseq, (yp_node_t *) program_node->statements, ret, src, popped, compile_context); + yp_scope_node_t *res_node = &scope_node; + yp_compile_node(iseq, (yp_node_t *) res_node, ret, src, popped, compile_context); } - ADD_INSN(ret, &dummy_line_node, leave); return; } case YP_NODE_RANGE_NODE: { @@ -1285,7 +1287,6 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, yp_statements_node_t *statements_node = (yp_statements_node_t *) node; yp_node_list_t node_list = statements_node->body; for (size_t index = 0; index < node_list.size; index++) { - // We only want to have popped == false for the last instruction if (!popped && (index != node_list.size - 1)) { yp_compile_node(iseq, node_list.nodes[index], ret, src, true, compile_context); } From 4aa98b2760944b04b827d6ba4037548a93ef94ff Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 30 Aug 2023 17:21:50 -0400 Subject: [PATCH 166/208] [ruby/yarp] Add a value to numbered references https://github.com/ruby/yarp/commit/5d9b048971 --- .../unparser/corpus/literal/dstr.txt | 2 +- .../unparser/corpus/literal/variables.txt | 2 +- test/yarp/snapshots/whitequark/nth_ref.txt | 2 +- yarp/config.yml | 3 ++ yarp/yarp.c | 35 ++++++++++++++++++- yarp/yarp.h | 1 + 6 files changed, 41 insertions(+), 4 deletions(-) diff --git a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt index facf0171e790ed..396ebcd3daf896 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/dstr.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/dstr.txt @@ -57,7 +57,7 @@ ProgramNode(0...299)( [StringNode(146...147)(nil, (146...147), nil, "a"), EmbeddedVariableNode(147...150)( (147...148), - NumberedReferenceReadNode(148...150)() + NumberedReferenceReadNode(148...150)(1) )], (150...151) ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/variables.txt b/test/yarp/snapshots/unparser/corpus/literal/variables.txt index dfb85ec01370ad..fe98a8cd57f251 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/variables.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/variables.txt @@ -5,7 +5,7 @@ ProgramNode(0...66)( InstanceVariableReadNode(2...4)(:@a), ClassVariableReadNode(5...8)(:@@a), GlobalVariableReadNode(9...11)(), - NumberedReferenceReadNode(12...14)(), + NumberedReferenceReadNode(12...14)(1), BackReferenceReadNode(15...17)(), ConstantReadNode(18...23)(), ConstantPathNode(24...37)( diff --git a/test/yarp/snapshots/whitequark/nth_ref.txt b/test/yarp/snapshots/whitequark/nth_ref.txt index 3bfe4c9a5c3413..ca131a208ce53f 100644 --- a/test/yarp/snapshots/whitequark/nth_ref.txt +++ b/test/yarp/snapshots/whitequark/nth_ref.txt @@ -1,4 +1,4 @@ ProgramNode(0...3)( [], - StatementsNode(0...3)([NumberedReferenceReadNode(0...3)()]) + StatementsNode(0...3)([NumberedReferenceReadNode(0...3)(10)]) ) diff --git a/yarp/config.yml b/yarp/config.yml index 32a605c61d9050..8ebfb772bf4bbd 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -1703,6 +1703,9 @@ nodes: ^^^^^ end - name: NumberedReferenceReadNode + fields: + - name: number + type: uint32 comment: | Represents reading a numbered reference to a capture in the previous match. diff --git a/yarp/yarp.c b/yarp/yarp.c index 5381eabe0065ca..7df7ca0653c47e 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -613,6 +613,38 @@ yp_scope_node_init(yp_node_t *node, yp_scope_node_t *scope) { /* Node creation functions */ /******************************************************************************/ +// Parse the decimal number represented by the range of bytes. returns +// UINT32_MAX if the number fails to parse. This function assumes that the range +// of bytes has already been validated to contain only decimal digits. +static uint32_t +parse_decimal_number(yp_parser_t *parser, const uint8_t *start, const uint8_t *end) { + ptrdiff_t diff = end - start; + assert(diff > 0 && ((unsigned long) diff < SIZE_MAX)); + size_t length = (size_t) diff; + + char *digits = calloc(length + 1, sizeof(char)); + memcpy(digits, start, length); + digits[length] = '\0'; + + char *endptr; + errno = 0; + unsigned long value = strtoul(digits, &endptr, 10); + + if ((digits == endptr) || (*endptr != '\0') || (errno == ERANGE)) { + yp_diagnostic_list_append(&parser->error_list, start, end, "invalid decimal number"); + value = UINT32_MAX; + } + + free(digits); + + if (value > UINT32_MAX) { + yp_diagnostic_list_append(&parser->error_list, start, end, "invalid decimal number"); + value = UINT32_MAX; + } + + return (uint32_t) value; +} + // Parse out the options for a regular expression. static inline yp_node_flags_t yp_regular_expression_flags_create(const yp_token_t *closing) { @@ -3289,7 +3321,8 @@ yp_numbered_reference_read_node_create(yp_parser_t *parser, const yp_token_t *na { .type = YP_NODE_NUMBERED_REFERENCE_READ_NODE, .location = YP_LOCATION_TOKEN_VALUE(name), - } + }, + .number = parse_decimal_number(parser, name->start + 1, name->end) }; return node; diff --git a/yarp/yarp.h b/yarp/yarp.h index 6488d01adcbfab..378efe0c93746d 100644 --- a/yarp/yarp.h +++ b/yarp/yarp.h @@ -16,6 +16,7 @@ #include "yarp/version.h" #include +#include #include #include #include From 0aa404b9573d028d87072f40ecb9b86dc161b7ef Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 30 Aug 2023 16:49:32 -0400 Subject: [PATCH 167/208] Change heap init environment variable names This commit changes RUBY_GC_HEAP_INIT_SIZE_{40,80,160,320,640}_SLOTS to RUBY_GC_HEAP_{0,1,2,3,4}_INIT_SLOTS. This is easier to use because the user does not need to determine the slot sizes (which can vary between 32 and 64 bit systems). They now just use the heap names (`GC.stat_heap.keys`). --- NEWS.md | 2 +- gc.c | 4 ++-- man/ruby.1 | 6 +++--- ruby.c | 2 +- test/ruby/test_gc.rb | 14 +++++++------- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/NEWS.md b/NEWS.md index 6d439762ac5416..3764db7df10cc1 100644 --- a/NEWS.md +++ b/NEWS.md @@ -13,7 +13,7 @@ Note that each entry is kept to a minimum, see links for details. They are not displayed by default even in verbose mode. Turn them on with `-W:performance` or `Warning[:performance] = true`. [[Feature #19538]] * The `RUBY_GC_HEAP_INIT_SLOTS` environment variable has been deprecated and - removed. Environment variables `RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS` should be + removed. Environment variables `RUBY_GC_HEAP_%d_INIT_SLOTS` should be used instead. [[Feature #19785]] ## Core classes updates diff --git a/gc.c b/gc.c index 9178c9462fa913..e2c2779b47e932 100644 --- a/gc.c +++ b/gc.c @@ -11633,8 +11633,8 @@ gc_set_initial_pages(rb_objspace_t *objspace) for (int i = 0; i < SIZE_POOL_COUNT; i++) { rb_size_pool_t *size_pool = &size_pools[i]; - char env_key[sizeof("RUBY_GC_HEAP_INIT_SIZE_" "_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(size_pool->slot_size) * CHAR_BIT)]; - snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS", size_pool->slot_size); + char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)]; + snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i); size_t size_pool_init_slots = gc_params.size_pool_init_slots[i]; if (get_envparam_size(env_key, &size_pool_init_slots, 0)) { diff --git a/man/ruby.1 b/man/ruby.1 index cbd14e0ecce2d1..e62decbf56d752 100644 --- a/man/ruby.1 +++ b/man/ruby.1 @@ -557,9 +557,9 @@ the following 11 environment variables: .It Ev RUBY_GC_HEAP_INIT_SLOTS Initial allocation slots. Applies to all slot sizes. Introduced in Ruby 2.1, default: 10000. .Pp -.It Ev RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS -Initial allocation of slots in a specific size pool. -The available size pools can be found in `GC.stat_heap`. +.It Ev RUBY_GC_HEAP_%d_INIT_SLOTS +Initial allocation of slots in a specific heap. +The available heaps can be found in the keys of `GC.stat_heap`. Introduced in Ruby 3.3. .Pp .It Ev RUBY_GC_HEAP_FREE_SLOTS diff --git a/ruby.c b/ruby.c index 213425950f9131..be2ea2d0cde267 100644 --- a/ruby.c +++ b/ruby.c @@ -1742,7 +1742,7 @@ ruby_opt_init(ruby_cmdline_options_t *opt) * Remove this in Ruby 3.4. */ if (getenv("RUBY_GC_HEAP_INIT_SLOTS")) { rb_warn_deprecated("The environment variable RUBY_GC_HEAP_INIT_SLOTS", - "environment variables RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS"); + "environment variables RUBY_GC_HEAP_%d_INIT_SLOTS"); } #if USE_RJIT diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 8fc511331ee0c3..3967ed54bc676c 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -377,17 +377,17 @@ def test_gc_parameter } assert_in_out_err([env, "-W0", "-e", "exit"], "", [], []) assert_in_out_err([env, "-W:deprecated", "-e", "exit"], "", [], - /The environment variable RUBY_GC_HEAP_INIT_SLOTS is deprecated; use environment variables RUBY_GC_HEAP_INIT_SIZE_%d_SLOTS instead/) + /The environment variable RUBY_GC_HEAP_INIT_SLOTS is deprecated; use environment variables RUBY_GC_HEAP_%d_INIT_SLOTS instead/) env = {} - GC.stat_heap.each do |_, s| - env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = "200000" + GC.stat_heap.keys.each do |heap| + env["RUBY_GC_HEAP_#{heap}_INIT_SLOTS"] = "200000" end assert_normal_exit("exit", "", :child_env => env) env = {} - GC.stat_heap.each do |_, s| - env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = "0" + GC.stat_heap.keys.each do |heap| + env["RUBY_GC_HEAP_#{heap}_INIT_SLOTS"] = "0" end assert_normal_exit("exit", "", :child_env => env) @@ -456,8 +456,8 @@ def test_gc_parameter_init_slots env = {} # Make the heap big enough to ensure the heap never needs to grow. sizes = GC.stat_heap.keys.reverse.map { |i| (i + 1) * 100_000 } - GC.stat_heap.each do |i, s| - env["RUBY_GC_HEAP_INIT_SIZE_#{s[:slot_size]}_SLOTS"] = sizes[i].to_s + GC.stat_heap.keys.each do |heap| + env["RUBY_GC_HEAP_#{heap}_INIT_SLOTS"] = sizes[heap].to_s end assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY) SIZES = #{sizes} From dc911a47cee378d1e495c61ec3e6dc24995bdf8e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 30 Aug 2023 15:04:38 +0900 Subject: [PATCH 168/208] sync_default_gems.rb: Move ignored_paths to ignore_file_pattern_for --- tool/sync_default_gems.rb | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index 78d9f1b5a2daaa..a54bd15e1f0f54 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -447,10 +447,19 @@ def ignore_file_pattern_for(gem) # Gem-specific patterns case gem + when "rubygems" + # We don't copy any vcr_cassettes to this repository. Because the directory does not + # exist, rename detection doesn't work. So it starts with the original path `bundler/`. + %r[\A(?: + bundler/spec/support/artifice/vcr_cassettes + )\z]mx when "yarp" + # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs + # since ruby/ruby/doc is not something owned by YARP. %r[\A(?: Makefile\.in |configure\.ac + |docs/.* |fuzz/.* |rust/.* |tasks/.* @@ -527,27 +536,6 @@ def commits_in_ranges(gem, repo, default_branch, ranges) #++ def resolve_conflicts(gem, sha, edit) - # Forcibly remove any files that we don't want to copy to this repository. - # We also ignore them as new `toplevels` even when they don't conflict. - ignored_paths = [] - case gem - when "rubygems" - # We don't copy any vcr_cassettes to this repository. Because the directory does not - # exist, rename detection doesn't work. So it starts with the original path `bundler/`. - ignored_paths += %w[bundler/spec/support/artifice/vcr_cassettes] - when "yarp" - # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs - # since ruby/ruby/doc is not something owned by YARP. - ignored_paths += %w[docs/] - end - ignored_paths.each do |path| - if File.exist?(path) - puts "Removing: #{path}" - system("git", "reset", path) - rm_rf(path) - end - end - # git has inexact rename detection, so they follow directory renames even for new files. # However, new files are considered a `CONFLICT (file location)`, so you need to git-add them here. # We hope that they are not other kinds of conflicts, assuming we don't modify them in this repository. From 97df09f276424636c39653a5480f20a70cc71050 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 30 Aug 2023 15:05:50 +0900 Subject: [PATCH 169/208] sync_default_gems.rb: Refactor - Filter out files to be ignored first, then resolve conflicts. - Add "added by gem" files, instead of hard-code paths to add. - Remove gem specific patterns covered by more generic rules. --- tool/sync_default_gems.rb | 149 +++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 68 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index a54bd15e1f0f54..fa37a66f4b7607 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -457,13 +457,7 @@ def ignore_file_pattern_for(gem) # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs # since ruby/ruby/doc is not something owned by YARP. %r[\A(?: - Makefile\.in - |configure\.ac - |docs/.* - |fuzz/.* - |rust/.* - |tasks/.* - |ext/yarp/extconf\.rb + ext/yarp/extconf\.rb )\z]mx end&.tap do |pattern| patterns << pattern @@ -536,22 +530,9 @@ def commits_in_ranges(gem, repo, default_branch, ranges) #++ def resolve_conflicts(gem, sha, edit) - # git has inexact rename detection, so they follow directory renames even for new files. - # However, new files are considered a `CONFLICT (file location)`, so you need to git-add them here. - # We hope that they are not other kinds of conflicts, assuming we don't modify them in this repository. - case gem - when "rubygems" - system(*%w[git add spec/bundler]) - when "yarp" - system(*%w[git add lib/yarp]) - system(*%w[git add test/yarp]) - system(*%w[git add yarp]) - end - # Skip this commit if everything has been removed as `ignored_paths`. changes = pipe_readlines(%W"git status --porcelain -z") if changes.empty? - `git reset` && `git checkout .` && `git clean -fd` puts "Skip empty commit #{sha}" return false end @@ -560,23 +541,16 @@ def resolve_conflicts(gem, sha, edit) deleted = changes.grep(/^DD /) {$'} system(*%W"git rm -f --", *deleted) unless deleted.empty? + # Import UA: added by them + added = changes.grep(/^UA /) {$'} + system(*%W"git add --", *added) unless added.empty? + # Discover unmerged files # AU: unmerged, added by us # DU: unmerged, deleted by us # UU: unmerged, both modified - # UA: unmerged, added by them # AA: unmerged, both added - unmerged = changes.map {|line| line[/\A(?:.U|[UA]A) (.*)/, 1]} - unmerged.compact! - ignore_file_pattern = ignore_file_pattern_for(gem) - ignore, conflict = unmerged.partition {|name| ignore_file_pattern =~ name} - # Reset ignored files if they conflict - unless ignore.empty? - system(*%W"git reset HEAD --", *ignore) - File.unlink(*ignore) - ignore = pipe_readlines(%W"git status --porcelain -z" + ignore).map! {|line| line[/\A.. (.*)/, 1]} - system(*%W"git checkout HEAD --", *ignore) unless ignore.empty? - end + conflict = changes.grep(/\A(?:.U|AA) /) {$'} # If -e option is given, open each conflicted file with an editor unless conflict.empty? if edit @@ -586,52 +560,71 @@ def resolve_conflicts(gem, sha, edit) end if editor system([editor, conflict].join(' ')) + return system(*%w"git add --", *conflict) end end + return false end - # Attempt to commit the cherry-pick - system({"GIT_EDITOR"=>"true"}, *%W"git cherry-pick --no-edit --continue") || nil + return true end - def remove_toplevel_addtions(gem, sha) - # Forcibly remove any new top-level entries, and any changes under - # /test/fixtures, /test/lib, or /tool. - changed = pipe_readlines(%W"git diff --name-only -z HEAD~..HEAD --") - toplevels = changed.map {|f| f[%r[\A(?!tool/)[^/]+/?]]}.compact - toplevels.delete_if do |top| - if system(*%w"git checkout -f HEAD~ --", top, err: File::NULL) - # previously existent path - system(*%w"git checkout -f HEAD --", top, out: File::NULL) + def filter_pickup_files(changed, ignore_file_pattern, base) + toplevels = {} + remove = [] + ignore = [] + changed = changed.reject do |f| + case + when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { + remove << top unless + toplevels[top] = system(*%w"git cat-file -e", "#{base}:#{top}") + } + # Remove any new top-level directories. true + when !f.include?("/"), + f.start_with?("test/fixtures/", "test/lib/", "tool/") + # Forcibly reset any top-level entries, and any changes under + # /test/fixtures, /test/lib, or /tool. + ignore << f + when ignore_file_pattern.match?(f) + # Forcibly reset any changes matching ignore_file_pattern. + ignore << f end end - unless toplevels.empty? - puts "Remove files added to toplevel: #{toplevels.join(', ')}" - system(*%w"git rm -r --", *toplevels) + return changed, remove, ignore + end + + def pickup_files(gem, changed, picked) + # Forcibly remove any files that we don't want to copy to this + # repository. + + ignore_file_pattern = ignore_file_pattern_for(gem) + + base = picked ? "HEAD~" : "HEAD" + changed, remove, ignore = filter_pickup_files(changed, ignore_file_pattern, base) + + unless remove.empty? + puts "Remove added files: #{remove.join(', ')}" + system(*%w"git rm -fr --", *remove) + system(*%w"git commit --amend --no-edit --", *remove) if picked end - tools = changed.select {|f|f.start_with?("test/fixtures/", "test/lib/", "tool/")} - unless tools.empty? - system(*%W"git rm -r --", *tools) - system(*%W"git checkout HEAD~ --", *tools) + + unless ignore.empty? + puts "Reset ignored files: #{ignore.join(', ')}" + system(*%W"git checkout -f", base, "--", *ignore) end - if toplevels.empty? and tools.empty? - return true - elsif system(*%W"git diff --quiet HEAD~") - `git reset HEAD~ --` && `git checkout .` && `git clean -fd` - puts "Skip commit #{sha} only for tools or toplevel" - return false - elsif system(*%W"git commit --amend --no-edit --", *toplevels, *tools) - return true - else - `git reset HEAD~ --` && `git checkout .` && `git clean -fd` + + if changed.empty? return nil end + + return changed end def pickup_commit(gem, sha, edit) # Attempt to cherry-pick a commit result = IO.popen(%W"git cherry-pick #{sha}", &:read) + picked = $?.success? if result =~ /nothing\ to\ commit/ `git reset` puts "Skip empty commit #{sha}" @@ -643,14 +636,36 @@ def pickup_commit(gem, sha, edit) return false end - # Skip the commit if it's empty or the cherry-pick attempt failed - if /^CONFLICT/ =~ result and !resolve_conflicts(gem, sha, edit) + if picked + changed = pipe_readlines(%w"git diff-tree --name-only -r -z HEAD~..HEAD --") + else + changed = pipe_readlines(%w"git diff --name-only -r -z HEAD --") + end + + # Pick up files to merge. + unless changed = pickup_files(gem, changed, picked) + puts "Skip commit #{sha} only for tools or toplevel" + if picked + `git reset --hard HEAD~` + else + `git cherry-pick --abort` + end + return false + end + + # If the cherry-pick attempt failed, try to resolve conflicts. + # Skip the commit, if it contains unresolved conflicts or no files to pick up. + unless picked or resolve_conflicts(gem, sha, edit) `git reset` && `git checkout .` && `git clean -fd` - return nil + return picked || nil # Fail unless cherry-picked end - result = remove_toplevel_addtions(gem, sha) - return result unless result + # Commit cherry-picked commit + if picked + system(*%w"git commit --amend --no-edit") + else + system(*%w"git cherry-pick --continue --no-edit") + end or return nil # Amend the commit if RDoc references need to be replaced head = `git log --format=%H -1 HEAD`.chomp @@ -683,10 +698,8 @@ def sync_default_gems_with_commits(gem, ranges, edit: nil) commits = commits_in_ranges(gem, repo, default_branch, ranges) # Ignore Merge commits and already-merged commits. - ignore_file_pattern = ignore_file_pattern_for(gem) commits.delete_if do |sha, subject| - files = pipe_readlines(%W"git diff-tree -z --no-commit-id --name-only -r #{sha}") - subject.start_with?("Merge", "Auto Merge") or files.all?(ignore_file_pattern) + subject.start_with?("Merge", "Auto Merge") end if commits.empty? From 3d2a83b9093019274b8ef22d4010fac433599533 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 31 Aug 2023 10:35:10 +0900 Subject: [PATCH 170/208] sync_default_gems.rb: Remove the pattern for bundler The "bundler" directory at the top-level will be removed by more generic rule for top-level new entries. --- tool/sync_default_gems.rb | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index fa37a66f4b7607..a4700bdb200073 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -447,12 +447,6 @@ def ignore_file_pattern_for(gem) # Gem-specific patterns case gem - when "rubygems" - # We don't copy any vcr_cassettes to this repository. Because the directory does not - # exist, rename detection doesn't work. So it starts with the original path `bundler/`. - %r[\A(?: - bundler/spec/support/artifice/vcr_cassettes - )\z]mx when "yarp" # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs # since ruby/ruby/doc is not something owned by YARP. From cbe36ef6ccb9cb19d97612a85eede68cb3c073cb Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 31 Aug 2023 11:24:08 +0900 Subject: [PATCH 171/208] sync_default_gems.rb: Remove the pattern for yarp Changes to `ext/yarp/extconf.rb` are detected as conflicts now, and ignored. --- tool/sync_default_gems.rb | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index a4700bdb200073..a51e324831cb97 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -447,12 +447,7 @@ def ignore_file_pattern_for(gem) # Gem-specific patterns case gem - when "yarp" - # Rename detection never works between ruby/ruby/doc and ruby/yarp/docs - # since ruby/ruby/doc is not something owned by YARP. - %r[\A(?: - ext/yarp/extconf\.rb - )\z]mx + when nil end&.tap do |pattern| patterns << pattern end From 2ce1b77ce0e776281142ab23523b745c9416105e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 31 Aug 2023 11:32:29 +0900 Subject: [PATCH 172/208] sync_default_gems.rb: Discard some error messages These messages are expected: - `cat-file -e` at newly added entries. - `commit --amend` when all changes removed. --- tool/sync_default_gems.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index a51e324831cb97..2879ff60812380 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -566,7 +566,7 @@ def filter_pickup_files(changed, ignore_file_pattern, base) case when toplevels.fetch(top = f[%r[\A[^/]+(?=/|\z)]m]) { remove << top unless - toplevels[top] = system(*%w"git cat-file -e", "#{base}:#{top}") + toplevels[top] = system(*%w"git cat-file -e", "#{base}:#{top}", err: File::NULL) } # Remove any new top-level directories. true @@ -595,7 +595,9 @@ def pickup_files(gem, changed, picked) unless remove.empty? puts "Remove added files: #{remove.join(', ')}" system(*%w"git rm -fr --", *remove) - system(*%w"git commit --amend --no-edit --", *remove) if picked + if picked + system(*%w"git commit --amend --no-edit --", *remove, %i[out err] => File::NULL) + end end unless ignore.empty? From fa70e361e08b47d957ca163fbc76cc0e2d89e91f Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 31 Aug 2023 14:06:35 +0900 Subject: [PATCH 173/208] CodeQL: Increase memory size [ci skip] Query evaluation ran out of Java heap frequently since CodeQL 2.14.3. --- .github/workflows/codeql-analysis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 04edf2364101a7..dcf3ee87062c54 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -78,6 +78,7 @@ jobs: category: '/language:${{ matrix.language }}' upload: False output: sarif-results + ram: 8192 - name: filter-sarif uses: advanced-security/filter-sarif@f3b8118a9349d88f7b1c0c488476411145b6270d # v1.0 From b7364069bf22856acd64c8faa3c0e296d9535c6a Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 31 Aug 2023 11:48:52 +0900 Subject: [PATCH 174/208] [YARP] Remove Java templates [ci skip] --- .../org/yarp/AbstractNodeVisitor.java.erb | 14 - yarp/templates/java/org/yarp/Loader.java.erb | 293 ------------------ yarp/templates/java/org/yarp/Nodes.java.erb | 291 ----------------- 3 files changed, 598 deletions(-) delete mode 100644 yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb delete mode 100644 yarp/templates/java/org/yarp/Loader.java.erb delete mode 100644 yarp/templates/java/org/yarp/Nodes.java.erb diff --git a/yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb b/yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb deleted file mode 100644 index fa9f65a84acbcb..00000000000000 --- a/yarp/templates/java/org/yarp/AbstractNodeVisitor.java.erb +++ /dev/null @@ -1,14 +0,0 @@ -package org.yarp; - -// GENERATED BY <%= File.basename(__FILE__) %> -public abstract class AbstractNodeVisitor { - - protected abstract T defaultVisit(Nodes.Node node); - - <%- nodes.each do |node| -%> - public T visit<%= node.name -%>(Nodes.<%= node.name -%> node) { - return defaultVisit(node); - } - - <%- end -%> -} diff --git a/yarp/templates/java/org/yarp/Loader.java.erb b/yarp/templates/java/org/yarp/Loader.java.erb deleted file mode 100644 index 4cab15be2082be..00000000000000 --- a/yarp/templates/java/org/yarp/Loader.java.erb +++ /dev/null @@ -1,293 +0,0 @@ -package org.yarp; - -import org.yarp.ParseResult; - -import java.lang.Short; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; - -// GENERATED BY <%= File.basename(__FILE__) %> -// @formatter:off -public class Loader { - - public static ParseResult load(byte[] serialized, Nodes.Source source) { - return new Loader(serialized, source).load(); - } - - private static final class ConstantPool { - - private final byte[] source; - private final int bufferOffset; - private final byte[][] cache; - - ConstantPool(byte[] source, int bufferOffset, int length) { - this.source = source; - this.bufferOffset = bufferOffset; - cache = new byte[length][]; - } - - byte[] get(ByteBuffer buffer, int oneBasedIndex) { - int index = oneBasedIndex - 1; - byte[] constant = cache[index]; - if (constant == null) { - int offset = bufferOffset + index * 8; - int start = buffer.getInt(offset); - int length = buffer.getInt(offset + 4); - - constant = new byte[length]; - System.arraycopy(source, start, constant, 0, length); - cache[index] = constant; - } - return constant; - } - - } - - private final ByteBuffer buffer; - private ConstantPool constantPool; - private final Nodes.Source source; - - private Loader(byte[] serialized, Nodes.Source source) { - this.buffer = ByteBuffer.wrap(serialized).order(ByteOrder.nativeOrder()); - this.source = source; - } - - private ParseResult load() { - expect((byte) 'Y'); - expect((byte) 'A'); - expect((byte) 'R'); - expect((byte) 'P'); - - expect((byte) 0); - expect((byte) 9); - expect((byte) 0); - - // This loads the name of the encoding. We don't actually do anything - // with it just yet. - int encodingLength = loadVarInt(); - byte[] encodingName = new byte[encodingLength]; - buffer.get(encodingName); - - ParseResult.Comment[] comments = loadComments(); - ParseResult.Error[] errors = loadSyntaxErrors(); - ParseResult.Warning[] warnings = loadWarnings(); - - int constantPoolBufferOffset = buffer.getInt(); - int constantPoolLength = loadVarInt(); - this.constantPool = new ConstantPool(source.bytes, constantPoolBufferOffset, constantPoolLength); - - Nodes.Node node = loadNode(); - - int left = constantPoolBufferOffset - buffer.position(); - if (left != 0) { - throw new Error("Expected to consume all bytes while deserializing but there were " + left + " bytes left"); - } - - boolean[] newlineMarked = new boolean[1 + source.getLineCount()]; - MarkNewlinesVisitor visitor = new MarkNewlinesVisitor(source, newlineMarked); - node.accept(visitor); - - return new ParseResult(node, comments, errors, warnings); - } - - private byte[] loadEmbeddedString() { - int length = loadVarInt(); - byte[] string = new byte[length]; - buffer.get(string); - return string; - } - - private byte[] loadString() { - switch (buffer.get()) { - case 1: - int start = loadVarInt(); - int length = loadVarInt(); - byte[] string = new byte[length]; - System.arraycopy(source.bytes, start, string, 0, length); - return string; - case 2: - return loadEmbeddedString(); - default: - throw new Error("Expected 0 or 1 but was " + buffer.get()); - } - } - - private ParseResult.Comment[] loadComments() { - int count = loadVarInt(); - ParseResult.Comment[] comments = new ParseResult.Comment[count]; - - for (int i = 0; i < count; i++) { - ParseResult.CommentType type = ParseResult.CommentType.VALUES[buffer.get()]; - Nodes.Location location = loadLocation(); - - ParseResult.Comment comment = new ParseResult.Comment(type, location); - comments[i] = comment; - } - - return comments; - } - - private ParseResult.Error[] loadSyntaxErrors() { - int count = loadVarInt(); - ParseResult.Error[] errors = new ParseResult.Error[count]; - - // error messages only contain ASCII characters - for (int i = 0; i < count; i++) { - byte[] bytes = loadEmbeddedString(); - String message = new String(bytes, StandardCharsets.US_ASCII); - Nodes.Location location = loadLocation(); - - ParseResult.Error error = new ParseResult.Error(message, location); - errors[i] = error; - } - - return errors; - } - - private ParseResult.Warning[] loadWarnings() { - int count = loadVarInt(); - ParseResult.Warning[] warnings = new ParseResult.Warning[count]; - - // warning messages only contain ASCII characters - for (int i = 0; i < count; i++) { - byte[] bytes = loadEmbeddedString(); - String message = new String(bytes, StandardCharsets.US_ASCII); - Nodes.Location location = loadLocation(); - - ParseResult.Warning warning = new ParseResult.Warning(message, location); - warnings[i] = warning; - } - - return warnings; - } - - private Nodes.Node loadOptionalNode() { - if (buffer.get(buffer.position()) != 0) { - return loadNode(); - } else { - buffer.position(buffer.position() + 1); // continue after the 0 byte - return null; - } - } - - private Nodes.Location[] loadLocations() { - int length = loadVarInt(); - if (length == 0) { - return Nodes.Location.EMPTY_ARRAY; - } - Nodes.Location[] locations = new Nodes.Location[length]; - for (int i = 0; i < length; i++) { - locations[i] = loadLocation(); - } - return locations; - } - - private byte[] loadConstant() { - return constantPool.get(buffer, loadVarInt()); - } - - private byte[][] loadConstants() { - int length = loadVarInt(); - if (length == 0) { - return Nodes.EMPTY_BYTE_ARRAY_ARRAY; - } - byte[][] constants = new byte[length][]; - for (int i = 0; i < length; i++) { - constants[i] = constantPool.get(buffer, loadVarInt()); - } - return constants; - } - - private Nodes.Node[] loadNodes() { - int length = loadVarInt(); - if (length == 0) { - return Nodes.Node.EMPTY_ARRAY; - } - Nodes.Node[] nodes = new Nodes.Node[length]; - for (int i = 0; i < length; i++) { - nodes[i] = loadNode(); - } - return nodes; - } - - private Nodes.Location loadLocation() { - return new Nodes.Location(loadVarInt(), loadVarInt()); - } - - private Nodes.Location loadOptionalLocation() { - if (buffer.get() != 0) { - return loadLocation(); - } else { - return null; - } - } - - // From https://github.com/protocolbuffers/protobuf/blob/v23.1/java/core/src/main/java/com/google/protobuf/BinaryReader.java#L1507 - private int loadVarInt() { - int x; - if ((x = buffer.get()) >= 0) { - return x; - } else if ((x ^= (buffer.get() << 7)) < 0) { - x ^= (~0 << 7); - } else if ((x ^= (buffer.get() << 14)) >= 0) { - x ^= (~0 << 7) ^ (~0 << 14); - } else if ((x ^= (buffer.get() << 21)) < 0) { - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21); - } else { - x ^= buffer.get() << 28; - x ^= (~0 << 7) ^ (~0 << 14) ^ (~0 << 21) ^ (~0 << 28); - } - return x; - } - - private short loadFlags() { - int flags = loadVarInt(); - assert flags >= 0 && flags <= Short.MAX_VALUE; - return (short) flags; - } - - private Nodes.Node loadNode() { - int type = buffer.get() & 0xFF; - int startOffset = loadVarInt(); - int length = loadVarInt(); - - switch (type) { - <%- nodes.each_with_index do |node, index| -%> - case <%= index + 1 %>: - <%- - params = node.needs_serialized_length? ? ["buffer.getInt()"] : [] - params.concat node.fields.map { |field| - case field - when YARP::NodeField then "#{field.java_cast}loadNode()" - when YARP::OptionalNodeField then "#{field.java_cast}loadOptionalNode()" - when YARP::StringField then "loadString()" - when YARP::NodeListField then "loadNodes()" - when YARP::LocationListField then "loadLocations()" - when YARP::ConstantField then "loadConstant()" - when YARP::ConstantListField then "loadConstants()" - when YARP::LocationField then "loadLocation()" - when YARP::OptionalLocationField then "loadOptionalLocation()" - when YARP::UInt32Field then "loadVarInt()" - when YARP::FlagsField then "loadFlags()" - else raise - end - } - params.concat ["startOffset", "length"] - -%> - return new Nodes.<%= node.name %>(<%= params.join(", ") -%>); - <%- end -%> - default: - throw new Error("Unknown node type: " + type); - } - } - - private void expect(byte value) { - byte b = buffer.get(); - if (b != value) { - throw new Error("Expected " + value + " but was " + b + " at position " + buffer.position()); - } - } - -} -// @formatter:on diff --git a/yarp/templates/java/org/yarp/Nodes.java.erb b/yarp/templates/java/org/yarp/Nodes.java.erb deleted file mode 100644 index 772e1e21e4d8d3..00000000000000 --- a/yarp/templates/java/org/yarp/Nodes.java.erb +++ /dev/null @@ -1,291 +0,0 @@ -package org.yarp; - -import java.lang.Override; -import java.lang.String; -import java.lang.StringBuilder; -import java.util.ArrayList; -import java.util.Arrays; - -// GENERATED BY <%= File.basename(__FILE__) %> -// @formatter:off -public abstract class Nodes { - - public static final byte[][] EMPTY_BYTE_ARRAY_ARRAY = {}; - - public static final class Location { - - public static final Location[] EMPTY_ARRAY = {}; - - public final int startOffset; - public final int length; - - public Location(int startOffset, int length) { - this.startOffset = startOffset; - this.length = length; - } - - public int endOffset() { - return startOffset + length; - } - } - - public static final class Source { - public final byte[] bytes; - private final int[] lineOffsets; - - public Source(byte[] bytes) { - this(bytes, computeLineOffsets(bytes)); - } - - public Source(byte[] bytes, int[] lineOffsets) { - assert lineOffsets[0] == 0; - this.bytes = bytes; - this.lineOffsets = lineOffsets; - } - - public static int[] computeLineOffsets(byte[] bytes) { - int[] lineOffsets = new int[8]; - int lineOffsetsSize = 0; - lineOffsets[lineOffsetsSize++] = 0; - - for (int i = 0; i < bytes.length; i++) { - if (bytes[i] == '\n') { - if (lineOffsetsSize == lineOffsets.length) { - lineOffsets = Arrays.copyOf(lineOffsets, lineOffsets.length * 2); - } - lineOffsets[lineOffsetsSize++] = i + 1; - } - } - return Arrays.copyOf(lineOffsets, lineOffsetsSize); - } - - public int line(int byteOffset) { - assert byteOffset >= 0 && byteOffset < bytes.length : byteOffset; - int index = Arrays.binarySearch(lineOffsets, byteOffset); - int line; - if (index < 0) { - line = -index - 1; - } else { - line = index + 1; - } - assert line >= 1 && line <= getLineCount() : line; - return line; - } - - public int getLineCount() { - return lineOffsets.length; - } - } - - public static abstract class Node { - - public static final Node[] EMPTY_ARRAY = {}; - - public final int startOffset; - public final int length; - private boolean newLineFlag = false; - - public Node(int startOffset, int length) { - this.startOffset = startOffset; - this.length = length; - } - - public final int endOffset() { - return startOffset + length; - } - - public final boolean hasNewLineFlag() { - return newLineFlag; - } - - public void setNewLineFlag(Source source, boolean[] newlineMarked) { - int line = source.line(this.startOffset); - if (!newlineMarked[line]) { - newlineMarked[line] = true; - this.newLineFlag = true; - } - } - - public abstract T accept(AbstractNodeVisitor visitor); - - public abstract void visitChildNodes(AbstractNodeVisitor visitor); - - public abstract Node[] childNodes(); - - @Override - public String toString() { - return toString(""); - } - - private String toString(String indent) { - StringBuilder builder = new StringBuilder(); - builder.append(indent).append(this.getClass().getSimpleName()); - if (hasNewLineFlag()) { - builder.append("[Li]"); - } - builder.append('\n'); - for (Node child : childNodes()) { - if (child != null) { - builder.append(child.toString(indent + " ")); - } - } - return builder.toString(); - } - } -<%# FLAGS -%> - <%- flags.each do |group| -%> - - public static final class <%= group.name %> implements Comparable<<%= group.name %>> { - <%- group.values.each_with_index do |value, index| -%> - - // <%= value.comment %> - public static final short <%= value.name %> = 1 << <%= index %>; - <%- end -%> - - <%- group.values.each do |value| -%> - public static boolean is<%= value.camelcase %>(short flags) { - return (flags & <%= value.name %>) != 0; - } - - <%- end -%> - private final short flags; - - public <%= group.name %>(short flags) { - this.flags = flags; - } - - @Override - public int hashCode() { - return flags; - } - - @Override - public boolean equals(Object other) { - if (!(other instanceof <%= group.name %>)) { - return false; - } - - return flags == ((<%= group.name %>) other).flags; - } - - @Override - public int compareTo(<%= group.name %> other) { - return flags - other.flags; - } - - <%- group.values.each do |value| -%> - public boolean is<%= value.camelcase %>() { - return (flags & <%= value.name %>) != 0; - } - - <%- end -%> - } -<%- end -%> -<%# NODES -%> - <%- nodes.each do |node| -%> - - <%= "#{node.comment.split("\n").map { |line| "// #{line}" }.join("\n ")}\n" if node.comment -%> - public static final class <%= node.name -%> extends Node { - <%- if node.needs_serialized_length? -%> - public final int serializedLength; - <%- end -%> - <%- node.fields.each do |field| -%> - public final <%= field.java_type %> <%= field.name %>;<%= ' // optional' if field.class.name.start_with?('Optional') %> - <%- end -%> - - <%- - params = node.needs_serialized_length? ? ["int serializedLength"] : [] - params.concat node.fields.map { "#{_1.java_type} #{_1.name}" } - params.concat ["int startOffset", "int length"] - -%> - public <%=node.name -%>(<%= params.join(", ") %>) { - super(startOffset, length); - <%- if node.needs_serialized_length? -%> - this.serializedLength = serializedLength; - <%- end -%> - <%- node.fields.each do |field| -%> - this.<%= field.name %> = <%= field.name %>; - <%- end -%> - } - <%# methods for flags -%> - <%- node.fields.each do |field| -%> - <%- if field.is_a?(YARP::FlagsField) -%> - <%- flags.find { |flag| flag.name == field.kind }.tap { raise "Expected to find #{field.kind}" unless _1 }.values.each do |value| -%> - - public boolean is<%= value.camelcase %>() { - return <%= field.kind %>.is<%= value.camelcase %>(this.<%= field.name %>); - } - <%- end -%> - <%- end -%> - <%- end -%> - <%# potential override of setNewLineFlag() -%> - <%- if node.newline == false -%> - - @Override - public void setNewLineFlag(Source source, boolean[] newlineMarked) { - // Never mark <%= node.name %> with a newline flag, mark children instead - } - <%- elsif node.newline.is_a?(String) -%> - - @Override - public void setNewLineFlag(Source source, boolean[] newlineMarked) { - <%- field = node.fields.find { |f| f.name == node.newline } or raise node.newline -%> - <%- case field -%> - <%- when YARP::NodeField, YARP::OptionalNodeField -%> - this.<%= field.name %>.setNewLineFlag(source, newlineMarked); - <%- when YARP::NodeListField -%> - Node first = this.<%= field.name %>.length > 0 ? this.<%= field.name %>[0] : null; - if (first != null) { - first.setNewLineFlag(source, newlineMarked); - } - <%- else raise field.class.name -%> - <%- end -%> - } - <%- end -%> - - public void visitChildNodes(AbstractNodeVisitor visitor) { - <%- node.fields.each do |field| -%> - <%- case field -%> - <%- when YARP::NodeListField -%> - for (Nodes.Node child : this.<%= field.name %>) { - child.accept(visitor); - } - <%- when YARP::NodeField -%> - this.<%= field.name %>.accept(visitor); - <%- when YARP::OptionalNodeField -%> - if (this.<%= field.name %> != null) { - this.<%= field.name %>.accept(visitor); - } - <%- end -%> - <%- end -%> - } - - public Node[] childNodes() { - <%- if node.fields.none?(YARP::NodeListField) and node.fields.none?(YARP::NodeKindField) -%> - return EMPTY_ARRAY; - <%- elsif node.fields.one?(YARP::NodeListField) and node.fields.none?(YARP::NodeKindField) -%> - return this.<%= node.fields.grep(YARP::NodeListField).first.name %>; - <%- elsif node.fields.none?(YARP::NodeListField) -%> - return new Node[] { <%= node.fields.grep(YARP::NodeKindField).map { "this.#{_1.name}" }.join(', ') %> }; - <%- else -%> - ArrayList childNodes = new ArrayList<>(); - <%- node.fields.each do |field| -%> - <%- case field -%> - <%- when YARP::NodeField, YARP::OptionalNodeField -%> - childNodes.add(this.<%= field.name %>); - <%- when YARP::NodeListField -%> - childNodes.addAll(Arrays.asList(this.<%= field.name %>)); - <%- end -%> - <%- end -%> - return childNodes.toArray(EMPTY_ARRAY); - <%- end -%> - } - - public T accept(AbstractNodeVisitor visitor) { - return visitor.visit<%= node.name -%>(this); - } - } - <%- end -%> - -} -// @formatter:on From 07ddb3589fcc07f68e19dee56e0722f9e0ea0682 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Thu, 31 Aug 2023 16:35:07 +0900 Subject: [PATCH 175/208] [ruby/shellwords] omit blank line https://github.com/ruby/shellwords/commit/b45de514ab --- lib/shellwords.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/shellwords.rb b/lib/shellwords.rb index 5866601b1792e5..58f4af26ae56cd 100644 --- a/lib/shellwords.rb +++ b/lib/shellwords.rb @@ -68,7 +68,6 @@ # 1: {IEEE Std 1003.1-2008, 2016 Edition, the Shell & Utilities volume}[http://pubs.opengroup.org/onlinepubs/9699919799/utilities/contents.html] module Shellwords - VERSION = "0.1.0" # Splits a string into an array of tokens in the same way the UNIX From 24b2bb465f6847e5d5df1845e022c940fdf63f48 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 18 Jan 2023 10:25:25 +0900 Subject: [PATCH 176/208] Separate test for `Kernel#Integer` with `base` argument --- test/ruby/test_integer.rb | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/test/ruby/test_integer.rb b/test/ruby/test_integer.rb index a3e64ddb388840..3f35c3bfbfd550 100644 --- a/test/ruby/test_integer.rb +++ b/test/ruby/test_integer.rb @@ -138,20 +138,6 @@ def test_Integer assert_equal(1234, Integer(1234)) assert_equal(1, Integer(1.234)) - # base argument - assert_equal(1234, Integer("1234", 10)) - assert_equal(668, Integer("1234", 8)) - assert_equal(4660, Integer("1234", 16)) - assert_equal(49360, Integer("1234", 36)) - # decimal, not octal - assert_equal(1234, Integer("01234", 10)) - assert_raise(ArgumentError) { Integer("0x123", 10) } - assert_raise(ArgumentError) { Integer(1234, 10) } - assert_raise(ArgumentError) { Integer(12.34, 10) } - assert_raise(ArgumentError) { Integer(Object.new, 1) } - - assert_raise(ArgumentError) { Integer(1, 1, 1) } - assert_equal(2 ** 50, Integer(2.0 ** 50)) assert_raise(TypeError) { Integer(nil) } @@ -252,6 +238,21 @@ def (obj = Object.new).to_str assert_equal(16, Integer(obj)) end + def test_Integer_with_base + assert_equal(1234, Integer("1234", 10)) + assert_equal(668, Integer("1234", 8)) + assert_equal(4660, Integer("1234", 16)) + assert_equal(49360, Integer("1234", 36)) + # decimal, not octal + assert_equal(1234, Integer("01234", 10)) + assert_raise(ArgumentError) { Integer("0x123", 10) } + assert_raise(ArgumentError) { Integer(1234, 10) } + assert_raise(ArgumentError) { Integer(12.34, 10) } + assert_raise(ArgumentError) { Integer(Object.new, 1) } + + assert_raise(ArgumentError) { Integer(1, 1, 1) } + end + def test_int_p assert_not_predicate(1.0, :integer?) assert_predicate(1, :integer?) From c45176dbca2bd082cb199e9411e4dfc5ec162352 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 18 Jan 2023 10:30:44 +0900 Subject: [PATCH 177/208] [Bug #19349] Respect `#to_int` of `base` argument --- kernel.rb | 93 +++++++++++++++++++++++++++++ object.c | 122 +++----------------------------------- test/ruby/test_integer.rb | 11 ++++ 3 files changed, 112 insertions(+), 114 deletions(-) diff --git a/kernel.rb b/kernel.rb index c8cbc991757fab..43d1abec31118e 100644 --- a/kernel.rb +++ b/kernel.rb @@ -216,4 +216,97 @@ def Float(arg, exception: true) Primitive.rb_f_float(arg, exception) end end + + # call-seq: + # Integer(object, base = 0, exception: true) -> integer or nil + # + # Returns an integer converted from +object+. + # + # Tries to convert +object+ to an integer + # using +to_int+ first and +to_i+ second; + # see below for exceptions. + # + # With a non-zero +base+, +object+ must be a string or convertible + # to a string. + # + # ==== numeric objects + # + # With integer argument +object+ given, returns +object+: + # + # Integer(1) # => 1 + # Integer(-1) # => -1 + # + # With floating-point argument +object+ given, + # returns +object+ truncated to an integer: + # + # Integer(1.9) # => 1 # Rounds toward zero. + # Integer(-1.9) # => -1 # Rounds toward zero. + # + # ==== string objects + # + # With string argument +object+ and zero +base+ given, + # returns +object+ converted to an integer in base 10: + # + # Integer('100') # => 100 + # Integer('-100') # => -100 + # + # With +base+ zero, string +object+ may contain leading characters + # to specify the actual base (radix indicator): + # + # Integer('0100') # => 64 # Leading '0' specifies base 8. + # Integer('0b100') # => 4 # Leading '0b', specifies base 2. + # Integer('0x100') # => 256 # Leading '0x' specifies base 16. + # + # With a positive +base+ (in range 2..36) given, returns +object+ + # converted to an integer in the given base: + # + # Integer('100', 2) # => 4 + # Integer('100', 8) # => 64 + # Integer('-100', 16) # => -256 + # + # With a negative +base+ (in range -36..-2) given, returns +object+ + # converted to an integer in the radix indicator if exists or + # +-base+: + # + # Integer('0x100', -2) # => 256 + # Integer('100', -2) # => 4 + # Integer('0b100', -8) # => 4 + # Integer('100', -8) # => 64 + # Integer('0o100', -10) # => 64 + # Integer('100', -10) # => 100 + # + # +base+ -1 is equal the -10 case. + # + # When converting strings, surrounding whitespace and embedded underscores + # are allowed and ignored: + # + # Integer(' 100 ') # => 100 + # Integer('-1_0_0', 16) # => -256 + # + # ==== other classes + # + # Examples with +object+ of various other classes: + # + # Integer(Rational(9, 10)) # => 0 # Rounds toward zero. + # Integer(Complex(2, 0)) # => 2 # Imaginary part must be zero. + # Integer(Time.now) # => 1650974042 + # + # ==== keywords + # + # With optional keyword argument +exception+ given as +true+ (the default): + # + # - Raises TypeError if +object+ does not respond to +to_int+ or +to_i+. + # - Raises TypeError if +object+ is +nil+. + # - Raise ArgumentError if +object+ is an invalid string. + # + # With +exception+ given as +false+, an exception of any kind is suppressed + # and +nil+ is returned. + + def Integer(arg, base = 0, exception: true) + if Primitive.mandatory_only? + Primitive.rb_f_integer1(arg) + else + Primitive.rb_f_integer(arg, base, exception); + end + end end diff --git a/object.c b/object.c index 70cb64eb850d17..c32cb82dd991e1 100644 --- a/object.c +++ b/object.c @@ -3312,121 +3312,17 @@ rb_opts_exception_p(VALUE opts, int default_value) return default_value; } -#define opts_exception_p(opts) rb_opts_exception_p((opts), TRUE) - -/* - * call-seq: - * Integer(object, base = 0, exception: true) -> integer or nil - * - * Returns an integer converted from +object+. - * - * Tries to convert +object+ to an integer - * using +to_int+ first and +to_i+ second; - * see below for exceptions. - * - * With a non-zero +base+, +object+ must be a string or convertible - * to a string. - * - * ==== numeric objects - * - * With integer argument +object+ given, returns +object+: - * - * Integer(1) # => 1 - * Integer(-1) # => -1 - * - * With floating-point argument +object+ given, - * returns +object+ truncated to an integer: - * - * Integer(1.9) # => 1 # Rounds toward zero. - * Integer(-1.9) # => -1 # Rounds toward zero. - * - * ==== string objects - * - * With string argument +object+ and zero +base+ given, - * returns +object+ converted to an integer in base 10: - * - * Integer('100') # => 100 - * Integer('-100') # => -100 - * - * With +base+ zero, string +object+ may contain leading characters - * to specify the actual base (radix indicator): - * - * Integer('0100') # => 64 # Leading '0' specifies base 8. - * Integer('0b100') # => 4 # Leading '0b', specifies base 2. - * Integer('0x100') # => 256 # Leading '0x' specifies base 16. - * - * With a positive +base+ (in range 2..36) given, returns +object+ - * converted to an integer in the given base: - * - * Integer('100', 2) # => 4 - * Integer('100', 8) # => 64 - * Integer('-100', 16) # => -256 - * - * With a negative +base+ (in range -36..-2) given, returns +object+ - * converted to an integer in the radix indicator if exists or - * +-base+: - * - * Integer('0x100', -2) # => 256 - * Integer('100', -2) # => 4 - * Integer('0b100', -8) # => 4 - * Integer('100', -8) # => 64 - * Integer('0o100', -10) # => 64 - * Integer('100', -10) # => 100 - * - * +base+ -1 is equal the -10 case. - * - * When converting strings, surrounding whitespace and embedded underscores - * are allowed and ignored: - * - * Integer(' 100 ') # => 100 - * Integer('-1_0_0', 16) # => -256 - * - * ==== other classes - * - * Examples with +object+ of various other classes: - * - * Integer(Rational(9, 10)) # => 0 # Rounds toward zero. - * Integer(Complex(2, 0)) # => 2 # Imaginary part must be zero. - * Integer(Time.now) # => 1650974042 - * - * ==== keywords - * - * With optional keyword argument +exception+ given as +true+ (the default): - * - * - Raises TypeError if +object+ does not respond to +to_int+ or +to_i+. - * - Raises TypeError if +object+ is +nil+. - * - Raise ArgumentError if +object+ is an invalid string. - * - * With +exception+ given as +false+, an exception of any kind is suppressed - * and +nil+ is returned. - * - */ - static VALUE -rb_f_integer(int argc, VALUE *argv, VALUE obj) +rb_f_integer1(rb_execution_context_t *ec, VALUE obj, VALUE arg) { - VALUE arg = Qnil, opts = Qnil; - int base = 0; - - if (argc > 1) { - int narg = 1; - VALUE vbase = rb_check_to_int(argv[1]); - if (!NIL_P(vbase)) { - base = NUM2INT(vbase); - narg = 2; - } - if (argc > narg) { - VALUE hash = rb_check_hash_type(argv[argc-1]); - if (!NIL_P(hash)) { - opts = rb_extract_keywords(&hash); - if (!hash) --argc; - } - } - } - rb_check_arity(argc, 1, 2); - arg = argv[0]; + return rb_convert_to_integer(arg, 0, TRUE); +} - return rb_convert_to_integer(arg, base, opts_exception_p(opts)); +static VALUE +rb_f_integer(rb_execution_context_t *ec, VALUE obj, VALUE arg, VALUE base, VALUE exception) +{ + int exc = rb_bool_expected(exception, "exception", TRUE); + return rb_convert_to_integer(arg, NUM2INT(base), exc); } static double @@ -4475,8 +4371,6 @@ InitVM_Object(void) rb_define_global_function("sprintf", f_sprintf, -1); rb_define_global_function("format", f_sprintf, -1); - rb_define_global_function("Integer", rb_f_integer, -1); - rb_define_global_function("String", rb_f_string, 1); rb_define_global_function("Array", rb_f_array, 1); rb_define_global_function("Hash", rb_f_hash, 1); diff --git a/test/ruby/test_integer.rb b/test/ruby/test_integer.rb index 3f35c3bfbfd550..3e8d68702c0271 100644 --- a/test/ruby/test_integer.rb +++ b/test/ruby/test_integer.rb @@ -251,6 +251,17 @@ def test_Integer_with_base assert_raise(ArgumentError) { Integer(Object.new, 1) } assert_raise(ArgumentError) { Integer(1, 1, 1) } + + def (base = Object.new).to_int + 8 + end + assert_equal(8, Integer("10", base)) + + assert_raise(TypeError) { Integer("10", "8") } + def (base = Object.new).to_int + "8" + end + assert_raise(TypeError) { Integer("10", base) } end def test_int_p From eb3d94f4baff70d2e120c9472a3851a4aa9c90d9 Mon Sep 17 00:00:00 2001 From: Burdette Lamar Date: Thu, 31 Aug 2023 08:06:11 -0500 Subject: [PATCH 178/208] [DOC] RDoc for Kernel#system (#8309) --- process.c | 148 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 112 insertions(+), 36 deletions(-) diff --git a/process.c b/process.c index 377da3829a814c..0161ddb455f5d6 100644 --- a/process.c +++ b/process.c @@ -3068,7 +3068,7 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * * CONTRIBUTING.md COPYING COPYING.ja * - * Raises an exception if the new process fails to execute. + * Raises an exception if the new process could not execute. * * Argument +exe_path+ * @@ -3101,7 +3101,7 @@ NORETURN(static VALUE f_exec(int c, const VALUE *a, VALUE _)); * C* * hello world * - * Raises an exception if the new process fails to execute. + * Raises an exception if the new process could not execute. */ static VALUE @@ -4679,56 +4679,132 @@ rb_spawn(int argc, const VALUE *argv) /* * call-seq: - * system([env,] command... [,options], exception: false) -> true, false or nil + * system([env, ] command_line, options = {}, exception: false) -> true, false, or nil + * system([env, ] exe_path, *args, options = {}, exception: false) -> true, false, or nil * - * Executes _command..._ in a subshell. - * _command..._ is one of following forms. + * Creates a new child process by doing one of the following + * in that process: + * + * - Passing string +command_line+ to the shell. + * - Invoking the executable at +exe_path+. * * This method has potential security vulnerabilities if called with untrusted input; * see {Command Injection}[rdoc-ref:command_injection.rdoc]. * - * [commandline] - * command line string which is passed to the standard shell - * [cmdname, arg1, ...] - * command name and one or more arguments (no shell) - * [[cmdname, argv0], arg1, ...] - * command name, argv[0] and zero or more arguments (no shell) + * Returns: * - * system returns +true+ if the command gives zero exit status, - * +false+ for non zero exit status. - * Returns +nil+ if command execution fails. - * An error status is available in $?. + * - +true+ if the command exits with status zero. + * - +false+ if the exit status is a non-zero integer. + * - +nil+ if the command could not execute. * - * If the exception: true argument is passed, the method - * raises an exception instead of returning +false+ or +nil+. + * Raises an exception (instead of returning +false+ or +nil+) + * if keyword argument +exception+ is set to +true+. * - * The arguments are processed in the same way as - * for Kernel#spawn. + * Assigns the command's error status to $?. * - * The hash arguments, env and options, are same as #exec and #spawn. - * See Kernel#spawn for details. + * The new process is created using the + * {system system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/system.html]; + * it may inherit some of its environment from the calling program + * (possibly including open file descriptors). * - * system("echo *") - * system("echo", "*") + * Argument +env+, if given, is a hash that affects +ENV+ for the new process; + * see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. * - * produces: + * Argument +options+ is a hash of options for the new process; + * see {Execution Options}[rdoc-ref:Process@Execution+Options]. + * + * The first required argument is one of the following: + * + * - +command_line+ if it is a string, + * and if it begins with a shell reserved word or special built-in, + * or if it contains one or more metacharacters. + * - +exe_path+ otherwise. + * + * Argument +command_line+ + * + * \String argument +command_line+ is a command line to be passed to a shell; + * it must begin with a shell reserved word, begin with a special built-in, + * or contain meta characters: + * + * system('echo') # => true # Built-in. + * system('if true; then echo "Foo"; fi') # => true # Shell reserved word. + * system('date > /tmp/date.tmp') # => true # Contains meta character. + * system('date > /nop/date.tmp') # => false + * system('date > /nop/date.tmp', exception: true) # Raises RuntimeError. + * + * Assigns the command's error status to $?: + * + * system('echo') # => true # Built-in. + * $? # => # + * system('date > /nop/date.tmp') # => false + * $? # => # + * + * The command line may also contain arguments and options for the command: + * + * system('echo "Foo"') # => true + * + * Output: + * + * Foo + * + * On a Unix-like system, the shell is /bin/sh; + * otherwise the shell is determined by environment variable + * ENV['RUBYSHELL'], if defined, or ENV['COMSPEC'] otherwise. + * + * Except for the +COMSPEC+ case, + * the entire string +command_line+ is passed as an argument + * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. * - * config.h main.rb - * * + * The shell performs normal shell expansion on the command line: + * + * system('echo C*') # => true * - * Error handling: + * Output: + * + * CONTRIBUTING.md COPYING COPYING.ja * - * system("cat nonexistent.txt") - * # => false - * system("catt nonexistent.txt") - * # => nil + * Raises an exception if the new process could not execute. * - * system("cat nonexistent.txt", exception: true) - * # RuntimeError (Command failed with exit 1: cat) - * system("catt nonexistent.txt", exception: true) - * # Errno::ENOENT (No such file or directory - catt) + * Argument +exe_path+ + * + * Argument +exe_path+ is one of the following: + * + * - The string path to an executable to be called. + * - A 2-element array containing the path to an executable + * and the string to be used as the name of the executing process. + * + * Example: + * + * system('/usr/bin/date') # => true # Path to date on Unix-style system. + * system('foo') # => nil # Command failed. + * + * Output: + * + * Mon Aug 28 11:43:10 AM CDT 2023 + * + * Assigns the command's error status to $?: + * + * system('/usr/bin/date') # => true + * $? # => # + * system('foo') # => nil + * $? # => # + * + * Ruby invokes the executable directly, with no shell and no shell expansion: + * + * system('doesnt_exist') # => nil + * + * If one or more +args+ is given, each is an argument or option + * to be passed to the executable: + * + * system('echo', 'C*') # => true + * system('echo', 'hello', 'world') # => true + * + * Output: + * + * C* + * hello world * - * See Kernel#exec for the standard shell. + * Raises an exception if the new process could not execute. */ static VALUE From 4f0d58260a7f506ead198064d12a967edd11fe5e Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 30 Aug 2023 15:28:22 -0400 Subject: [PATCH 179/208] Correctly calculate initial pages The old algorithm could calculate an undercount for the initial pages due to two issues: 1. It did not take into account that some heap pages will have one less slot due to alignment. It assumed that every heap page would be able to be fully filled with slots. Pages that are unaligned with the slot size will lose one slot. The new algorithm assumes that every page will be unaligned. 2. It performed integer division, which truncates down. This means that the number of pages might not actually satisfy the number of slots. This can cause the heap to grow in `gc_sweep_finish_size_pool` after allocating all of the allocatable pages because the total number of slots would be less than the initial configured number of slots. --- gc.c | 30 ++++++++++++++++++++--------- test/ruby/test_gc.rb | 46 ++++++++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/gc.c b/gc.c index e2c2779b47e932..59c507369e2216 100644 --- a/gc.c +++ b/gc.c @@ -2305,11 +2305,22 @@ heap_add_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *he #endif static size_t -minimum_pages_for_size_pool(rb_objspace_t *objspace, int size_pool_idx) +slots_to_pages_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t slots) { - rb_size_pool_t *size_pool = &size_pools[size_pool_idx]; - int multiple = size_pool->slot_size / BASE_SLOT_SIZE; - return gc_params.size_pool_init_slots[size_pool_idx] * multiple / HEAP_PAGE_OBJ_LIMIT; + size_t multiple = size_pool->slot_size / BASE_SLOT_SIZE; + /* Due to alignment, heap pages may have one less slot. We should + * ensure there is enough pages to guarantee that we will have at + * least the required number of slots after allocating all the pages. */ + size_t slots_per_page = (HEAP_PAGE_OBJ_LIMIT / multiple) - 1; + return CEILDIV(slots, slots_per_page); +} + +static size_t +minimum_pages_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) +{ + size_t size_pool_idx = size_pool - size_pools; + size_t init_slots = gc_params.size_pool_init_slots[size_pool_idx]; + return slots_to_pages_for_size_pool(objspace, size_pool, init_slots); } static size_t @@ -2322,7 +2333,7 @@ heap_extend_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool, size_t fre next_used = (size_t)(used * gc_params.growth_factor); } else if (total_slots == 0) { - next_used = minimum_pages_for_size_pool(objspace, (int)(size_pool - size_pools)); + next_used = minimum_pages_for_size_pool(objspace, size_pool); } else { /* Find `f' where free_slots = f * total_slots * goal_ratio @@ -3728,10 +3739,12 @@ Init_heap(void) /* Set size pools allocatable pages. */ for (int i = 0; i < SIZE_POOL_COUNT; i++) { + rb_size_pool_t *size_pool = &size_pools[i]; + /* Set the default value of size_pool_init_slots. */ gc_params.size_pool_init_slots[i] = GC_HEAP_INIT_SLOTS; - size_pools[i].allocatable_pages = minimum_pages_for_size_pool(objspace, i); + size_pool->allocatable_pages = minimum_pages_for_size_pool(objspace, size_pool); } heap_pages_expand_sorted(objspace); @@ -10829,7 +10842,7 @@ gc_verify_compaction_references(rb_execution_context_t *ec, VALUE self, VALUE do size_t minimum_pages = 0; if (RTEST(expand_heap)) { - minimum_pages = minimum_pages_for_size_pool(objspace, i); + minimum_pages = minimum_pages_for_size_pool(objspace, size_pool); } heap_add_pages(objspace, size_pool, heap, MAX(minimum_pages, heap->total_pages)); @@ -11643,8 +11656,7 @@ gc_set_initial_pages(rb_objspace_t *objspace) if (size_pool_init_slots > size_pool->eden_heap.total_slots) { size_t slots = size_pool_init_slots - size_pool->eden_heap.total_slots; - int multiple = size_pool->slot_size / BASE_SLOT_SIZE; - size_pool->allocatable_pages = slots * multiple / HEAP_PAGE_OBJ_LIMIT; + size_pool->allocatable_pages = slots_to_pages_for_size_pool(objspace, size_pool, slots); } else { /* We already have more slots than size_pool_init_slots allows, so diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 3967ed54bc676c..0b4062e99f946c 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -444,12 +444,12 @@ def test_gc_parameter_init_slots # Constant from gc.c. GC_HEAP_INIT_SLOTS = 10_000 GC.stat_heap.each do |_, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - # Give a 0.9x delta because integer division in minimum_pages_for_size_pool can sometimes cause number to be - # less than GC_HEAP_INIT_SLOTS. - assert_operator(total_slots, :>=, GC_HEAP_INIT_SLOTS * 0.9, s) + assert_operator(total_slots, :>=, GC_HEAP_INIT_SLOTS, s) end RUBY @@ -462,10 +462,16 @@ def test_gc_parameter_init_slots assert_separately([env, "-W0"], __FILE__, __LINE__, <<~RUBY) SIZES = #{sizes} GC.stat_heap.each do |i, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - assert_in_epsilon(SIZES[i], total_slots, 0.01, s) + + # The delta is calculated as follows: + # - For allocated pages, each page can vary by 1 slot due to alignment. + # - For allocatable pages, we can end up with at most 1 extra page of slots. + assert_in_delta(SIZES[i], total_slots, s[:heap_eden_pages] + slots_per_page, s) end RUBY @@ -486,10 +492,16 @@ def test_gc_parameter_init_slots # Check that we still have the same number of slots as initially configured. GC.stat_heap.each do |i, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - assert_in_epsilon(SIZES[i], total_slots, 0.01, s) + + # The delta is calculated as follows: + # - For allocated pages, each page can vary by 1 slot due to alignment. + # - For allocatable pages, we can end up with at most 1 extra page of slots. + assert_in_delta(SIZES[i], total_slots, s[:heap_eden_pages] + slots_per_page, s) end RUBY @@ -525,10 +537,16 @@ def test_gc_parameter_init_slots # Check that we still have the same number of slots as initially configured. GC.stat_heap.each do |i, s| - # Sometimes pages will have 1 less slot due to alignment, so always increase slots_per_page by 1. - slots_per_page = (s[:heap_eden_slots] / s[:heap_eden_pages]) + 1 + multiple = s[:slot_size] / (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + # Allocatable pages are assumed to have lost 1 slot due to alignment. + slots_per_page = (GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] / multiple) - 1 + total_slots = s[:heap_eden_slots] + s[:heap_allocatable_pages] * slots_per_page - assert_in_epsilon(SIZES[i], total_slots, 0.01, s) + + # The delta is calculated as follows: + # - For allocated pages, each page can vary by 1 slot due to alignment. + # - For allocatable pages, we can end up with at most 1 extra page of slots. + assert_in_delta(SIZES[i], total_slots, s[:heap_eden_pages] + slots_per_page, s) end RUBY end From 8804a70387bbff1f1497feef0fa1cfa37df1845b Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Thu, 31 Aug 2023 15:08:03 +0100 Subject: [PATCH 180/208] [ruby/irb] Require Reline 0.3.8+ (https://github.com/ruby/irb/pull/702) Reline 0.3.8 reduces the chance of having a deadlock with the debugger while using the new `irb:rdbg` integration. https://github.com/ruby/irb/commit/9b8c56b7d4 --- lib/irb/irb.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irb/irb.gemspec b/lib/irb/irb.gemspec index 4d47481d8794ec..50f6a22209aa7a 100644 --- a/lib/irb/irb.gemspec +++ b/lib/irb/irb.gemspec @@ -41,6 +41,6 @@ Gem::Specification.new do |spec| spec.required_ruby_version = Gem::Requirement.new(">= 2.7") - spec.add_dependency "reline", ">= 0.3.6" + spec.add_dependency "reline", ">= 0.3.8" spec.add_dependency "rdoc", "~> 6.5" end From b8e782c1b5d0c13c97160d1bae5bdc95c0fca1f8 Mon Sep 17 00:00:00 2001 From: Burdette Lamar Date: Thu, 31 Aug 2023 09:27:10 -0500 Subject: [PATCH 181/208] [DOC] Link fix (#8340) --- file.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/file.c b/file.c index 30dd1cc73b164a..4a5c8a8f8aa965 100644 --- a/file.c +++ b/file.c @@ -7436,7 +7436,7 @@ Init_File(void) * There are two families of constants here: * * - Those having to do with {file access}[rdoc-ref:File::Constants@File+Access]. - * - Those having to do with {file globbing}[rdoc-ref:File::Constants@File+Globbing]. + * - Those having to do with {filename globbing}[rdoc-ref:File::Constants@Filename+Globbing+Constants+-28File-3A-3AFNM_-2A-29]. * * \File constants defined for the local process may be retrieved * with method File::Constants.constants: From 0270210e4984957427a4cf3824b724b62bfa2eaf Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 31 Aug 2023 09:23:30 +0200 Subject: [PATCH 182/208] TestSocket_UNIXSocket: stop testing empty packets OpenBSD and Solaris behave differently here. Linux does deliver the empty packet, which is questionable as it's undistinguishable from a closed connection. It seems that OpenBSD and Solaris simply drop it. We could test the platform before doing the assertion, but it would likely be fragile, and the entire web recommend to not ever send an empty packet, so the value of this assertion is low. --- test/socket/test_unix.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/socket/test_unix.rb b/test/socket/test_unix.rb index 9d9faa4387544e..b2da1e439dd019 100644 --- a/test/socket/test_unix.rb +++ b/test/socket/test_unix.rb @@ -488,9 +488,7 @@ def test_seqpacket_pair assert_kind_of(IO::WaitReadable, e) end - s2.send("", 0) s2.send("haha", 0) - assert_equal(nil, s1.recv(10)) # no way to distinguish empty packet from EOF with SOCK_SEQPACKET assert_equal("haha", s1.recv(10)) assert_raise(IO::EWOULDBLOCKWaitReadable) { s1.recv_nonblock(10) } From 43825fba6eb39980130fd9bf47acb2000cde55d3 Mon Sep 17 00:00:00 2001 From: John Hawthorn Date: Fri, 25 Aug 2023 18:52:02 -0700 Subject: [PATCH 183/208] YJIT: Handle getblockparamproxy with ifunc getblockparamproxy for "ifunc" behaves identically to iseq, in just pushing rb_block_param_proxy. --- test/ruby/test_yjit.rb | 21 +++++++++++++++++++-- yjit/src/codegen.rs | 24 ++++++++++++++---------- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/test/ruby/test_yjit.rb b/test/ruby/test_yjit.rb index 5347028550dfca..cc9507aff490e4 100644 --- a/test/ruby/test_yjit.rb +++ b/test/ruby/test_yjit.rb @@ -547,8 +547,7 @@ def foo &blk end def test_getblockparamproxy - # Currently two side exits as OPTIMIZED_METHOD_TYPE_CALL is unimplemented - assert_compiles(<<~'RUBY', insns: [:getblockparamproxy]) + assert_compiles(<<~'RUBY', insns: [:getblockparamproxy], exits: {}) def foo &blk p blk.call p blk.call @@ -559,6 +558,24 @@ def foo &blk RUBY end + def test_ifunc_getblockparamproxy + assert_compiles(<<~'RUBY', insns: [:getblockparamproxy], exits: {}) + class Foo + include Enumerable + + def each(&block) + block.call 1 + block.call 2 + block.call 3 + end + end + + foo = Foo.new + foo.map { _1 * 2 } + foo.map { _1 * 2 } + RUBY + end + def test_send_blockarg assert_compiles(<<~'RUBY', insns: [:getblockparamproxy, :send], exits: {}) def bar diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index c506b4360c10a6..b51704b2c606f0 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -8151,9 +8151,10 @@ fn gen_getblockparamproxy( // Peek at the block handler so we can check whether it's nil let comptime_handler = jit.peek_at_block_handler(level); - // Filter for the 3 cases we currently handle + // Filter for the 4 cases we currently handle if !(comptime_handler.as_u64() == 0 || // no block given comptime_handler.as_u64() & 0x3 == 0x1 || // iseq block (no associated GC managed object) + comptime_handler.as_u64() & 0x3 == 0x3 || // ifunc block (no associated GC managed object) unsafe { rb_obj_is_proc(comptime_handler) }.test() // block is a Proc ) { // Missing the symbol case, where we basically need to call Symbol#to_proc at runtime @@ -8181,7 +8182,7 @@ fn gen_getblockparamproxy( // Use block handler sample to guide specialization... // NOTE: we use jit_chain_guard() in this decision tree, and since - // there are only 3 cases, it should never reach the depth limit use + // there are only a few cases, it should never reach the depth limit use // the exit counter we pass to it. // // No block given @@ -8199,15 +8200,18 @@ fn gen_getblockparamproxy( ); jit_putobject(asm, Qnil); - } else if comptime_handler.as_u64() & 0x3 == 0x1 { - // Block handler is a tagged pointer. Look at the tag. 0x03 is from VM_BH_ISEQ_BLOCK_P(). - let block_handler = asm.and(block_handler, 0x3.into()); - - // Bail unless VM_BH_ISEQ_BLOCK_P(bh). This also checks for null. - asm.cmp(block_handler, 0x1.into()); - + } else if comptime_handler.as_u64() & 0x1 == 0x1 { + // This handles two cases which are nearly identical + // Block handler is a tagged pointer. Look at the tag. + // VM_BH_ISEQ_BLOCK_P(): block_handler & 0x03 == 0x01 + // VM_BH_IFUNC_P(): block_handler & 0x03 == 0x03 + // So to check for either of those cases we can use: val & 0x1 == 0x1 + const _: () = assert!(RUBY_SYMBOL_FLAG & 1 == 0, "guard below rejects symbol block handlers"); + // Procs are aligned heap pointers so testing the bit rejects them too. + + asm.test(block_handler, 0x1.into()); jit_chain_guard( - JCC_JNZ, + JCC_JZ, jit, asm, ocb, From 3678734fac7a4d3d7e160200153090dcba6e3d94 Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Thu, 31 Aug 2023 16:44:13 +0100 Subject: [PATCH 184/208] [ruby/irb] Drop rdoc's version requirement (https://github.com/ruby/irb/pull/704) 1. The newer versions of rdoc requires pysch 4.0+, which could break apps using Ruby 3.0 or 2.7. #703 has more detailed explanation on this. 2. We actually don't use any version-specific rdoc APIs. So having a version requirement is not necessary atm. https://github.com/ruby/irb/commit/3e6ba78c42 --- lib/irb/irb.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irb/irb.gemspec b/lib/irb/irb.gemspec index 50f6a22209aa7a..a008a39f9d5d81 100644 --- a/lib/irb/irb.gemspec +++ b/lib/irb/irb.gemspec @@ -42,5 +42,5 @@ Gem::Specification.new do |spec| spec.required_ruby_version = Gem::Requirement.new(">= 2.7") spec.add_dependency "reline", ">= 0.3.8" - spec.add_dependency "rdoc", "~> 6.5" + spec.add_dependency "rdoc" end From b90457b210e6b1dc465d82dd08db126ce1a8228e Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Thu, 31 Aug 2023 12:56:13 -0400 Subject: [PATCH 185/208] [YARP] Implement SourceNodes (File, Line, Encoding) (#8328) * [YARP] Implement SourceNodes (File, Line, Encoding) --- yarp/yarp_compiler.c | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index 57638a36fb5f93..bd1352ebd701bb 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -1272,6 +1272,39 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, return; } + case YP_NODE_SOURCE_ENCODING_NODE: { + const char *encoding = compile_context->parser->encoding.name; + if (!popped) { + rb_encoding *enc = rb_find_encoding(rb_str_new_cstr(encoding)); + if (!enc) { + rb_bug("Encoding not found!"); + } + ADD_INSN1(ret, &dummy_line_node, putobject, rb_enc_from_encoding(enc)); + } + return; + } + case YP_NODE_SOURCE_FILE_NODE: { + yp_source_file_node_t *source_file_node = (yp_source_file_node_t *)node; + + if (!popped) { + VALUE filepath; + if (source_file_node->filepath.length == 0) { + filepath = rb_fstring_lit(""); + } + else { + filepath = parse_string(&source_file_node->filepath); + } + + ADD_INSN1(ret, &dummy_line_node, putstring, filepath); + } + return; + } + case YP_NODE_SOURCE_LINE_NODE: { + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, putobject, INT2FIX(lineno)); + } + return; + } case YP_NODE_SPLAT_NODE: { yp_splat_node_t *splat_node = (yp_splat_node_t *)node; yp_compile_node(iseq, splat_node->expression, ret, src, popped, compile_context); From ace41c556a72392a0244d78bca35718f188cf004 Mon Sep 17 00:00:00 2001 From: Mau Magnaguagno Date: Thu, 31 Aug 2023 14:00:19 -0300 Subject: [PATCH 186/208] [YARP] Avoid if-else in yp_compile_node (#8336) Move last node case outside for loop. --- yarp/yarp_compiler.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index bd1352ebd701bb..f8fc004ce7afd8 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -1319,14 +1319,10 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, case YP_NODE_STATEMENTS_NODE: { yp_statements_node_t *statements_node = (yp_statements_node_t *) node; yp_node_list_t node_list = statements_node->body; - for (size_t index = 0; index < node_list.size; index++) { - if (!popped && (index != node_list.size - 1)) { - yp_compile_node(iseq, node_list.nodes[index], ret, src, true, compile_context); - } - else { - yp_compile_node(iseq, node_list.nodes[index], ret, src, false, compile_context); - } + for (size_t index = 0; index < node_list.size - 1; index++) { + yp_compile_node(iseq, node_list.nodes[index], ret, src, !popped, compile_context); } + yp_compile_node(iseq, node_list.nodes[node_list.size - 1], ret, src, false, compile_context); return; } case YP_NODE_STRING_CONCAT_NODE: { From 8470acc1ec6c06c62d857d5a01f6f9438f27fa10 Mon Sep 17 00:00:00 2001 From: Nathan Froyd Date: Thu, 31 Aug 2023 11:04:25 -0400 Subject: [PATCH 187/208] [ruby/yarp] add some `const` qualifiers to local variables https://github.com/ruby/yarp/commit/eb3c6eb928 --- yarp/yarp.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yarp/yarp.c b/yarp/yarp.c index 7df7ca0653c47e..376dfe32c3ce91 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -9890,7 +9890,7 @@ parse_heredoc_common_whitespace(yp_parser_t *parser, yp_node_list_t *nodes) { yp_node_t *node = nodes->nodes[index]; if (!YP_NODE_TYPE_P(node, YP_NODE_STRING_NODE)) continue; - yp_location_t *content_loc = &((yp_string_node_t *) node)->content_loc; + const yp_location_t *content_loc = &((yp_string_node_t *) node)->content_loc; // If the previous node wasn't a string node, we don't want to trim // whitespace. This could happen after an interpolated expression or @@ -10235,7 +10235,7 @@ parse_pattern_hash(yp_parser_t *parser, yp_node_t *first_assoc) { yp_node_t *key = ((yp_assoc_node_t *) first_assoc)->key; if (YP_NODE_TYPE_P(key, YP_NODE_SYMBOL_NODE)) { - yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; + const yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; yp_parser_local_add_location(parser, value_loc->start, value_loc->end); } } @@ -10263,7 +10263,7 @@ parse_pattern_hash(yp_parser_t *parser, yp_node_t *first_assoc) { if (!match_any_type_p(parser, 7, YP_TOKEN_COMMA, YP_TOKEN_KEYWORD_THEN, YP_TOKEN_BRACE_RIGHT, YP_TOKEN_BRACKET_RIGHT, YP_TOKEN_PARENTHESIS_RIGHT, YP_TOKEN_NEWLINE, YP_TOKEN_SEMICOLON)) { value = parse_pattern(parser, false, "Expected a pattern expression after the key."); } else { - yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; + const yp_location_t *value_loc = &((yp_symbol_node_t *) key)->value_loc; yp_parser_local_add_location(parser, value_loc->start, value_loc->end); } @@ -13215,7 +13215,7 @@ parse_expression_infix(yp_parser_t *parser, yp_node_t *node, yp_binding_power_t yp_string_list_t named_captures; yp_string_list_init(&named_captures); - yp_location_t *content_loc = &((yp_regular_expression_node_t *) node)->content_loc; + const yp_location_t *content_loc = &((yp_regular_expression_node_t *) node)->content_loc; if (yp_regexp_named_capture_group_names(content_loc->start, (size_t) (content_loc->end - content_loc->start), &named_captures, parser->encoding_changed, &parser->encoding)) { for (size_t index = 0; index < named_captures.length; index++) { From 84fa8ae84eeb20a35ff2040dbf107c6d9babfdec Mon Sep 17 00:00:00 2001 From: Burdette Lamar Date: Thu, 31 Aug 2023 13:02:09 -0500 Subject: [PATCH 188/208] [DOC] RDoc for #spawn (#8342) --- process.c | 327 +++++++++++++++--------------------------------------- 1 file changed, 88 insertions(+), 239 deletions(-) diff --git a/process.c b/process.c index 0161ddb455f5d6..47db4fa6265341 100644 --- a/process.c +++ b/process.c @@ -4872,271 +4872,120 @@ rb_f_system(int argc, VALUE *argv, VALUE _) /* * call-seq: - * spawn([env,] command... [,options]) -> pid - * Process.spawn([env,] command... [,options]) -> pid - * - * spawn executes specified command and return its pid. - * - * pid = spawn("tar xf ruby-2.0.0-p195.tar.bz2") - * Process.wait pid - * - * pid = spawn(RbConfig.ruby, "-eputs'Hello, world!'") - * Process.wait pid - * - * This method is similar to Kernel#system but it doesn't wait for the command - * to finish. - * - * The parent process should - * use Process.wait to collect - * the termination status of its child or - * use Process.detach to register - * disinterest in their status; - * otherwise, the operating system may accumulate zombie processes. - * - * spawn has bunch of options to specify process attributes: - * - * env: hash - * name => val : set the environment variable - * name => nil : unset the environment variable - * - * the keys and the values except for +nil+ must be strings. - * command...: - * commandline : command line string which is passed to the standard shell - * cmdname, arg1, ... : command name and one or more arguments (This form does not use the shell. See below for caveats.) - * [cmdname, argv0], arg1, ... : command name, argv[0] and zero or more arguments (no shell) - * options: hash - * clearing environment variables: - * :unsetenv_others => true : clear environment variables except specified by env - * :unsetenv_others => false : don't clear (default) - * process group: - * :pgroup => true or 0 : make a new process group - * :pgroup => pgid : join the specified process group - * :pgroup => nil : don't change the process group (default) - * create new process group: Windows only - * :new_pgroup => true : the new process is the root process of a new process group - * :new_pgroup => false : don't create a new process group (default) - * resource limit: resourcename is core, cpu, data, etc. See Process.setrlimit. - * :rlimit_resourcename => limit - * :rlimit_resourcename => [cur_limit, max_limit] - * umask: - * :umask => int - * redirection: - * key: - * FD : single file descriptor in child process - * [FD, FD, ...] : multiple file descriptor in child process - * value: - * FD : redirect to the file descriptor in parent process - * string : redirect to file with open(string, "r" or "w") - * [string] : redirect to file with open(string, File::RDONLY) - * [string, open_mode] : redirect to file with open(string, open_mode, 0644) - * [string, open_mode, perm] : redirect to file with open(string, open_mode, perm) - * [:child, FD] : redirect to the redirected file descriptor - * :close : close the file descriptor in child process - * FD is one of follows - * :in : the file descriptor 0 which is the standard input - * :out : the file descriptor 1 which is the standard output - * :err : the file descriptor 2 which is the standard error - * integer : the file descriptor of specified the integer - * io : the file descriptor specified as io.fileno - * file descriptor inheritance: close non-redirected non-standard fds (3, 4, 5, ...) or not - * :close_others => false : inherit - * current directory: - * :chdir => str - * - * The cmdname, arg1, ... form does not use the shell. - * However, on different OSes, different things are provided as - * built-in commands. An example of this is +'echo'+, which is a - * built-in on Windows, but is a normal program on Linux and Mac OS X. - * This means that Process.spawn 'echo', '%Path%' will - * display the contents of the %Path% environment variable - * on Windows, but Process.spawn 'echo', '$PATH' prints - * the literal $PATH. - * - * If a hash is given as +env+, the environment is - * updated by +env+ before exec(2) in the child process. - * If a pair in +env+ has nil as the value, the variable is deleted. - * - * # set FOO as BAR and unset BAZ. - * pid = spawn({"FOO"=>"BAR", "BAZ"=>nil}, command) - * - * If a hash is given as +options+, - * it specifies - * process group, - * create new process group, - * resource limit, - * current directory, - * umask and - * redirects for the child process. - * Also, it can be specified to clear environment variables. - * - * The :unsetenv_others key in +options+ specifies - * to clear environment variables, other than specified by +env+. - * - * pid = spawn(command, :unsetenv_others=>true) # no environment variable - * pid = spawn({"FOO"=>"BAR"}, command, :unsetenv_others=>true) # FOO only - * - * The :pgroup key in +options+ specifies a process group. - * The corresponding value should be true, zero, a positive integer, or nil. - * true and zero cause the process to be a process leader of a new process group. - * A non-zero positive integer causes the process to join the provided process group. - * The default value, nil, causes the process to remain in the same process group. - * - * pid = spawn(command, :pgroup=>true) # process leader - * pid = spawn(command, :pgroup=>10) # belongs to the process group 10 - * - * The :new_pgroup key in +options+ specifies to pass - * +CREATE_NEW_PROCESS_GROUP+ flag to CreateProcessW() that is - * Windows API. This option is only for Windows. - * true means the new process is the root process of the new process group. - * The new process has CTRL+C disabled. This flag is necessary for - * Process.kill(:SIGINT, pid) on the subprocess. - * :new_pgroup is false by default. - * - * pid = spawn(command, :new_pgroup=>true) # new process group - * pid = spawn(command, :new_pgroup=>false) # same process group - * - * The :rlimit_foo key specifies a resource limit. - * foo should be one of resource types such as core. - * The corresponding value should be an integer or an array which have one or - * two integers: same as cur_limit and max_limit arguments for - * Process.setrlimit. - * - * cur, max = Process.getrlimit(:CORE) - * pid = spawn(command, :rlimit_core=>[0,max]) # disable core temporary. - * pid = spawn(command, :rlimit_core=>max) # enable core dump - * pid = spawn(command, :rlimit_core=>0) # never dump core. - * - * The :umask key in +options+ specifies the umask. - * - * pid = spawn(command, :umask=>077) - * - * The :in, :out, :err, an integer, an IO and an array key specifies a redirection. - * The redirection maps a file descriptor in the child process. - * - * For example, stderr can be merged into stdout as follows: - * - * pid = spawn(command, :err=>:out) - * pid = spawn(command, 2=>1) - * pid = spawn(command, STDERR=>:out) - * pid = spawn(command, STDERR=>STDOUT) - * - * The hash keys specifies a file descriptor in the child process - * started by #spawn. - * :err, 2 and STDERR specifies the standard error stream (stderr). - * - * The hash values specifies a file descriptor in the parent process - * which invokes #spawn. - * :out, 1 and STDOUT specifies the standard output stream (stdout). - * - * In the above example, - * the standard output in the child process is not specified. - * So it is inherited from the parent process. - * - * The standard input stream (stdin) can be specified by :in, 0 and STDIN. - * - * A filename can be specified as a hash value. - * - * pid = spawn(command, :in=>File::NULL) # read mode - * pid = spawn(command, :out=>File::NULL) # write mode - * pid = spawn(command, :err=>"log") # write mode - * pid = spawn(command, [:out, :err]=>File::NULL) # write mode - * pid = spawn(command, 3=>File::NULL) # read mode + * spawn([env, ] command_line, options = {}) -> pid + * spawn([env, ] exe_path, *args, options = {}) -> pid * - * For stdout and stderr (and combination of them), - * it is opened in write mode. - * Otherwise read mode is used. + * Creates a new child process by doing one of the following + * in that process: + * + * - Passing string +command_line+ to the shell. + * - Invoking the executable at +exe_path+. + * + * This method has potential security vulnerabilities if called with untrusted input; + * see {Command Injection}[rdoc-ref:command_injection.rdoc]. + * + * Returns the process ID (pid) of the new process, + * without waiting for it to complete. + * + * To avoid zombie processes, the parent process should call either: + * + * - Process.wait, to collect the termination statuses of its children. + * - Process.detach, to register disinterest in their status. + * + * The new process is created using the + * {exec system call}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/functions/execve.html]; + * it may inherit some of its environment from the calling program + * (possibly including open file descriptors). + * + * Argument +env+, if given, is a hash that affects +ENV+ for the new process; + * see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + * + * Argument +options+ is a hash of options for the new process; + * see {Execution Options}[rdoc-ref:Process@Execution+Options]. + * + * The first required argument is one of the following: + * + * - +command_line+ if it is a string, + * and if it begins with a shell reserved word or special built-in, + * or if it contains one or more metacharacters. + * - +exe_path+ otherwise. * - * For specifying flags and permission of file creation explicitly, - * an array is used instead. + * Argument +command_line+ + * + * \String argument +command_line+ is a command line to be passed to a shell; + * it must begin with a shell reserved word, begin with a special built-in, + * or contain meta characters: + * + * spawn('echo') # => 798847 + * Process.wait # => 798847 + * spawn('if true; then echo "Foo"; fi') # => 798848 + * Process.wait # => 798848 + * spawn('date > /tmp/date.tmp') # => 798879 + * Process.wait # => 798849 + * spawn('date > /nop/date.tmp') # => 798882 # Issues error message. + * Process.wait # => 798882 * - * pid = spawn(command, :in=>["file"]) # read mode is assumed - * pid = spawn(command, :in=>["file", "r"]) - * pid = spawn(command, :out=>["log", "w"]) # 0644 assumed - * pid = spawn(command, :out=>["log", "w", 0600]) - * pid = spawn(command, :out=>["log", File::WRONLY|File::EXCL|File::CREAT, 0600]) + * The command line may also contain arguments and options for the command: * - * The array specifies a filename, flags and permission. - * The flags can be a string or an integer. - * If the flags is omitted or nil, File::RDONLY is assumed. - * The permission should be an integer. - * If the permission is omitted or nil, 0644 is assumed. + * spawn('echo "Foo"') # => 799031 + * Process.wait # => 799031 * - * If an array of IOs and integers are specified as a hash key, - * all the elements are redirected. + * Output: * - * # stdout and stderr is redirected to log file. - * # The file "log" is opened just once. - * pid = spawn(command, [:out, :err]=>["log", "w"]) + * Foo * - * Another way to merge multiple file descriptors is [:child, fd]. - * \[:child, fd] means the file descriptor in the child process. - * This is different from fd. - * For example, :err=>:out means redirecting child stderr to parent stdout. - * But :err=>[:child, :out] means redirecting child stderr to child stdout. - * They differ if stdout is redirected in the child process as follows. + * On a Unix-like system, the shell is /bin/sh; + * otherwise the shell is determined by environment variable + * ENV['RUBYSHELL'], if defined, or ENV['COMSPEC'] otherwise. * - * # stdout and stderr is redirected to log file. - * # The file "log" is opened just once. - * pid = spawn(command, :out=>["log", "w"], :err=>[:child, :out]) + * Except for the +COMSPEC+ case, + * the entire string +command_line+ is passed as an argument + * to {shell option -c}[https://pubs.opengroup.org/onlinepubs/9699919799.2018edition/utilities/sh.html]. * - * \[:child, :out] can be used to merge stderr into stdout in IO.popen. - * In this case, IO.popen redirects stdout to a pipe in the child process - * and [:child, :out] refers the redirected stdout. + * The shell performs normal shell expansion on the command line: * - * io = IO.popen(["sh", "-c", "echo out; echo err >&2", :err=>[:child, :out]]) - * p io.read #=> "out\nerr\n" + * spawn('echo C*') # => 799139 + * Process.wait # => 799139 * - * The :chdir key in +options+ specifies the current directory. + * Output: * - * pid = spawn(command, :chdir=>"/var/tmp") + * CONTRIBUTING.md COPYING COPYING.ja * - * spawn closes all non-standard unspecified descriptors by default. - * The "standard" descriptors are 0, 1 and 2. - * This behavior is specified by :close_others option. - * :close_others doesn't affect the standard descriptors which are - * closed only if :close is specified explicitly. + * Raises an exception if the new process could not execute. * - * pid = spawn(command, :close_others=>true) # close 3,4,5,... (default) - * pid = spawn(command, :close_others=>false) # don't close 3,4,5,... + * Argument +exe_path+ * - * :close_others is false by default for spawn and IO.popen. + * Argument +exe_path+ is one of the following: * - * Note that fds which close-on-exec flag is already set are closed - * regardless of :close_others option. + * - The string path to an executable to be called. + * - A 2-element array containing the path to an executable + * and the string to be used as the name of the executing process. * - * So IO.pipe and spawn can be used as IO.popen. + * Example: * - * # similar to r = IO.popen(command) - * r, w = IO.pipe - * pid = spawn(command, :out=>w) # r, w is closed in the child process. - * w.close + * spawn('/usr/bin/date') # => 799198 # Path to date on Unix-style system. + * Process.wait # => 799198 * - * :close is specified as a hash value to close a fd individually. + * Output: * - * f = open(foo) - * system(command, f=>:close) # don't inherit f. + * Thu Aug 31 10:06:48 AM CDT 2023 * - * If a file descriptor need to be inherited, - * io=>io can be used. + * Ruby invokes the executable directly, with no shell and no shell expansion. * - * # valgrind has --log-fd option for log destination. - * # log_w=>log_w indicates log_w.fileno inherits to child process. - * log_r, log_w = IO.pipe - * pid = spawn("valgrind", "--log-fd=#{log_w.fileno}", "echo", "a", log_w=>log_w) - * log_w.close - * p log_r.read + * If one or more +args+ is given, each is an argument or option + * to be passed to the executable: * - * It is also possible to exchange file descriptors. + * spawn('echo', 'C*') # => 799392 + * Process.wait # => 799392 + * spawn('echo', 'hello', 'world') # => 799393 + * Process.wait # => 799393 * - * pid = spawn(command, :out=>:err, :err=>:out) + * Output: * - * The hash keys specify file descriptors in the child process. - * The hash values specifies file descriptors in the parent process. - * So the above specifies exchanging stdout and stderr. - * Internally, +spawn+ uses an extra file descriptor to resolve such cyclic - * file descriptor mapping. + * C* + * hello world * - * See Kernel.exec for the standard shell. + * Raises an exception if the new process could not execute. */ static VALUE From 322548180d01ce99dcb8ecb3c36f2a9261554657 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 31 Aug 2023 15:35:56 +0100 Subject: [PATCH 189/208] Prevent rb_gc_mark_values from pinning objects This is an internal only function not exposed to the C extension API. It's only use so far is from rb_vm_mark, where it's used to mark the values in the vm->trap_list.cmd array. There shouldn't be any reason why these cannot move. This commit allows them to move by updating their references during the reference updating step of compaction. To do this we've introduced another internal function rb_gc_update_values as a partner to rb_gc_mark_values. This allows us to refactor rb_gc_mark_values to not pin --- gc.c | 8 +++++++- internal/gc.h | 1 + vm.c | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/gc.c b/gc.c index 59c507369e2216..d780bb0613d7e1 100644 --- a/gc.c +++ b/gc.c @@ -6348,7 +6348,7 @@ rb_gc_mark_values(long n, const VALUE *values) rb_objspace_t *objspace = &rb_objspace; for (i=0; ioverloaded_cme_table); + rb_gc_update_values(RUBY_NSIG, vm->trap_list.cmd); + if (vm->coverages) { vm->coverages = rb_gc_location(vm->coverages); vm->me2counter = rb_gc_location(vm->me2counter); From 945945dad434dd2c014a4d310dc7dc51e6d4321e Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 31 Aug 2023 15:45:57 +0100 Subject: [PATCH 190/208] Remove gc_mark_values Now that gc_mark_values and rb_gc_mark_values are identical, we should remove one. --- gc.c | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/gc.c b/gc.c index d780bb0613d7e1..4c6fdbb3009f7e 100644 --- a/gc.c +++ b/gc.c @@ -6331,16 +6331,6 @@ rb_gc_mark_locations(const VALUE *start, const VALUE *end) gc_mark_locations(&rb_objspace, start, end, gc_mark_maybe); } -static void -gc_mark_values(rb_objspace_t *objspace, long n, const VALUE *values) -{ - long i; - - for (i=0; iep[VM_ENV_DATA_INDEX_ENV] == obj); GC_ASSERT(VM_ENV_ESCAPED_P(env->ep)); - gc_mark_values(objspace, (long)env->env_size, env->env); + rb_gc_mark_values((long)env->env_size, env->env); VM_ENV_FLAGS_SET(env->ep, VM_ENV_FLAG_WB_REQUIRED); gc_mark(objspace, (VALUE)rb_vm_env_prev_env(env)); gc_mark(objspace, (VALUE)env->iseq); From 9930363aab6ac4b8d7034baff85cd86c17953dc9 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 1 Sep 2023 03:15:47 +0900 Subject: [PATCH 191/208] [Bug-18878] Parse qualified const with brace block as method call --- parse.y | 27 +++++++++++++++++++-------- test/ruby/test_syntax.rb | 8 ++++++++ 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/parse.y b/parse.y index 36ba905db12e61..1b103ca5fed72c 100644 --- a/parse.y +++ b/parse.y @@ -910,6 +910,13 @@ set_line_body(NODE *body, int line) } } +static void +set_embraced_location(NODE *node, const rb_code_location_t *beg, const rb_code_location_t *end) +{ + node->nd_body->nd_loc = code_loc_gen(beg, end); + nd_set_line(node, beg->end_pos.lineno); +} + #define yyparse ruby_yyparse static NODE* cond(struct parser_params *p, NODE *node, const YYLTYPE *loc); @@ -2250,8 +2257,7 @@ cmd_brace_block : tLBRACE_ARG brace_body '}' { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } ; @@ -2314,6 +2320,14 @@ command : fcall command_args %prec tLOWEST /*% %*/ /*% ripper: method_add_block!(command_call!($1, $2, $3, $4), $5) %*/ } + | primary_value tCOLON2 tCONSTANT '{' brace_body '}' + { + /*%%%*/ + set_embraced_location($5, &@4, &@6); + $$ = new_command_qcall(p, ID2VAL(idCOLON2), $1, $3, Qnull, $5, &@3, &@$); + /*% %*/ + /*% ripper: method_add_block!(command_call!($1, $2, $3, Qnull), $5) %*/ + } | keyword_super command_args { /*%%%*/ @@ -4274,8 +4288,7 @@ do_block : k_do_block do_body k_end { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } ; @@ -4393,16 +4406,14 @@ brace_block : '{' brace_body '}' { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } | k_do do_body k_end { $$ = $2; /*%%%*/ - $$->nd_body->nd_loc = code_loc_gen(&@1, &@3); - nd_set_line($$, @1.end_pos.lineno); + set_embraced_location($$, &@1, &@3); /*% %*/ } ; diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index bcc37e7bbbda46..c65d7af4c257e9 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1640,6 +1640,14 @@ def test_classdef_in_cond def test_command_with_cmd_brace_block assert_valid_syntax('obj.foo (1) {}') assert_valid_syntax('obj::foo (1) {}') + assert_valid_syntax('bar {}') + assert_valid_syntax('Bar {}') + assert_valid_syntax('bar() {}') + assert_valid_syntax('Bar() {}') + assert_valid_syntax('Foo::bar {}') + assert_valid_syntax('Foo::Bar {}') + assert_valid_syntax('Foo::bar() {}') + assert_valid_syntax('Foo::Bar() {}') end def test_numbered_parameter From df4c77608e76068deed58b2781674b0eb247c325 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 31 Aug 2023 18:40:29 -0400 Subject: [PATCH 192/208] [ruby/yarp] fix: octal, hex, and unicode strings at the end of a file (https://github.com/ruby/yarp/pull/1371) * refactor: move EOF check into yp_unescape_calculate_difference parser_lex is a bit more readable when we can rely on that behavior * fix: octal and hex digits at the end of a file Previously this resulted in invalid memory access. * fix: unicode strings at the end of a file Previously this resulted in invalid memory access. * Unterminated curly-bracket unicode is a syntax error https://github.com/ruby/yarp/commit/21cf11acb5 --- test/yarp/errors_test.rb | 7 +++++++ test/yarp/fuzzer_test.rb | 13 ++++++++++++ yarp/unescape.c | 40 +++++++++++++++++++++++------------- yarp/yarp.c | 44 ++++++++++++++++++---------------------- 4 files changed, 66 insertions(+), 38 deletions(-) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index 0e1931ec6ac68d..9e60f5d98bca27 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -621,6 +621,13 @@ def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_charact ] end + def test_unterminated_unicode_brackets_should_be_a_syntax_error + assert_errors expression('?\\u{3'), '?\\u{3', [ + ["invalid Unicode escape.", 1..5], + ["invalid Unicode escape.", 1..5], + ] + end + def test_method_parameters_after_block expected = DefNode( Location(), diff --git a/test/yarp/fuzzer_test.rb b/test/yarp/fuzzer_test.rb index 568c2eaf08d9c5..f4abcd4ac8000f 100644 --- a/test/yarp/fuzzer_test.rb +++ b/test/yarp/fuzzer_test.rb @@ -22,6 +22,19 @@ def self.snippet(name, source) snippet "incomplete escaped list", "%w[\\" snippet "incomplete escaped regex", "/a\\" snippet "unterminated heredoc with unterminated escape at end of file", "<= end || !yp_char_is_octal_digit(backslash[2])) { return 2; } - *value = (uint8_t) ((*value << 3) | (backslash[2] - '0')); - if (!yp_char_is_octal_digit(backslash[3])) { + if (backslash + 3 >= end || !yp_char_is_octal_digit(backslash[3])) { return 3; } - *value = (uint8_t) ((*value << 3) | (backslash[3] - '0')); return 4; } @@ -93,12 +91,15 @@ unescape_hexadecimal_digit(const uint8_t value) { // Scan the 1-2 digits of hexadecimal into the value. Returns the number of // digits scanned. static inline size_t -unescape_hexadecimal(const uint8_t *backslash, uint8_t *value) { +unescape_hexadecimal(const uint8_t *backslash, uint8_t *value, const uint8_t *end) { + *value = 0; + if (backslash + 2 >= end || !yp_char_is_hexadecimal_digit(backslash[2])) { + return 2; + } *value = unescape_hexadecimal_digit(backslash[2]); - if (!yp_char_is_hexadecimal_digit(backslash[3])) { + if (backslash + 3 >= end || !yp_char_is_hexadecimal_digit(backslash[3])) { return 3; } - *value = (uint8_t) ((*value << 4) | unescape_hexadecimal_digit(backslash[3])); return 4; } @@ -204,7 +205,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { uint8_t value; - const uint8_t *cursor = backslash + unescape_octal(backslash, &value); + const uint8_t *cursor = backslash + unescape_octal(backslash, &value, end); if (dest) { dest[(*dest_length)++] = unescape_char(value, flags); @@ -214,7 +215,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \xnn hexadecimal bit pattern, where nn is 1-2 hexadecimal digits ([0-9a-fA-F]) case 'x': { uint8_t value; - const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value); + const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value, end); if (dest) { dest[(*dest_length)++] = unescape_char(value, flags); @@ -236,13 +237,14 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t unicode_cursor += yp_strspn_whitespace(unicode_cursor, end - unicode_cursor); - while ((*unicode_cursor != '}') && (unicode_cursor < end)) { + while ((unicode_cursor < end) && (*unicode_cursor != '}')) { const uint8_t *unicode_start = unicode_cursor; size_t hexadecimal_length = yp_strspn_hexadecimal_digit(unicode_cursor, end - unicode_cursor); // \u{nnnn} character literal allows only 1-6 hexadecimal digits - if (hexadecimal_length > 6) + if (hexadecimal_length > 6) { yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); + } // there are not hexadecimal characters if (hexadecimal_length == 0) { @@ -269,10 +271,16 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) yp_diagnostic_list_append(&parser->error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); - return unicode_cursor + 1; + + if (unicode_cursor < end && *unicode_cursor == '}') { + unicode_cursor++; + } else { + yp_diagnostic_list_append(&parser->error_list, backslash, unicode_cursor, "invalid Unicode escape."); + } + return unicode_cursor; } - if ((backslash + 2) < end && yp_char_is_hexadecimal_digits(backslash + 2, 4)) { + if ((backslash + 5) < end && yp_char_is_hexadecimal_digits(backslash + 2, 4)) { uint32_t value; unescape_unicode(backslash + 2, 4, &value); @@ -538,6 +546,10 @@ size_t yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *backslash, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { assert(unescape_type != YP_UNESCAPE_NONE); + if (backslash + 1 >= parser->end) { + return 0; + } + switch (backslash[1]) { case '\\': case '\'': diff --git a/yarp/yarp.c b/yarp/yarp.c index 376dfe32c3ce91..176e9f76b605ae 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -7002,17 +7002,16 @@ parser_lex(yp_parser_t *parser) { // literally. In this case we'll skip past the next character // and find the next breakpoint. if (*breakpoint == '\\') { - // Check that we're not at the end of the file. - if (breakpoint + 1 >= parser->end) { + yp_unescape_type_t unescape_type = lex_mode->as.list.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); + if (difference == 0) { + // we're at the end of the file breakpoint = NULL; continue; } - yp_unescape_type_t unescape_type = lex_mode->as.list.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; - size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); - // If the result is an escaped newline ... - if (*(breakpoint + difference - 1) == '\n') { + if (breakpoint[difference - 1] == '\n') { if (parser->heredoc_end) { // ... if we are on the same line as a heredoc, flush the heredoc and // continue parsing after heredoc_end. @@ -7141,16 +7140,15 @@ parser_lex(yp_parser_t *parser) { // literally. In this case we'll skip past the next character // and find the next breakpoint. if (*breakpoint == '\\') { - // Check that we're not at the end of the file. - if (breakpoint + 1 >= parser->end) { + size_t difference = yp_unescape_calculate_difference(parser, breakpoint, YP_UNESCAPE_ALL, false); + if (difference == 0) { + // we're at the end of the file breakpoint = NULL; continue; } - size_t difference = yp_unescape_calculate_difference(parser, breakpoint, YP_UNESCAPE_ALL, false); - // If the result is an escaped newline ... - if (*(breakpoint + difference - 1) == '\n') { + if (breakpoint[difference - 1] == '\n') { if (parser->heredoc_end) { // ... if we are on the same line as a heredoc, flush the heredoc and // continue parsing after heredoc_end. @@ -7293,20 +7291,19 @@ parser_lex(yp_parser_t *parser) { breakpoint = yp_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1)); break; case '\\': { - // Check that we're not at the end of the file. - if (breakpoint + 1 >= parser->end) { - breakpoint = NULL; - break; - } - // If we hit escapes, then we need to treat the next token // literally. In this case we'll skip past the next character and // find the next breakpoint. yp_unescape_type_t unescape_type = parser->lex_modes.current->as.string.interpolation ? YP_UNESCAPE_ALL : YP_UNESCAPE_MINIMAL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); + if (difference == 0) { + // we're at the end of the file + breakpoint = NULL; + break; + } // If the result is an escaped newline ... - if (*(breakpoint + difference - 1) == '\n') { + if (breakpoint[difference - 1] == '\n') { if (parser->heredoc_end) { // ... if we are on the same line as a heredoc, flush the heredoc and // continue parsing after heredoc_end. @@ -7463,12 +7460,6 @@ parser_lex(yp_parser_t *parser) { break; } case '\\': { - // Check that we're not at the end of the file. - if (breakpoint + 1 >= parser->end) { - breakpoint = NULL; - break; - } - // If we hit an escape, then we need to skip past // however many characters the escape takes up. However // it's important that if \n or \r\n are escaped that we @@ -7481,6 +7472,11 @@ parser_lex(yp_parser_t *parser) { } else { yp_unescape_type_t unescape_type = (quote == YP_HEREDOC_QUOTE_SINGLE) ? YP_UNESCAPE_MINIMAL : YP_UNESCAPE_ALL; size_t difference = yp_unescape_calculate_difference(parser, breakpoint, unescape_type, false); + if (difference == 0) { + // we're at the end of the file + breakpoint = NULL; + break; + } yp_newline_list_check_append(&parser->newline_list, breakpoint + difference - 1); From 45cd011d73ed1fac195d828c0565e2cac57f65fc Mon Sep 17 00:00:00 2001 From: yui-knk Date: Thu, 31 Aug 2023 21:35:55 +0900 Subject: [PATCH 193/208] [Bug #19281] Allow semicolon in parenthesis at the first argument of command call Allow compstmt in the first argument of command call wrapped with parenthesis like following arguments with parenthesis. --- parse.y | 9 +-------- test/ruby/test_syntax.rb | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/parse.y b/parse.y index 1b103ca5fed72c..8b88b1b38df6a8 100644 --- a/parse.y +++ b/parse.y @@ -3362,14 +3362,7 @@ primary : literal /*% %*/ /*% ripper: begin!($3) %*/ } - | tLPAREN_ARG {SET_LEX_STATE(EXPR_ENDARG);} rparen - { - /*%%%*/ - $$ = NEW_BEGIN(0, &@$); - /*% %*/ - /*% ripper: paren!(0) %*/ - } - | tLPAREN_ARG stmt {SET_LEX_STATE(EXPR_ENDARG);} rparen + | tLPAREN_ARG compstmt {SET_LEX_STATE(EXPR_ENDARG);} ')' { /*%%%*/ if (nd_type_p($2, NODE_SELF)) $2->nd_state = 0; diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index c65d7af4c257e9..cda84c6368e383 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -1650,6 +1650,21 @@ def test_command_with_cmd_brace_block assert_valid_syntax('Foo::Bar() {}') end + def test_command_newline_in_tlparen_args + assert_valid_syntax("p (1\n2\n),(3),(4)") + assert_valid_syntax("p (\n),(),()") + assert_valid_syntax("a.b (1\n2\n),(3),(4)") + assert_valid_syntax("a.b (\n),(),()") + end + + def test_command_semicolon_in_tlparen_at_the_first_arg + bug19281 = '[ruby-core:111499] [Bug #19281]' + assert_valid_syntax('p (1;2),(3),(4)', bug19281) + assert_valid_syntax('p (;),(),()', bug19281) + assert_valid_syntax('a.b (1;2),(3),(4)', bug19281) + assert_valid_syntax('a.b (;),(),()', bug19281) + end + def test_numbered_parameter assert_valid_syntax('proc {_1}') assert_equal(3, eval('[1,2].then {_1+_2}')) From 61f1657f68b9cddf8956380e53b8e7053eeb30c8 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 1 Sep 2023 13:36:15 +0900 Subject: [PATCH 194/208] Use macro argument not the variable directly --- iseq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iseq.c b/iseq.c index 4466de8bd105c9..7d64177e995705 100644 --- a/iseq.c +++ b/iseq.c @@ -730,7 +730,7 @@ set_compile_option_from_hash(rb_compile_option_t *option, VALUE opt) else if (flag == Qfalse) { (o)->mem = 0; } \ } #define SET_COMPILE_OPTION_NUM(o, h, mem) \ - { VALUE num = rb_hash_aref(opt, ID2SYM(rb_intern(#mem))); \ + { VALUE num = rb_hash_aref((h), ID2SYM(rb_intern(#mem))); \ if (!NIL_P(num)) (o)->mem = NUM2INT(num); \ } SET_COMPILE_OPTION(option, opt, inline_const_cache); From 1fbc8cdf0686657c12cc9014cb4714730adc2ec6 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 1 Sep 2023 13:39:36 +0900 Subject: [PATCH 195/208] Copy `rb_compile_option_t` only if needed Use `COMPILE_OPTION_DEFAULT` if nothing to change. --- iseq.c | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/iseq.c b/iseq.c index 7d64177e995705..3b91317d339ab4 100644 --- a/iseq.c +++ b/iseq.c @@ -916,13 +916,12 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea rb_iseq_t *iseq = iseq_alloc(); rb_compile_option_t new_opt; - if (option) { + if (!option) option = &COMPILE_OPTION_DEFAULT; + if (ast) { new_opt = *option; + rb_iseq_make_compile_option(&new_opt, make_compile_option_from_ast(ast)); + option = &new_opt; } - else { - new_opt = COMPILE_OPTION_DEFAULT; - } - if (ast) rb_iseq_make_compile_option(&new_opt, make_compile_option_from_ast(ast)); VALUE script_lines = Qnil; @@ -934,7 +933,7 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea } prepare_iseq_build(iseq, name, path, realpath, first_lineno, node ? &node->nd_loc : NULL, node ? nd_node_id(node) : -1, - parent, isolated_depth, type, script_lines, &new_opt); + parent, isolated_depth, type, script_lines, option); rb_iseq_compile_node(iseq, node); finish_iseq_build(iseq); @@ -950,19 +949,11 @@ yp_iseq_new_with_opt(yp_node_t *node, yp_parser_t *parser, VALUE name, VALUE pat enum rb_iseq_type type, const rb_compile_option_t *option) { rb_iseq_t *iseq = iseq_alloc(); - rb_compile_option_t new_opt; - - if (option) { - new_opt = *option; - } - else { - new_opt = COMPILE_OPTION_DEFAULT; - } - VALUE script_lines = Qnil; - rb_code_location_t code_loc; + if (!option) option = &COMPILE_OPTION_DEFAULT; + if (node) { yp_line_column_t start_line_col = yp_newline_list_line_column(&(parser->newline_list), node->location.start); yp_line_column_t end_line_col= yp_newline_list_line_column(&(parser->newline_list), node->location.end); @@ -981,7 +972,7 @@ yp_iseq_new_with_opt(yp_node_t *node, yp_parser_t *parser, VALUE name, VALUE pat // TODO: node_id int node_id = -1; prepare_iseq_build(iseq, name, path, realpath, first_lineno, &code_loc, node_id, - parent, isolated_depth, type, script_lines, &new_opt); + parent, isolated_depth, type, script_lines, option); rb_iseq_compile_yarp_node(iseq, node, parser); From 4c040fe8503d664dce46bf2fd9d2cfe8be6fbf41 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 1 Sep 2023 14:06:42 +0900 Subject: [PATCH 196/208] Copy compile options from AST directly without intermediate Hash --- iseq.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/iseq.c b/iseq.c index 3b91317d339ab4..db7303aac5bd96 100644 --- a/iseq.c +++ b/iseq.c @@ -747,20 +747,15 @@ set_compile_option_from_hash(rb_compile_option_t *option, VALUE opt) #undef SET_COMPILE_OPTION_NUM } -static VALUE -make_compile_option_from_ast(const rb_ast_body_t *ast) +static rb_compile_option_t * +set_compile_option_from_ast(rb_compile_option_t *option, const rb_ast_body_t *ast) { - VALUE opt = rb_obj_hide(rb_ident_hash_new()); - if (ast->frozen_string_literal >= 0) rb_hash_aset(opt, rb_sym_intern_ascii_cstr("frozen_string_literal"), RBOOL(ast->frozen_string_literal)); - if (ast->coverage_enabled >= 0) rb_hash_aset(opt, rb_sym_intern_ascii_cstr("coverage_enabled"), RBOOL(ast->coverage_enabled)); - return opt; -} - -static void -rb_iseq_make_compile_option(rb_compile_option_t *option, VALUE opt) -{ - Check_Type(opt, T_HASH); - set_compile_option_from_hash(option, opt); +#define SET_COMPILE_OPTION(o, a, mem) \ + ((a)->mem < 0 ? 0 : ((o)->mem = (a)->mem > 0)) + SET_COMPILE_OPTION(option, ast, frozen_string_literal); + SET_COMPILE_OPTION(option, ast, coverage_enabled); +#undef SET_COMPILE_OPTION + return option; } static void @@ -919,8 +914,7 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea if (!option) option = &COMPILE_OPTION_DEFAULT; if (ast) { new_opt = *option; - rb_iseq_make_compile_option(&new_opt, make_compile_option_from_ast(ast)); - option = &new_opt; + option = set_compile_option_from_ast(&new_opt, ast); } VALUE script_lines = Qnil; From 2efd59e2eb4ebf68f85ab6f07b1a4aea0584248a Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Fri, 1 Sep 2023 14:22:06 +0900 Subject: [PATCH 197/208] [rubygems/rubygems] Use assertion message strictly https://github.com/rubygems/rubygems/commit/98da5b9826 --- spec/bundler/runtime/setup_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/bundler/runtime/setup_spec.rb b/spec/bundler/runtime/setup_spec.rb index 09f179c51739dd..f02f81f6974104 100644 --- a/spec/bundler/runtime/setup_spec.rb +++ b/spec/bundler/runtime/setup_spec.rb @@ -1598,7 +1598,7 @@ module Gem::BUNDLED_GEMS require 'csv' R - expect(err).to be_empty + expect(err).not_to include("Add csv to your Gemfile") end it "don't warn with bundled gems when it's declared in Gemfile" do From 7fb56df726d4e661fdec10266a54cf27e52d9892 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 30 Aug 2023 21:29:11 +0200 Subject: [PATCH 198/208] [ruby/yarp] Fix comments for methods using desugar_or_write_defined_node https://github.com/ruby/yarp/commit/a39147736e --- lib/yarp/desugar_visitor.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb index d0c0494e0c013e..426ecf541f6210 100644 --- a/lib/yarp/desugar_visitor.rb +++ b/lib/yarp/desugar_visitor.rb @@ -42,7 +42,7 @@ def visit_constant_and_write_node(node) # # becomes # - # Foo || Foo = bar + # defined?(Foo) ? Foo : Foo = bar def visit_constant_or_write_node(node) desugar_or_write_defined_node(node, ConstantReadNode, ConstantWriteNode) end @@ -74,7 +74,7 @@ def visit_constant_path_and_write_node(node) # # becomes # - # Foo::Bar || Foo::Bar = baz + # defined?(Foo::Bar) ? Foo::Bar : Foo::Bar = baz def visit_constant_path_or_write_node(node) IfNode.new( node.operator_loc, @@ -132,7 +132,7 @@ def visit_global_variable_and_write_node(node) # # becomes # - # $foo || $foo = bar + # defined?($foo) ? $foo : $foo = bar def visit_global_variable_or_write_node(node) desugar_or_write_defined_node(node, GlobalVariableReadNode, GlobalVariableWriteNode) end @@ -244,7 +244,7 @@ def desugar_or_write_node(node, read_class, write_class, arguments: []) ) end - # Don't desugar `x ||= y` to `defined?(x) ? x : x = y` + # Desugar `x ||= y` to `defined?(x) ? x : x = y` def desugar_or_write_defined_node(node, read_class, write_class, arguments: []) IfNode.new( node.operator_loc, From 4172036bc6ba77aded874f67b15d657bd7ee3241 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 30 Aug 2023 22:22:04 +0200 Subject: [PATCH 199/208] [ruby/yarp] Do not desugar Foo::Bar {||,&&,+}= baz as it is incorrect without a temporary variable * See https://github.com/ruby/yarp/pull/1329#discussion_r1310775433 for details. https://github.com/ruby/yarp/commit/f0fdcba0c3 --- lib/yarp/desugar_visitor.rb | 63 ------------------------------- test/yarp/desugar_visitor_test.rb | 11 ++++-- 2 files changed, 8 insertions(+), 66 deletions(-) diff --git a/lib/yarp/desugar_visitor.rb b/lib/yarp/desugar_visitor.rb index 426ecf541f6210..a988449dc07a78 100644 --- a/lib/yarp/desugar_visitor.rb +++ b/lib/yarp/desugar_visitor.rb @@ -56,69 +56,6 @@ def visit_constant_operator_write_node(node) desugar_operator_write_node(node, ConstantReadNode, ConstantWriteNode) end - # Foo::Bar &&= baz - # - # becomes - # - # Foo::Bar && Foo::Bar = baz - def visit_constant_path_and_write_node(node) - AndNode.new( - node.target, - ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location), - node.operator_loc, - node.location - ) - end - - # Foo::Bar ||= baz - # - # becomes - # - # defined?(Foo::Bar) ? Foo::Bar : Foo::Bar = baz - def visit_constant_path_or_write_node(node) - IfNode.new( - node.operator_loc, - DefinedNode.new(nil, node.target, nil, node.operator_loc, node.target.location), - StatementsNode.new([node.target], node.location), - ElseNode.new( - node.operator_loc, - StatementsNode.new( - [ConstantPathWriteNode.new(node.target, node.value, node.operator_loc, node.location)], - node.location - ), - node.operator_loc, - node.location - ), - node.operator_loc, - node.location - ) - end - - # Foo::Bar += baz - # - # becomes - # - # Foo::Bar = Foo::Bar + baz - def visit_constant_path_operator_write_node(node) - ConstantPathWriteNode.new( - node.target, - CallNode.new( - node.target, - nil, - node.operator_loc.copy(length: node.operator_loc.length - 1), - nil, - ArgumentsNode.new([node.value], node.value.location), - nil, - nil, - 0, - node.operator_loc.slice.chomp("="), - node.location - ), - node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), - node.location - ) - end - # $foo &&= bar # # becomes diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index c03b02e67a00a1..1ef2710c8e804e 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -6,7 +6,7 @@ module YARP class DesugarVisitorTest < TestCase def test_and_write assert_desugars("(AndNode (ClassVariableReadNode) (ClassVariableWriteNode (CallNode)))", "@@foo &&= bar") - assert_desugars("(AndNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))", "Foo::Bar &&= baz") + assert_not_desugared("Foo::Bar &&= baz", "Desugaring would execute Foo twice or need temporary variables") assert_desugars("(AndNode (ConstantReadNode) (ConstantWriteNode (CallNode)))", "Foo &&= bar") assert_desugars("(AndNode (GlobalVariableReadNode) (GlobalVariableWriteNode (CallNode)))", "$foo &&= bar") assert_desugars("(AndNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo &&= bar") @@ -16,7 +16,7 @@ def test_and_write def test_or_write assert_desugars("(IfNode (DefinedNode (ClassVariableReadNode)) (StatementsNode (ClassVariableReadNode)) (ElseNode (StatementsNode (ClassVariableWriteNode (CallNode)))))", "@@foo ||= bar") - assert_desugars("(IfNode (DefinedNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (StatementsNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode))) (ElseNode (StatementsNode (ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode)))))", "Foo::Bar ||= baz") + assert_not_desugared("Foo::Bar ||= baz", "Desugaring would execute Foo twice or need temporary variables") assert_desugars("(IfNode (DefinedNode (ConstantReadNode)) (StatementsNode (ConstantReadNode)) (ElseNode (StatementsNode (ConstantWriteNode (CallNode)))))", "Foo ||= bar") assert_desugars("(IfNode (DefinedNode (GlobalVariableReadNode)) (StatementsNode (GlobalVariableReadNode)) (ElseNode (StatementsNode (GlobalVariableWriteNode (CallNode)))))", "$foo ||= bar") assert_desugars("(OrNode (InstanceVariableReadNode) (InstanceVariableWriteNode (CallNode)))", "@foo ||= bar") @@ -26,7 +26,7 @@ def test_or_write def test_operator_write assert_desugars("(ClassVariableWriteNode (CallNode (ClassVariableReadNode) (ArgumentsNode (CallNode))))", "@@foo += bar") - assert_desugars("(ConstantPathWriteNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (CallNode (ConstantPathNode (ConstantReadNode) (ConstantReadNode)) (ArgumentsNode (CallNode))))", "Foo::Bar += baz") + assert_not_desugared("Foo::Bar += baz", "Desugaring would execute Foo twice or need temporary variables") assert_desugars("(ConstantWriteNode (CallNode (ConstantReadNode) (ArgumentsNode (CallNode))))", "Foo += bar") assert_desugars("(GlobalVariableWriteNode (CallNode (GlobalVariableReadNode) (ArgumentsNode (CallNode))))", "$foo += bar") assert_desugars("(InstanceVariableWriteNode (CallNode (InstanceVariableReadNode) (ArgumentsNode (CallNode))))", "@foo += bar") @@ -55,5 +55,10 @@ def assert_desugars(expected, source) ast = YARP.parse(source).value.accept(DesugarVisitor.new) assert_equal expected, ast_inspect(ast.statements.body.last) end + + def assert_not_desugared(source, reason) + ast = YARP.parse(source).value + assert_equal_nodes(ast, ast.accept(YARP::DesugarVisitor.new)) + end end end From f1f6f1b39ebec35462efed9b20bfac4b6409133f Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 30 Aug 2023 22:43:40 +0200 Subject: [PATCH 200/208] [ruby/yarp] Make operator_loc the last field for GlobalVariableWriteNode * Consistent with ClassVariableWriteNode, ConstantWriteNode, InstanceVariableWriteNode, LocalVariableWriteNode. * Fixes desugaring of global variable with operators. https://github.com/ruby/yarp/commit/fb5a53fc0b --- test/yarp/errors_test.rb | 4 ++-- test/yarp/snapshots/seattlerb/bug202.txt | 2 +- .../yarp/snapshots/unparser/corpus/literal/assignment.txt | 2 +- test/yarp/snapshots/variables.txt | 8 ++++---- test/yarp/snapshots/whitequark/gvasgn.txt | 2 +- yarp/config.yml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index 9e60f5d98bca27..f29bd746c81217 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -1003,8 +1003,8 @@ def test_dont_allow_setting_to_back_and_nth_reference expected = BeginNode( Location(), StatementsNode([ - GlobalVariableWriteNode(Location(), Location(), NilNode()), - GlobalVariableWriteNode(Location(), Location(), NilNode()) + GlobalVariableWriteNode(Location(), NilNode(), Location()), + GlobalVariableWriteNode(Location(), NilNode(), Location()) ]), nil, nil, diff --git a/test/yarp/snapshots/seattlerb/bug202.txt b/test/yarp/snapshots/seattlerb/bug202.txt index 7bb548f79830b6..64e666ef3f55fe 100644 --- a/test/yarp/snapshots/seattlerb/bug202.txt +++ b/test/yarp/snapshots/seattlerb/bug202.txt @@ -1,7 +1,7 @@ ProgramNode(0...22)( [:测试], StatementsNode(0...22)( - [GlobalVariableWriteNode(0...11)((0...7), (8...9), IntegerNode(10...11)()), + [GlobalVariableWriteNode(0...11)((0...7), IntegerNode(10...11)(), (8...9)), LocalVariableWriteNode(12...22)( :测试, 0, diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 78ab4d3d78d1a8..6fb76babca040c 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -1,7 +1,7 @@ ProgramNode(0...704)( [:a, :b, :foo, :c, :x], StatementsNode(0...704)( - [GlobalVariableWriteNode(0...6)((0...2), (3...4), IntegerNode(5...6)()), + [GlobalVariableWriteNode(0...6)((0...2), IntegerNode(5...6)(), (3...4)), MultiWriteNode(7...24)( [GlobalVariableTargetNode(8...10)(), GlobalVariableTargetNode(12...14)()], diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index 419942d1eb997e..11f622ce9b4221 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -28,8 +28,8 @@ ProgramNode(0...293)( ), GlobalVariableWriteNode(50...58)( (50...54), - (55...56), - IntegerNode(57...58)() + IntegerNode(57...58)(), + (55...56) ), GlobalVariableReadNode(60...64)(), InstanceVariableReadNode(66...70)(:@abc), @@ -57,12 +57,12 @@ ProgramNode(0...293)( ), GlobalVariableWriteNode(110...121)( (110...114), - (115...116), ArrayNode(117...121)( [IntegerNode(117...118)(), IntegerNode(120...121)()], nil, nil - ) + ), + (115...116) ), MultiWriteNode(123...137)( [InstanceVariableTargetNode(123...127)(:@foo), diff --git a/test/yarp/snapshots/whitequark/gvasgn.txt b/test/yarp/snapshots/whitequark/gvasgn.txt index d6a53275909939..c4030831e3f3f5 100644 --- a/test/yarp/snapshots/whitequark/gvasgn.txt +++ b/test/yarp/snapshots/whitequark/gvasgn.txt @@ -1,6 +1,6 @@ ProgramNode(0...9)( [], StatementsNode(0...9)( - [GlobalVariableWriteNode(0...9)((0...4), (5...6), IntegerNode(7...9)())] + [GlobalVariableWriteNode(0...9)((0...4), IntegerNode(7...9)(), (5...6))] ) ) diff --git a/yarp/config.yml b/yarp/config.yml index 8ebfb772bf4bbd..543a770dd10dbd 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -1221,10 +1221,10 @@ nodes: fields: - name: name_loc type: location - - name: operator_loc - type: location - name: value type: node + - name: operator_loc + type: location comment: | Represents writing to a global variable. From a21b5a943fe20892168eb76ecd1723191650da30 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 30 Aug 2023 22:48:17 +0200 Subject: [PATCH 201/208] [ruby/yarp] Move name_loc before value for LocalVariableWriteNode * Consistent with ClassVariableWriteNode, ConstantWriteNode, InstanceVariableWriteNode, GlobalVariableWriteNode. * Fixes desugaring of local variable with operators. https://github.com/ruby/yarp/commit/9a66737775 --- test/yarp/errors_test.rb | 20 +++++++++---------- test/yarp/snapshots/blocks.txt | 2 +- test/yarp/snapshots/classes.txt | 4 ++-- test/yarp/snapshots/dos_endings.txt | 4 ++-- test/yarp/snapshots/methods.txt | 8 ++++---- test/yarp/snapshots/modules.txt | 4 ++-- test/yarp/snapshots/rescue.txt | 2 +- test/yarp/snapshots/seattlerb/bug202.txt | 2 +- test/yarp/snapshots/seattlerb/dasgn_icky2.txt | 2 +- .../seattlerb/difficult1_line_numbers2.txt | 4 ++-- .../yarp/snapshots/seattlerb/difficult3_4.txt | 2 +- .../heredoc__backslash_dos_format.txt | 2 +- .../seattlerb/heredoc_bad_hex_escape.txt | 2 +- .../seattlerb/heredoc_bad_oct_escape.txt | 2 +- .../snapshots/seattlerb/heredoc_lineno.txt | 4 ++-- .../snapshots/seattlerb/heredoc_squiggly.txt | 2 +- ...squiggly_blank_line_plus_interpolation.txt | 2 +- .../heredoc_squiggly_blank_lines.txt | 2 +- .../seattlerb/heredoc_squiggly_interp.txt | 2 +- .../seattlerb/heredoc_squiggly_tabs.txt | 2 +- .../seattlerb/heredoc_squiggly_tabs_extra.txt | 2 +- .../heredoc_squiggly_visually_blank_lines.txt | 2 +- .../seattlerb/lasgn_arg_rescue_arg.txt | 2 +- .../lasgn_call_bracket_rescue_arg.txt | 2 +- .../lasgn_call_nobracket_rescue_arg.txt | 2 +- .../snapshots/seattlerb/lasgn_command.txt | 2 +- test/yarp/snapshots/seattlerb/lasgn_env.txt | 2 +- .../seattlerb/lasgn_lasgn_command_call.txt | 4 ++-- .../seattlerb/lasgn_middle_splat.txt | 2 +- .../seattlerb/magic_encoding_comment.txt | 2 +- .../snapshots/seattlerb/parse_line_block.txt | 2 +- .../seattlerb/parse_line_heredoc.txt | 2 +- .../parse_line_heredoc_regexp_chars.txt | 2 +- test/yarp/snapshots/seattlerb/pct_nl.txt | 2 +- .../seattlerb/safe_call_rhs_newline.txt | 2 +- test/yarp/snapshots/ternary_operator.txt | 2 +- .../unparser/corpus/literal/assignment.txt | 10 +++++----- .../snapshots/unparser/corpus/literal/if.txt | 8 ++++---- .../unparser/corpus/literal/kwbegin.txt | 2 +- .../unparser/corpus/literal/rescue.txt | 2 +- .../unparser/corpus/literal/send.txt | 2 +- .../unparser/corpus/literal/while.txt | 16 +++++++-------- .../unparser/corpus/semantic/while.txt | 8 ++++---- test/yarp/snapshots/variables.txt | 10 +++++----- test/yarp/snapshots/while.txt | 4 ++-- test/yarp/snapshots/whitequark/asgn_cmd.txt | 6 +++--- test/yarp/snapshots/whitequark/asgn_mrhs.txt | 6 +++--- .../class_definition_in_while_cond.txt | 4 ++-- test/yarp/snapshots/whitequark/lvasgn.txt | 2 +- .../snapshots/whitequark/parser_bug_507.txt | 2 +- .../snapshots/whitequark/rescue_mod_asgn.txt | 2 +- .../snapshots/whitequark/ruby_bug_12073.txt | 2 +- .../snapshots/whitequark/ruby_bug_12402.txt | 4 ++-- .../snapshots/whitequark/ruby_bug_12669.txt | 8 ++++---- .../snapshots/whitequark/ruby_bug_9669.txt | 2 +- .../whitequark/ternary_ambiguous_symbol.txt | 2 +- yarp/config.yml | 4 ++-- 57 files changed, 107 insertions(+), 107 deletions(-) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index f29bd746c81217..faf840e925f5bc 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -513,16 +513,16 @@ def test_cannot_assign_to_a_reserved_numbered_parameter expected = BeginNode( Location(), StatementsNode([ - LocalVariableWriteNode(:_1, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_2, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_3, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_4, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_5, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_6, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_7, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_8, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_9, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()), - LocalVariableWriteNode(:_10, 0, SymbolNode(Location(), Location(), nil, "a"), Location(), Location()) + LocalVariableWriteNode(:_1, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_2, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_3, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_4, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_5, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_6, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_7, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_8, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_9, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()), + LocalVariableWriteNode(:_10, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()) ]), nil, nil, diff --git a/test/yarp/snapshots/blocks.txt b/test/yarp/snapshots/blocks.txt index 8f836208ca81b9..4a2c3f7585eeff 100644 --- a/test/yarp/snapshots/blocks.txt +++ b/test/yarp/snapshots/blocks.txt @@ -453,8 +453,8 @@ ProgramNode(0...402)( LocalVariableWriteNode(292...300)( :fork, 0, - IntegerNode(299...300)(), (292...296), + IntegerNode(299...300)(), (297...298) ), CallNode(301...316)( diff --git a/test/yarp/snapshots/classes.txt b/test/yarp/snapshots/classes.txt index 33b3d042148f9b..1d8813f40e342f 100644 --- a/test/yarp/snapshots/classes.txt +++ b/test/yarp/snapshots/classes.txt @@ -11,8 +11,8 @@ ProgramNode(0...370)( [LocalVariableWriteNode(8...13)( :a, 0, - IntegerNode(12...13)(), (8...9), + IntegerNode(12...13)(), (10...11) )] ), @@ -63,8 +63,8 @@ ProgramNode(0...370)( [LocalVariableWriteNode(89...94)( :a, 0, - IntegerNode(93...94)(), (89...90), + IntegerNode(93...94)(), (91...92) )] ), diff --git a/test/yarp/snapshots/dos_endings.txt b/test/yarp/snapshots/dos_endings.txt index 240ae043a42477..7048028b2a193b 100644 --- a/test/yarp/snapshots/dos_endings.txt +++ b/test/yarp/snapshots/dos_endings.txt @@ -35,13 +35,14 @@ ProgramNode(0...108)( LocalVariableWriteNode(75...84)( :x, 0, - StringNode(79...84)((79...82), (82...82), (82...84), ""), (75...76), + StringNode(79...84)((79...82), (82...82), (82...84), ""), (77...78) ), LocalVariableWriteNode(88...108)( :a, 0, + (88...89), CallNode(92...108)( nil, nil, @@ -74,7 +75,6 @@ ProgramNode(0...108)( 0, "foo" ), - (88...89), (90...91) )] ) diff --git a/test/yarp/snapshots/methods.txt b/test/yarp/snapshots/methods.txt index b5ec66aae3d615..79d11f10381466 100644 --- a/test/yarp/snapshots/methods.txt +++ b/test/yarp/snapshots/methods.txt @@ -298,8 +298,8 @@ ProgramNode(0...1194)( LocalVariableWriteNode(275...280)( :a, 0, - IntegerNode(279...280)(), (275...276), + IntegerNode(279...280)(), (277...278) ), DefNode(282...291)( @@ -589,8 +589,8 @@ ProgramNode(0...1194)( [LocalVariableWriteNode(537...542)( :b, 0, - IntegerNode(541...542)(), (537...538), + IntegerNode(541...542)(), (539...540) )] ), @@ -839,6 +839,7 @@ ProgramNode(0...1194)( LocalVariableWriteNode(769...774)( :c, 0, + (769...770), CallNode(773...774)( nil, nil, @@ -850,7 +851,6 @@ ProgramNode(0...1194)( 2, "b" ), - (769...770), (771...772) ), (768...769), @@ -927,6 +927,7 @@ ProgramNode(0...1194)( LocalVariableWriteNode(833...838)( :a, 0, + (833...834), CallNode(837...838)( nil, nil, @@ -938,7 +939,6 @@ ProgramNode(0...1194)( 2, "b" ), - (833...834), (835...836) ), (832...833), diff --git a/test/yarp/snapshots/modules.txt b/test/yarp/snapshots/modules.txt index a3cf30b9ff8786..aa8c1a743d749b 100644 --- a/test/yarp/snapshots/modules.txt +++ b/test/yarp/snapshots/modules.txt @@ -9,8 +9,8 @@ ProgramNode(0...140)( [LocalVariableWriteNode(9...14)( :a, 0, - IntegerNode(13...14)(), (9...10), + IntegerNode(13...14)(), (11...12) )] ), @@ -62,8 +62,8 @@ ProgramNode(0...140)( [LocalVariableWriteNode(67...72)( :x, 0, - IntegerNode(71...72)(), (67...68), + IntegerNode(71...72)(), (69...70) )] ), diff --git a/test/yarp/snapshots/rescue.txt b/test/yarp/snapshots/rescue.txt index b8944afd19bcf2..c52a261fceee6f 100644 --- a/test/yarp/snapshots/rescue.txt +++ b/test/yarp/snapshots/rescue.txt @@ -192,6 +192,7 @@ ProgramNode(0...316)( LocalVariableWriteNode(217...235)( :a, 0, + (217...218), RescueModifierNode(221...235)( CallNode(221...224)( nil, @@ -207,7 +208,6 @@ ProgramNode(0...316)( (225...231), NilNode(232...235)() ), - (217...218), (219...220) ), StatementsNode(238...241)( diff --git a/test/yarp/snapshots/seattlerb/bug202.txt b/test/yarp/snapshots/seattlerb/bug202.txt index 64e666ef3f55fe..cbed9a6712f87e 100644 --- a/test/yarp/snapshots/seattlerb/bug202.txt +++ b/test/yarp/snapshots/seattlerb/bug202.txt @@ -5,8 +5,8 @@ ProgramNode(0...22)( LocalVariableWriteNode(12...22)( :测试, 0, - IntegerNode(21...22)(), (12...18), + IntegerNode(21...22)(), (19...20) )] ) diff --git a/test/yarp/snapshots/seattlerb/dasgn_icky2.txt b/test/yarp/snapshots/seattlerb/dasgn_icky2.txt index 2aba42cc791ac3..8fbbf8ec0ab0e3 100644 --- a/test/yarp/snapshots/seattlerb/dasgn_icky2.txt +++ b/test/yarp/snapshots/seattlerb/dasgn_icky2.txt @@ -15,8 +15,8 @@ ProgramNode(0...76)( [LocalVariableWriteNode(7...14)( :v, 0, - NilNode(11...14)(), (7...8), + NilNode(11...14)(), (9...10) ), BeginNode(17...72)( diff --git a/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt b/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt index 86433437ce1ea6..d19d6d817b6e7d 100644 --- a/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt +++ b/test/yarp/snapshots/seattlerb/difficult1_line_numbers2.txt @@ -21,8 +21,8 @@ ProgramNode(0...48)( LocalVariableWriteNode(24...29)( :b, 0, - IntegerNode(28...29)(), (24...25), + IntegerNode(28...29)(), (26...27) ), CallNode(32...35)( @@ -39,8 +39,8 @@ ProgramNode(0...48)( LocalVariableWriteNode(38...42)( :c, 0, - IntegerNode(41...42)(), (38...39), + IntegerNode(41...42)(), (40...41) )] ), diff --git a/test/yarp/snapshots/seattlerb/difficult3_4.txt b/test/yarp/snapshots/seattlerb/difficult3_4.txt index afca77c5fa310a..40ebfca6612174 100644 --- a/test/yarp/snapshots/seattlerb/difficult3_4.txt +++ b/test/yarp/snapshots/seattlerb/difficult3_4.txt @@ -4,6 +4,7 @@ ProgramNode(0...17)( [LocalVariableWriteNode(0...17)( :a, 0, + (0...1), IfNode(2...17)( nil, CallNode(2...3)(nil, nil, (2...3), nil, nil, nil, nil, 2, "b"), @@ -15,7 +16,6 @@ ProgramNode(0...17)( ), nil ), - (0...1), (1...2) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt b/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt index d8a5d9354f388a..befdff81ee76ce 100644 --- a/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt +++ b/test/yarp/snapshots/seattlerb/heredoc__backslash_dos_format.txt @@ -4,12 +4,12 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :str, 0, + (0...3), InterpolatedStringNode(6...12)( (6...12), [StringNode(14...30)(nil, (14...30), nil, "beforeafter\r\n")], (30...35) ), - (0...3), (4...5) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt b/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt index 4bc1846ef128c1..842da16d1d52f6 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_bad_hex_escape.txt @@ -4,12 +4,12 @@ ProgramNode(0...9)( [LocalVariableWriteNode(0...9)( :s, 0, + (0...1), InterpolatedStringNode(4...9)( (4...9), [StringNode(10...17)(nil, (10...17), nil, "a\xE9b\n")], (17...21) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt b/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt index ebfb3b4da35c24..14c02bdfb33c37 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_bad_oct_escape.txt @@ -4,12 +4,12 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :s, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...23)(nil, (11...23), nil, "a\xA7b\n" + "cöd\n")], (23...27) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_lineno.txt b/test/yarp/snapshots/seattlerb/heredoc_lineno.txt index 27708e25381e78..9bbc817c20f30d 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_lineno.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_lineno.txt @@ -4,6 +4,7 @@ ProgramNode(0...41)( [LocalVariableWriteNode(0...11)( :c, 0, + (0...1), InterpolatedStringNode(4...11)( (4...11), [StringNode(12...30)( @@ -14,14 +15,13 @@ ProgramNode(0...41)( )], (30...34) ), - (0...1), (2...3) ), LocalVariableWriteNode(35...41)( :d, 0, - IntegerNode(39...41)(), (35...36), + IntegerNode(39...41)(), (37...38) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt index a1013d9e84e231..6491c86c04c5aa 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly.txt @@ -4,12 +4,12 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), InterpolatedStringNode(4...12)( (4...12), [StringNode(13...25)(nil, (13...25), nil, "x\n" + "y\n" + "z\n")], (25...31) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt index e78764e926461f..f2c3c1b1a902b9 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt @@ -4,6 +4,7 @@ ProgramNode(0...20)( [LocalVariableWriteNode(0...20)( :a, 0, + (0...1), CallNode(4...20)( nil, nil, @@ -49,7 +50,6 @@ ProgramNode(0...20)( 0, "foo" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt index 3bb7807c5c6d6a..4b59c352461881 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_blank_lines.txt @@ -4,12 +4,12 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :a, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...20)(nil, (11...20), nil, "x\n" + "\n" + "z\n")], (20...24) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt index f895e997e7ca18..e03304a7d147b0 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_interp.txt @@ -4,6 +4,7 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :a, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...22)(nil, (11...22), nil, " w\n" + "x"), @@ -15,7 +16,6 @@ ProgramNode(0...10)( StringNode(27...36)(nil, (27...36), nil, " y\n" + " z\n")], (36...42) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt index 13d1844e3c7425..1592d1310476e9 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs.txt @@ -4,6 +4,7 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), InterpolatedStringNode(4...12)( (4...12), [StringNode(13...43)( @@ -14,7 +15,6 @@ ProgramNode(0...12)( )], (43...49) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt index da631943fceab6..63ff5887c9e25e 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_tabs_extra.txt @@ -4,6 +4,7 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), InterpolatedStringNode(4...12)( (4...12), [StringNode(13...37)( @@ -14,7 +15,6 @@ ProgramNode(0...12)( )], (37...43) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt b/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt index 680ffbe0a68dd0..866c5fac084cd5 100644 --- a/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt +++ b/test/yarp/snapshots/seattlerb/heredoc_squiggly_visually_blank_lines.txt @@ -4,12 +4,12 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...10)( :a, 0, + (0...1), InterpolatedStringNode(4...10)( (4...10), [StringNode(11...21)(nil, (11...21), nil, "x\n" + "\n" + "z\n")], (21...25) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt b/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt index 03ec05d3a348c0..ad36a5edb4eab8 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_arg_rescue_arg.txt @@ -4,12 +4,12 @@ ProgramNode(0...14)( [LocalVariableWriteNode(0...14)( :a, 0, + (0...1), RescueModifierNode(4...14)( IntegerNode(4...5)(), (6...12), IntegerNode(13...14)() ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt b/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt index c28b8c2178ac29..86469f1c9129e5 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_call_bracket_rescue_arg.txt @@ -4,6 +4,7 @@ ProgramNode(0...17)( [LocalVariableWriteNode(0...17)( :a, 0, + (0...1), RescueModifierNode(4...17)( CallNode(4...8)( nil, @@ -19,7 +20,6 @@ ProgramNode(0...17)( (9...15), IntegerNode(16...17)() ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt b/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt index bfb7c9b678854d..72e54e010160ce 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_call_nobracket_rescue_arg.txt @@ -4,6 +4,7 @@ ProgramNode(0...16)( [LocalVariableWriteNode(0...16)( :a, 0, + (0...1), CallNode(4...16)( nil, nil, @@ -21,7 +22,6 @@ ProgramNode(0...16)( 0, "b" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_command.txt b/test/yarp/snapshots/seattlerb/lasgn_command.txt index 696be7871609cc..b1912a7c6d22cf 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_command.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_command.txt @@ -4,6 +4,7 @@ ProgramNode(0...9)( [LocalVariableWriteNode(0...9)( :a, 0, + (0...1), CallNode(4...9)( CallNode(4...5)(nil, nil, (4...5), nil, nil, nil, nil, 2, "b"), (5...6), @@ -15,7 +16,6 @@ ProgramNode(0...9)( 0, "c" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_env.txt b/test/yarp/snapshots/seattlerb/lasgn_env.txt index 33f2f4701c53f5..cfd276b1b28557 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_env.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_env.txt @@ -4,8 +4,8 @@ ProgramNode(0...6)( [LocalVariableWriteNode(0...6)( :a, 0, - IntegerNode(4...6)(), (0...1), + IntegerNode(4...6)(), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt b/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt index 4c272c16808a7d..49f22eeb4311b1 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_lasgn_command_call.txt @@ -4,9 +4,11 @@ ProgramNode(0...11)( [LocalVariableWriteNode(0...11)( :a, 0, + (0...1), LocalVariableWriteNode(4...11)( :b, 0, + (4...5), CallNode(8...11)( nil, nil, @@ -18,10 +20,8 @@ ProgramNode(0...11)( 0, "c" ), - (4...5), (6...7) ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt b/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt index 623f6dc1257f95..acb6c844673f99 100644 --- a/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt +++ b/test/yarp/snapshots/seattlerb/lasgn_middle_splat.txt @@ -4,6 +4,7 @@ ProgramNode(0...12)( [LocalVariableWriteNode(0...12)( :a, 0, + (0...1), ArrayNode(4...12)( [CallNode(4...5)(nil, nil, (4...5), nil, nil, nil, nil, 2, "b"), SplatNode(7...9)( @@ -14,7 +15,6 @@ ProgramNode(0...12)( nil, nil ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt b/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt index 08c8d9ea3fca24..b8b1ec343d09c9 100644 --- a/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt +++ b/test/yarp/snapshots/seattlerb/magic_encoding_comment.txt @@ -16,8 +16,8 @@ ProgramNode(18...90)( [LocalVariableWriteNode(67...81)( :così, 0, - SymbolNode(75...81)((75...76), (76...81), nil, "però"), (67...72), + SymbolNode(75...81)((75...76), (76...81), nil, "però"), (73...74) )] ), diff --git a/test/yarp/snapshots/seattlerb/parse_line_block.txt b/test/yarp/snapshots/seattlerb/parse_line_block.txt index 51282de9773dc5..7ca169a86b5cec 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_block.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_block.txt @@ -4,8 +4,8 @@ ProgramNode(0...10)( [LocalVariableWriteNode(0...6)( :a, 0, - IntegerNode(4...6)(), (0...1), + IntegerNode(4...6)(), (2...3) ), CallNode(7...10)( diff --git a/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt b/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt index 8e41b1648bc6f3..955d94e98fb289 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_heredoc.txt @@ -4,6 +4,7 @@ ProgramNode(6...88)( [LocalVariableWriteNode(6...31)( :string, 0, + (6...12), CallNode(15...31)( InterpolatedStringNode(15...25)( (15...25), @@ -24,7 +25,6 @@ ProgramNode(6...88)( 0, "strip" ), - (6...12), (13...14) ), CallNode(77...88)( diff --git a/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt b/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt index b3ed88d90eb66d..3d52fcbff0cec7 100644 --- a/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt +++ b/test/yarp/snapshots/seattlerb/parse_line_heredoc_regexp_chars.txt @@ -4,6 +4,7 @@ ProgramNode(6...74)( [LocalVariableWriteNode(6...22)( :string, 0, + (6...12), InterpolatedStringNode(15...22)( (15...22), [StringNode(23...48)( @@ -14,7 +15,6 @@ ProgramNode(6...74)( )], (48...57) ), - (6...12), (13...14) ), CallNode(63...74)( diff --git a/test/yarp/snapshots/seattlerb/pct_nl.txt b/test/yarp/snapshots/seattlerb/pct_nl.txt index 8435c33e6abfb4..4cdba9d6d621ee 100644 --- a/test/yarp/snapshots/seattlerb/pct_nl.txt +++ b/test/yarp/snapshots/seattlerb/pct_nl.txt @@ -4,8 +4,8 @@ ProgramNode(0...7)( [LocalVariableWriteNode(0...7)( :x, 0, - StringNode(4...7)((4...6), (6...6), (6...7), ""), (0...1), + StringNode(4...7)((4...6), (6...6), (6...7), ""), (2...3) )] ) diff --git a/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt b/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt index 4000ecf59b70a1..7785c7921ece14 100644 --- a/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt +++ b/test/yarp/snapshots/seattlerb/safe_call_rhs_newline.txt @@ -4,6 +4,7 @@ ProgramNode(0...8)( [LocalVariableWriteNode(0...8)( :c, 0, + (0...1), CallNode(4...8)( CallNode(4...5)(nil, nil, (4...5), nil, nil, nil, nil, 2, "a"), (5...7), @@ -15,7 +16,6 @@ ProgramNode(0...8)( 1, "b" ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/ternary_operator.txt b/test/yarp/snapshots/ternary_operator.txt index 6055e2434ffb92..9751d53a317c21 100644 --- a/test/yarp/snapshots/ternary_operator.txt +++ b/test/yarp/snapshots/ternary_operator.txt @@ -147,8 +147,8 @@ ProgramNode(0...131)( [LocalVariableWriteNode(124...129)( :_a, 0, - IntegerNode(128...129)(), (124...126), + IntegerNode(128...129)(), (127...128) )] ), diff --git a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt index 6fb76babca040c..7680b90f618165 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/assignment.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/assignment.txt @@ -323,6 +323,7 @@ ProgramNode(0...704)( LocalVariableWriteNode(351...367)( :a, 0, + (351...352), ParenthesesNode(355...367)( StatementsNode(356...366)( [MultiWriteNode(356...366)( @@ -337,19 +338,19 @@ ProgramNode(0...704)( (355...356), (366...367) ), - (351...352), (353...354) ), LocalVariableWriteNode(368...373)( :a, 0, - IntegerNode(372...373)(), (368...369), + IntegerNode(372...373)(), (370...371) ), LocalVariableWriteNode(374...385)( :foo, 0, + (374...377), CallNode(380...385)( nil, nil, @@ -361,7 +362,6 @@ ProgramNode(0...704)( 0, "foo" ), - (374...377), (378...379) ), CallNode(386...395)( @@ -538,8 +538,8 @@ ProgramNode(0...704)( LocalVariableWriteNode(507...514)( :x, 0, - StringNode(511...514)((511...513), (513...513), (513...514), ""), (507...508), + StringNode(511...514)((511...513), (513...513), (513...514), ""), (509...510) ), CallNode(515...522)( @@ -615,6 +615,7 @@ ProgramNode(0...704)( LocalVariableWriteNode(562...576)( :x, 0, + (562...563), InterpolatedStringNode(566...576)( (566...576), [StringNode(577...579)(nil, (577...579), nil, " "), @@ -622,7 +623,6 @@ ProgramNode(0...704)( StringNode(582...583)(nil, (582...583), nil, "\n")], (583...591) ), - (562...563), (564...565) ), CallNode(591...605)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/if.txt b/test/yarp/snapshots/unparser/corpus/literal/if.txt index 1bfbe7e26ff5c5..77fb5c8c636042 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/if.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/if.txt @@ -61,6 +61,7 @@ ProgramNode(0...246)( [LocalVariableWriteNode(113...122)( :foo, 0, + (113...116), CallNode(119...122)( nil, nil, @@ -72,7 +73,6 @@ ProgramNode(0...246)( 2, "bar" ), - (113...116), (117...118) )] ), @@ -95,6 +95,7 @@ ProgramNode(0...246)( [LocalVariableWriteNode(146...155)( :foo, 0, + (146...149), CallNode(152...155)( nil, nil, @@ -106,7 +107,6 @@ ProgramNode(0...246)( 2, "bar" ), - (146...149), (150...151) )] ), @@ -134,6 +134,7 @@ ProgramNode(0...246)( [LocalVariableWriteNode(184...193)( :foo, 0, + (184...187), CallNode(190...193)( nil, nil, @@ -145,7 +146,6 @@ ProgramNode(0...246)( 2, "bar" ), - (184...187), (188...189) )] ), @@ -190,8 +190,8 @@ ProgramNode(0...246)( [LocalVariableWriteNode(225...236)( :pair, 0, - SymbolNode(232...236)((232...233), (233...236), nil, "foo"), (225...229), + SymbolNode(232...236)((232...233), (233...236), nil, "foo"), (230...231) ), LocalVariableReadNode(239...242)(:foo, 0)] diff --git a/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt b/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt index 7e9670dae50d94..1f28ad87b416b0 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/kwbegin.txt @@ -259,6 +259,7 @@ ProgramNode(0...530)( LocalVariableWriteNode(307...316)( :foo, 0, + (307...310), CallNode(313...316)( nil, nil, @@ -270,7 +271,6 @@ ProgramNode(0...530)( 2, "bar" ), - (307...310), (311...312) ) )] diff --git a/test/yarp/snapshots/unparser/corpus/literal/rescue.txt b/test/yarp/snapshots/unparser/corpus/literal/rescue.txt index ed653b5d1362e9..8898a6e1a0db24 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/rescue.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/rescue.txt @@ -29,6 +29,7 @@ ProgramNode(0...64)( LocalVariableWriteNode(37...64)( :x, 0, + (37...38), ParenthesesNode(41...64)( StatementsNode(42...63)( [RescueModifierNode(42...63)( @@ -65,7 +66,6 @@ ProgramNode(0...64)( (41...42), (63...64) ), - (37...38), (39...40) )] ) diff --git a/test/yarp/snapshots/unparser/corpus/literal/send.txt b/test/yarp/snapshots/unparser/corpus/literal/send.txt index 1d3cd313b706dc..7ec3a338ce9ffc 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/send.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/send.txt @@ -48,8 +48,8 @@ ProgramNode(0...999)( [LocalVariableWriteNode(48...57)( :local, 0, - IntegerNode(56...57)(), (48...53), + IntegerNode(56...57)(), (54...55) ), CallNode(60...69)( diff --git a/test/yarp/snapshots/unparser/corpus/literal/while.txt b/test/yarp/snapshots/unparser/corpus/literal/while.txt index 003c03d6147727..7fcd66a01e61d1 100644 --- a/test/yarp/snapshots/unparser/corpus/literal/while.txt +++ b/test/yarp/snapshots/unparser/corpus/literal/while.txt @@ -48,8 +48,8 @@ ProgramNode(0...620)( [LocalVariableWriteNode(43...52)( :foo, 0, - LocalVariableReadNode(49...52)(:bar, 0), (43...46), + LocalVariableReadNode(49...52)(:bar, 0), (47...48) )] ), @@ -101,6 +101,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(80...89)( :foo, 0, + (80...83), CallNode(86...89)( nil, nil, @@ -112,7 +113,6 @@ ProgramNode(0...620)( 2, "bar" ), - (80...83), (84...85) )] ), @@ -140,6 +140,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(123...132)( :foo, 0, + (123...126), CallNode(129...132)( nil, nil, @@ -151,7 +152,6 @@ ProgramNode(0...620)( 2, "bar" ), - (123...126), (127...128) )] ), @@ -174,6 +174,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(159...168)( :foo, 0, + (159...162), CallNode(165...168)( nil, nil, @@ -185,7 +186,6 @@ ProgramNode(0...620)( 2, "bar" ), - (159...162), (163...164) )] ), @@ -218,6 +218,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(209...218)( :foo, 0, + (209...212), CallNode(215...218)( nil, nil, @@ -229,7 +230,6 @@ ProgramNode(0...620)( 2, "bar" ), - (209...212), (213...214) )] ), @@ -286,6 +286,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(274...283)( :foo, 0, + (274...277), CallNode(280...283)( nil, nil, @@ -297,7 +298,6 @@ ProgramNode(0...620)( 2, "bar" ), - (274...277), (278...279) )] ), @@ -351,6 +351,7 @@ ProgramNode(0...620)( [LocalVariableWriteNode(345...354)( :foo, 0, + (345...348), CallNode(351...354)( nil, nil, @@ -362,7 +363,6 @@ ProgramNode(0...620)( 2, "bar" ), - (345...348), (349...350) )] ), @@ -382,6 +382,7 @@ ProgramNode(0...620)( LocalVariableWriteNode(371...402)( :x, 0, + (371...372), ParenthesesNode(375...402)( StatementsNode(376...401)( [WhileNode(376...401)( @@ -426,7 +427,6 @@ ProgramNode(0...620)( (375...376), (401...402) ), - (371...372), (373...374) ), WhileNode(403...428)( diff --git a/test/yarp/snapshots/unparser/corpus/semantic/while.txt b/test/yarp/snapshots/unparser/corpus/semantic/while.txt index c381369c414e36..d6d4198447aabe 100644 --- a/test/yarp/snapshots/unparser/corpus/semantic/while.txt +++ b/test/yarp/snapshots/unparser/corpus/semantic/while.txt @@ -47,6 +47,7 @@ ProgramNode(0...188)( [LocalVariableWriteNode(36...45)( :foo, 0, + (36...39), CallNode(42...45)( nil, nil, @@ -58,7 +59,6 @@ ProgramNode(0...188)( 2, "bar" ), - (36...39), (40...41) )] ), @@ -93,8 +93,8 @@ ProgramNode(0...188)( LocalVariableWriteNode(83...88)( :a, 0, - CallNode(87...88)(nil, nil, (87...88), nil, nil, nil, nil, 2, "b"), (83...84), + CallNode(87...88)(nil, nil, (87...88), nil, nil, nil, nil, 2, "b"), (85...86) ), StatementsNode(91...92)([LocalVariableReadNode(91...92)(:a, 0)]), @@ -145,6 +145,7 @@ ProgramNode(0...188)( [LocalVariableWriteNode(143...152)( :foo, 0, + (143...146), CallNode(149...152)( nil, nil, @@ -156,7 +157,6 @@ ProgramNode(0...188)( 2, "exp" ), - (143...146), (147...148) ), WhileNode(155...184)( @@ -167,6 +167,7 @@ ProgramNode(0...188)( [LocalVariableWriteNode(169...178)( :foo, 0, + (169...172), CallNode(175...178)( nil, nil, @@ -178,7 +179,6 @@ ProgramNode(0...188)( 2, "bar" ), - (169...172), (173...174) )] ), diff --git a/test/yarp/snapshots/variables.txt b/test/yarp/snapshots/variables.txt index 11f622ce9b4221..edd27690a1a2ec 100644 --- a/test/yarp/snapshots/variables.txt +++ b/test/yarp/snapshots/variables.txt @@ -43,8 +43,8 @@ ProgramNode(0...293)( LocalVariableWriteNode(85...92)( :abc, 0, - IntegerNode(91...92)(), (85...88), + IntegerNode(91...92)(), (89...90) ), MultiWriteNode(94...108)( @@ -85,30 +85,30 @@ ProgramNode(0...293)( LocalVariableWriteNode(152...159)( :foo, 0, - IntegerNode(158...159)(), (152...155), + IntegerNode(158...159)(), (156...157) ), LocalVariableWriteNode(161...171)( :foo, 0, + (161...164), ArrayNode(167...171)( [IntegerNode(167...168)(), IntegerNode(170...171)()], nil, nil ), - (161...164), (165...166) ), LocalVariableWriteNode(173...183)( :foo, 0, + (173...176), ArrayNode(179...183)( [IntegerNode(179...180)(), IntegerNode(182...183)()], nil, nil ), - (173...176), (177...178) ), MultiWriteNode(185...198)( @@ -177,6 +177,7 @@ ProgramNode(0...293)( LocalVariableWriteNode(260...270)( :foo, 0, + (260...263), ArrayNode(266...270)( [SplatNode(266...270)( (266...267), @@ -185,7 +186,6 @@ ProgramNode(0...293)( nil, nil ), - (260...263), (264...265) ), ConstantWriteNode(272...282)( diff --git a/test/yarp/snapshots/while.txt b/test/yarp/snapshots/while.txt index 667dd30359bd3d..4aa5fe85f1428d 100644 --- a/test/yarp/snapshots/while.txt +++ b/test/yarp/snapshots/while.txt @@ -123,6 +123,7 @@ ProgramNode(0...314)( [LocalVariableWriteNode(179...193)( :a, 0, + (179...180), CallNode(183...193)( nil, nil, @@ -134,7 +135,6 @@ ProgramNode(0...314)( 0, "tap" ), - (179...180), (181...182) )] ), @@ -182,6 +182,7 @@ ProgramNode(0...314)( [LocalVariableWriteNode(283...297)( :a, 0, + (283...284), CallNode(287...297)( nil, nil, @@ -193,7 +194,6 @@ ProgramNode(0...314)( 0, "tap" ), - (283...284), (285...286) )] ), diff --git a/test/yarp/snapshots/whitequark/asgn_cmd.txt b/test/yarp/snapshots/whitequark/asgn_cmd.txt index 218c28dc0f24ff..ef24cc0477f9e8 100644 --- a/test/yarp/snapshots/whitequark/asgn_cmd.txt +++ b/test/yarp/snapshots/whitequark/asgn_cmd.txt @@ -4,9 +4,11 @@ ProgramNode(0...30)( [LocalVariableWriteNode(0...17)( :foo, 0, + (0...3), LocalVariableWriteNode(6...17)( :bar, 0, + (6...9), CallNode(12...17)( nil, nil, @@ -18,15 +20,14 @@ ProgramNode(0...30)( 0, "m" ), - (6...9), (10...11) ), - (0...3), (4...5) ), LocalVariableWriteNode(19...30)( :foo, 0, + (19...22), CallNode(25...30)( nil, nil, @@ -38,7 +39,6 @@ ProgramNode(0...30)( 0, "m" ), - (19...22), (23...24) )] ) diff --git a/test/yarp/snapshots/whitequark/asgn_mrhs.txt b/test/yarp/snapshots/whitequark/asgn_mrhs.txt index fddbc439e5ae05..aaf2bedb6f340c 100644 --- a/test/yarp/snapshots/whitequark/asgn_mrhs.txt +++ b/test/yarp/snapshots/whitequark/asgn_mrhs.txt @@ -4,6 +4,7 @@ ProgramNode(0...41)( [LocalVariableWriteNode(0...10)( :foo, 0, + (0...3), ArrayNode(6...10)( [SplatNode(6...10)( (6...7), @@ -12,24 +13,24 @@ ProgramNode(0...41)( nil, nil ), - (0...3), (4...5) ), LocalVariableWriteNode(12...24)( :foo, 0, + (12...15), ArrayNode(18...24)( [CallNode(18...21)(nil, nil, (18...21), nil, nil, nil, nil, 2, "bar"), IntegerNode(23...24)()], nil, nil ), - (12...15), (16...17) ), LocalVariableWriteNode(26...41)( :foo, 0, + (26...29), ArrayNode(32...41)( [CallNode(32...35)(nil, nil, (32...35), nil, nil, nil, nil, 2, "baz"), SplatNode(37...41)( @@ -49,7 +50,6 @@ ProgramNode(0...41)( nil, nil ), - (26...29), (30...31) )] ) diff --git a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt index a3108d7060ca79..1e954972a8df0b 100644 --- a/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt +++ b/test/yarp/snapshots/whitequark/class_definition_in_while_cond.txt @@ -13,6 +13,7 @@ ProgramNode(0...197)( [LocalVariableWriteNode(21...35)( :a, 0, + (21...22), CallNode(25...35)( nil, nil, @@ -24,7 +25,6 @@ ProgramNode(0...197)( 0, "tap" ), - (21...22), (23...24) )] ), @@ -72,6 +72,7 @@ ProgramNode(0...197)( [LocalVariableWriteNode(120...134)( :a, 0, + (120...121), CallNode(124...134)( nil, nil, @@ -83,7 +84,6 @@ ProgramNode(0...197)( 0, "tap" ), - (120...121), (122...123) )] ), diff --git a/test/yarp/snapshots/whitequark/lvasgn.txt b/test/yarp/snapshots/whitequark/lvasgn.txt index 403c7f8042c9b2..bb762001ba0d17 100644 --- a/test/yarp/snapshots/whitequark/lvasgn.txt +++ b/test/yarp/snapshots/whitequark/lvasgn.txt @@ -4,8 +4,8 @@ ProgramNode(0...13)( [LocalVariableWriteNode(0...8)( :var, 0, - IntegerNode(6...8)(), (0...3), + IntegerNode(6...8)(), (4...5) ), LocalVariableReadNode(10...13)(:var, 0)] diff --git a/test/yarp/snapshots/whitequark/parser_bug_507.txt b/test/yarp/snapshots/whitequark/parser_bug_507.txt index 4d6d91a763f225..f305b7daeab39a 100644 --- a/test/yarp/snapshots/whitequark/parser_bug_507.txt +++ b/test/yarp/snapshots/whitequark/parser_bug_507.txt @@ -4,6 +4,7 @@ ProgramNode(0...19)( [LocalVariableWriteNode(0...19)( :m, 0, + (0...1), LambdaNode(4...19)( [:args], (4...6), @@ -25,7 +26,6 @@ ProgramNode(0...19)( ), nil ), - (0...1), (2...3) )] ) diff --git a/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt b/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt index c0c1b20cef787c..b0ca3f6c951009 100644 --- a/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt +++ b/test/yarp/snapshots/whitequark/rescue_mod_asgn.txt @@ -4,12 +4,12 @@ ProgramNode(0...21)( [LocalVariableWriteNode(0...21)( :foo, 0, + (0...3), RescueModifierNode(6...21)( CallNode(6...10)(nil, nil, (6...10), nil, nil, nil, nil, 2, "meth"), (11...17), CallNode(18...21)(nil, nil, (18...21), nil, nil, nil, nil, 2, "bar") ), - (0...3), (4...5) )] ) diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12073.txt b/test/yarp/snapshots/whitequark/ruby_bug_12073.txt index 7614f1b1201d99..8a3aee68a6998d 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12073.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12073.txt @@ -4,8 +4,8 @@ ProgramNode(0...49)( [LocalVariableWriteNode(0...5)( :a, 0, - IntegerNode(4...5)(), (0...1), + IntegerNode(4...5)(), (2...3) ), CallNode(7...13)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt index f2980c1d427a41..7bcef82926f8f0 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12402.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12402.txt @@ -72,6 +72,7 @@ ProgramNode(0...437)( LocalVariableWriteNode(59...85)( :foo, 0, + (59...62), CallNode(65...85)( nil, nil, @@ -99,12 +100,12 @@ ProgramNode(0...437)( 0, "raise" ), - (59...62), (63...64) ), LocalVariableWriteNode(87...114)( :foo, 0, + (87...90), RescueModifierNode(93...114)( CallNode(93...103)( nil, @@ -132,7 +133,6 @@ ProgramNode(0...437)( (104...110), NilNode(111...114)() ), - (87...90), (91...92) ), CallOperatorWriteNode(116...145)( diff --git a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt index f5ec48ce95ebb0..689a3d9a77a245 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_12669.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_12669.txt @@ -34,6 +34,7 @@ ProgramNode(0...74)( LocalVariableWriteNode(25...37)( :b, 0, + (25...26), CallNode(29...37)( nil, nil, @@ -47,7 +48,6 @@ ProgramNode(0...74)( 0, "raise" ), - (25...26), (27...28) ), :a, @@ -57,6 +57,7 @@ ProgramNode(0...74)( LocalVariableWriteNode(39...56)( :a, 0, + (39...40), LocalVariableOperatorWriteNode(43...56)( (43...44), (45...47), @@ -77,15 +78,16 @@ ProgramNode(0...74)( :+, 0 ), - (39...40), (41...42) ), LocalVariableWriteNode(58...74)( :a, 0, + (58...59), LocalVariableWriteNode(62...74)( :b, 0, + (62...63), CallNode(66...74)( nil, nil, @@ -99,10 +101,8 @@ ProgramNode(0...74)( 0, "raise" ), - (62...63), (64...65) ), - (58...59), (60...61) )] ) diff --git a/test/yarp/snapshots/whitequark/ruby_bug_9669.txt b/test/yarp/snapshots/whitequark/ruby_bug_9669.txt index 3cdaf914070f44..6302c3d317b073 100644 --- a/test/yarp/snapshots/whitequark/ruby_bug_9669.txt +++ b/test/yarp/snapshots/whitequark/ruby_bug_9669.txt @@ -25,6 +25,7 @@ ProgramNode(0...33)( LocalVariableWriteNode(21...33)( :o, 0, + (21...22), HashNode(25...33)( (25...26), [AssocNode(27...31)( @@ -34,7 +35,6 @@ ProgramNode(0...33)( )], (32...33) ), - (21...22), (23...24) )] ) diff --git a/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt b/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt index 245f8a07fe6656..3f77bfd3e5c30d 100644 --- a/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt +++ b/test/yarp/snapshots/whitequark/ternary_ambiguous_symbol.txt @@ -4,8 +4,8 @@ ProgramNode(0...13)( [LocalVariableWriteNode(0...3)( :t, 0, - IntegerNode(2...3)(), (0...1), + IntegerNode(2...3)(), (1...2) ), IfNode(4...13)( diff --git a/yarp/config.yml b/yarp/config.yml index 543a770dd10dbd..e65177302f455b 100644 --- a/yarp/config.yml +++ b/yarp/config.yml @@ -1595,10 +1595,10 @@ nodes: type: constant - name: depth type: uint32 - - name: value - type: node - name: name_loc type: location + - name: value + type: node - name: operator_loc type: location comment: | From 7f6407c356789db9039a0e45fbb8792236601956 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 30 Aug 2023 22:55:43 +0200 Subject: [PATCH 202/208] [ruby/yarp] Ensure node are present only once in the desugared AST https://github.com/ruby/yarp/commit/7b090bc23d --- test/yarp/desugar_visitor_test.rb | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/test/yarp/desugar_visitor_test.rb b/test/yarp/desugar_visitor_test.rb index 1ef2710c8e804e..3966d7bfcb6078 100644 --- a/test/yarp/desugar_visitor_test.rb +++ b/test/yarp/desugar_visitor_test.rb @@ -51,14 +51,36 @@ def ast_inspect(node) "(#{parts.join(" ")})" end + # Ensure every node is only present once in the AST. + # If the same node is present twice it would most likely indicate it is executed twice, which is invalid semantically. + # This also acts as a sanity check that Node#child_nodes returns only nodes or nil (which caught a couple bugs). + class EnsureEveryNodeOnceInAST < Visitor + def initialize + @all_nodes = {}.compare_by_identity + end + + def visit(node) + if node + if @all_nodes.include?(node) + raise "#{node.inspect} is present multiple times in the desugared AST and likely executed multiple times" + else + @all_nodes[node] = true + end + end + super(node) + end + end + def assert_desugars(expected, source) ast = YARP.parse(source).value.accept(DesugarVisitor.new) assert_equal expected, ast_inspect(ast.statements.body.last) + + ast.accept(EnsureEveryNodeOnceInAST.new) end def assert_not_desugared(source, reason) ast = YARP.parse(source).value - assert_equal_nodes(ast, ast.accept(YARP::DesugarVisitor.new)) + assert_equal_nodes(ast, ast.accept(DesugarVisitor.new)) end end end From 771576f02185b1044f23a451e3ac6b58494844dc Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Sun, 27 Aug 2023 19:29:35 -0400 Subject: [PATCH 203/208] Skip weak references to old objects in minor GC If we are in a minor GC and the object to mark is old, then the old object should already be marked and cannot be reclaimed in this GC cycle so we don't need to add it to the weak refences list. --- gc.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gc.c b/gc.c index 4c6fdbb3009f7e..0d2a9a4daa33e8 100644 --- a/gc.c +++ b/gc.c @@ -6903,6 +6903,16 @@ rb_gc_mark_weak(VALUE *ptr) GC_ASSERT(objspace->rgengc.parent_object == 0 || FL_TEST(objspace->rgengc.parent_object, FL_WB_PROTECTED)); + /* If we are in a minor GC and the other object is old, then obj should + * already be marked and cannot be reclaimed in this GC cycle so we don't + * need to add it to the weak refences list. */ + if (!is_full_marking(objspace) && RVALUE_OLD_P(obj)) { + GC_ASSERT(RVALUE_MARKED(obj)); + GC_ASSERT(!objspace->flags.during_compacting); + + return; + } + rgengc_check_relation(objspace, obj); rb_darray_append_without_gc(&objspace->weak_references, ptr); From 4f290bc3283accbec70ed2bbd613e40d88131220 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Fri, 1 Sep 2023 11:02:37 -0400 Subject: [PATCH 204/208] [YARP] Small fixes for existing ConstantNodes (#8346) Popped was slightly inaccurate for ConstantNodes and leading to issues if there was content after a ConstantNode. This fix doesn't pop any ConstantWriteNode values. --- yarp/yarp_compiler.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index f8fc004ce7afd8..07959b663c403d 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -624,11 +624,14 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, ADD_INSN(ret, &dummy_line_node, putnil); ADD_INSN1(ret, &dummy_line_node, putobject, Qtrue); ADD_INSN1(ret, &dummy_line_node, getconstant, ID2SYM(parse_node_symbol((yp_node_t *)constant_read_node))); + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); + } return; } case YP_NODE_CONSTANT_WRITE_NODE: { yp_constant_write_node_t *constant_write_node = (yp_constant_write_node_t *) node; - yp_compile_node(iseq, constant_write_node->value, ret, src, popped, compile_context); + yp_compile_node(iseq, constant_write_node->value, ret, src, false, compile_context); if (!popped) { ADD_INSN(ret, &dummy_line_node, dup); From 58e0a3699f3eadf5f34107f86ad2b654e4764730 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Fri, 1 Sep 2023 11:02:55 -0400 Subject: [PATCH 205/208] Fix YARP compiled send for popping (#8345) Prior to this commit, we were incorrectly popping callers and receivers on CallNodes. We shouldn't pop any of these as they might have side effects. --- yarp/yarp_compiler.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/yarp/yarp_compiler.c b/yarp/yarp_compiler.c index 07959b663c403d..cba528d2caaa44 100644 --- a/yarp/yarp_compiler.c +++ b/yarp/yarp_compiler.c @@ -495,14 +495,14 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, case YP_NODE_CALL_NODE: { yp_call_node_t *call_node = (yp_call_node_t *) node; - ID mid = parse_string_symbol(&call_node->name); + ID method_id = parse_string_symbol(&call_node->name); int flags = 0; int orig_argc = 0; if (call_node->receiver == NULL) { ADD_INSN(ret, &dummy_line_node, putself); } else { - yp_compile_node(iseq, call_node->receiver, ret, src, popped, compile_context); + yp_compile_node(iseq, call_node->receiver, ret, src, false, compile_context); } if (call_node->arguments == NULL) { @@ -511,7 +511,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } } else { yp_arguments_node_t *arguments = call_node->arguments; - yp_compile_node(iseq, (yp_node_t *) arguments, ret, src, popped, compile_context); + yp_compile_node(iseq, (yp_node_t *) arguments, ret, src, false, compile_context); orig_argc = (int)arguments->arguments.size; } @@ -523,7 +523,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, const rb_iseq_t *block_iseq = NEW_CHILD_ISEQ(&scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); ISEQ_COMPILE_DATA(iseq)->current_block = block_iseq; - ADD_SEND_WITH_BLOCK(ret, &dummy_line_node, mid, INT2FIX(orig_argc), block_iseq); + ADD_SEND_WITH_BLOCK(ret, &dummy_line_node, method_id, INT2FIX(orig_argc), block_iseq); } else { if (block_iseq == Qnil && flags == 0) { @@ -538,7 +538,10 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, } } - ADD_SEND_WITH_FLAG(ret, &dummy_line_node, mid, INT2NUM(orig_argc), INT2FIX(flags)); + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, method_id, INT2NUM(orig_argc), INT2FIX(flags)); + } + if (popped) { + ADD_INSN(ret, &dummy_line_node, pop); } return; } @@ -1323,7 +1326,7 @@ yp_compile_node(rb_iseq_t *iseq, const yp_node_t *node, LINK_ANCHOR *const ret, yp_statements_node_t *statements_node = (yp_statements_node_t *) node; yp_node_list_t node_list = statements_node->body; for (size_t index = 0; index < node_list.size - 1; index++) { - yp_compile_node(iseq, node_list.nodes[index], ret, src, !popped, compile_context); + yp_compile_node(iseq, node_list.nodes[index], ret, src, true, compile_context); } yp_compile_node(iseq, node_list.nodes[node_list.size - 1], ret, src, false, compile_context); return; From bead5396503175b6873d1b4e60bd8c8d2c82b772 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 16 Aug 2023 08:55:01 -0400 Subject: [PATCH 206/208] Incrementally mark even if we have free pages We move all pooled pages to free pages at the start of incremental marking, so we shouldn't run incremental marking only when we have run out of free pages. This causes incremental marking to always complete in a single step. --- gc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gc.c b/gc.c index 0d2a9a4daa33e8..2863240aa6e578 100644 --- a/gc.c +++ b/gc.c @@ -2392,7 +2392,7 @@ gc_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool, rb_heap_t *heap) gc_enter(objspace, gc_enter_event_continue, &lock_lev); /* Continue marking if in incremental marking. */ - if (heap->free_pages == NULL && is_incremental_marking(objspace)) { + if (is_incremental_marking(objspace)) { if (gc_marks_continue(objspace, size_pool, heap)) { gc_sweep(objspace); } From 512f8217cb378c289b7d79cdf033715afcf82667 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 1 Sep 2023 10:45:11 -0400 Subject: [PATCH 207/208] [ruby/yarp] fix: double-counting of errors in parsing escaped strings Essentially, this change updates `yp_unescape_calculate_difference` to not create syntax errors, and we rely entirely on `yp_unescape_manipulate_string` to report syntax errors. To do that, this PR adds another (!) parameter to `unescape`: `yp_list_t *error_list`. When present, `unescape` reports syntax errors (and otherwise does not). However, an edge case that needed to be addressed is reporting syntax errors in this case: ?\u{1234 2345} In a string context, it's possible to have multiple codepoints by doing something like `"\u{1234 2345}"`; however, in the character literal context, this is a syntax error -- only a single codepoint is allowed. Unfortunately, when `yp_unescape_manipulate_string` is called, there's nothing to indicate that we are in a "character literal" context and that only a single codepoint is valid. To make this work, this PR: - introduces a new static utility function in yarp.c, `yp_char_literal_node_create_and_unescape`, which is called when we're parsing `YP_TOKEN_CHARACTER_LITERAL` - introduces a new (unexported) function, `yp_unescape_manipulate_char_literal` which does the same thing as `yp_unescape_manipulate_string` but tells `unescape` that only a single codepoint is expected https://github.com/ruby/yarp/commit/f6a65840b5 --- test/yarp/errors_test.rb | 3 -- yarp/unescape.c | 94 +++++++++++++++++++++++++--------------- yarp/unescape.h | 4 +- yarp/yarp.c | 13 +++++- 4 files changed, 73 insertions(+), 41 deletions(-) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index faf840e925f5bc..caf5db650aae77 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -608,7 +608,6 @@ def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_nota assert_errors expected, '"\u{0000001}"', [ ["invalid Unicode escape.", 4..11], - ["invalid Unicode escape.", 4..11] ] end @@ -617,14 +616,12 @@ def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_charact assert_errors expected, '"\u{000z}"', [ ["unterminated Unicode escape", 7..7], - ["unterminated Unicode escape", 7..7] ] end def test_unterminated_unicode_brackets_should_be_a_syntax_error assert_errors expression('?\\u{3'), '?\\u{3', [ ["invalid Unicode escape.", 1..5], - ["invalid Unicode escape.", 1..5], ] end diff --git a/yarp/unescape.c b/yarp/unescape.c index 14c0faf2eb1edf..84f0095da177e7 100644 --- a/yarp/unescape.c +++ b/yarp/unescape.c @@ -156,7 +156,7 @@ unescape_unicode_write(uint8_t *dest, uint32_t value, const uint8_t *start, cons // If we get here, then the value is too big. This is an error, but we don't // want to just crash, so instead we'll add an error to the error list and put // in a replacement character instead. - yp_diagnostic_list_append(error_list, start, end, "Invalid Unicode escape sequence."); + if (error_list) yp_diagnostic_list_append(error_list, start, end, "Invalid Unicode escape sequence."); dest[0] = 0xEF; dest[1] = 0xBF; dest[2] = 0xBD; @@ -186,7 +186,15 @@ unescape_char(uint8_t value, const uint8_t flags) { // Read a specific escape sequence into the given destination. static const uint8_t * -unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t *backslash, const uint8_t *end, const uint8_t flags) { +unescape( + yp_parser_t *parser, + uint8_t *dest, + size_t *dest_length, + const uint8_t *backslash, + const uint8_t *end, + const uint8_t flags, + yp_list_t *error_list +) { switch (backslash[1]) { case 'a': case 'b': @@ -226,7 +234,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \unnnn Unicode character, where nnnn is exactly 4 hexadecimal digits ([0-9a-fA-F]) case 'u': { if ((flags & YP_UNESCAPE_FLAG_CONTROL) | (flags & YP_UNESCAPE_FLAG_META)) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Unicode escape sequence cannot be used with control or meta flags."); return backslash + 2; } @@ -243,12 +251,11 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \u{nnnn} character literal allows only 1-6 hexadecimal digits if (hexadecimal_length > 6) { - yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); + if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "invalid Unicode escape."); } - // there are not hexadecimal characters - if (hexadecimal_length == 0) { - yp_diagnostic_list_append(&parser->error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape"); + else if (hexadecimal_length == 0) { + if (error_list) yp_diagnostic_list_append(error_list, unicode_cursor, unicode_cursor + hexadecimal_length, "unterminated Unicode escape"); return unicode_cursor; } @@ -261,36 +268,36 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t uint32_t value; unescape_unicode(unicode_start, (size_t) (unicode_cursor - unicode_start), &value); if (dest) { - *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, &parser->error_list); + *dest_length += unescape_unicode_write(dest + *dest_length, value, unicode_start, unicode_cursor, error_list); } unicode_cursor += yp_strspn_whitespace(unicode_cursor, end - unicode_cursor); } // ?\u{nnnn} character literal should contain only one codepoint and cannot be like ?\u{nnnn mmmm} - if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) - yp_diagnostic_list_append(&parser->error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); - + if (flags & YP_UNESCAPE_FLAG_EXPECT_SINGLE && codepoints_count > 1) { + if (error_list) yp_diagnostic_list_append(error_list, extra_codepoints_start, unicode_cursor - 1, "Multiple codepoints at single character literal"); + } if (unicode_cursor < end && *unicode_cursor == '}') { unicode_cursor++; } else { - yp_diagnostic_list_append(&parser->error_list, backslash, unicode_cursor, "invalid Unicode escape."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, unicode_cursor, "invalid Unicode escape."); } + return unicode_cursor; } - - if ((backslash + 5) < end && yp_char_is_hexadecimal_digits(backslash + 2, 4)) { + else if ((backslash + 5) < end && yp_char_is_hexadecimal_digits(backslash + 2, 4)) { uint32_t value; unescape_unicode(backslash + 2, 4, &value); if (dest) { - *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, &parser->error_list); + *dest_length += unescape_unicode_write(dest + *dest_length, value, backslash + 2, backslash + 6, error_list); } return backslash + 6; } - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid Unicode escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid Unicode escape sequence"); return backslash + 2; } // \c\M-x meta control character, where x is an ASCII printable character @@ -298,18 +305,18 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \cx control character, where x is an ASCII printable character case 'c': if (backslash + 2 >= end) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_CONTROL) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); return backslash + 2; } switch (backslash[2]) { case '\\': - return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL); + return unescape(parser, dest, dest_length, backslash + 2, end, flags | YP_UNESCAPE_FLAG_CONTROL, error_list); case '?': if (dest) { dest[(*dest_length)++] = unescape_char(0x7f, flags); @@ -317,7 +324,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t return backslash + 3; default: { if (!char_is_ascii_printable(backslash[2])) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return backslash + 2; } @@ -331,23 +338,23 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \C-? delete, ASCII 7Fh (DEL) case 'C': if (backslash + 3 >= end) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_CONTROL) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Control escape sequence cannot be doubled."); return backslash + 2; } if (backslash[2] != '-') { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return backslash + 2; } switch (backslash[3]) { case '\\': - return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_CONTROL, error_list); case '?': if (dest) { dest[(*dest_length)++] = unescape_char(0x7f, flags); @@ -355,7 +362,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t return backslash + 4; default: if (!char_is_ascii_printable(backslash[3])) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid control escape sequence"); return backslash + 2; } @@ -369,22 +376,22 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \M-x meta character, where x is an ASCII printable character case 'M': { if (backslash + 3 >= end) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 1, "Invalid control escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 1, "Invalid control escape sequence"); return end; } if (flags & YP_UNESCAPE_FLAG_META) { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled."); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Meta escape sequence cannot be doubled."); return backslash + 2; } if (backslash[2] != '-') { - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid meta escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence"); return backslash + 2; } if (backslash[3] == '\\') { - return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META); + return unescape(parser, dest, dest_length, backslash + 3, end, flags | YP_UNESCAPE_FLAG_META, error_list); } if (char_is_ascii_printable(backslash[3])) { @@ -394,7 +401,7 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t return backslash + 4; } - yp_diagnostic_list_append(&parser->error_list, backslash, backslash + 2, "Invalid meta escape sequence"); + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid meta escape sequence"); return backslash + 3; } // \n @@ -448,8 +455,8 @@ unescape(yp_parser_t *parser, uint8_t *dest, size_t *dest_length, const uint8_t // \c\M-x same as above // \c? or \C-? delete, ASCII 7Fh (DEL) // -YP_EXPORTED_FUNCTION void -yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { +static void +yp_unescape_manipulate_string_or_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type, bool expect_single_codepoint) { if (unescape_type == YP_UNESCAPE_NONE) { // If we're not unescaping then we can reference the source directly. return; @@ -511,7 +518,13 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc // This is the only type of unescaping left. In this case we need to // handle all of the different unescapes. assert(unescape_type == YP_UNESCAPE_ALL); - cursor = unescape(parser, dest, &dest_length, backslash, end, YP_UNESCAPE_FLAG_NONE); + + uint8_t flags = YP_UNESCAPE_FLAG_NONE; + if (expect_single_codepoint) { + flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; + } + + cursor = unescape(parser, dest, &dest_length, backslash, end, flags, &parser->error_list); break; } @@ -539,6 +552,16 @@ yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unesc yp_string_owned_init(string, allocated, dest_length + ((size_t) (end - cursor))); } +YP_EXPORTED_FUNCTION void +yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { + yp_unescape_manipulate_string_or_char_literal(parser, string, unescape_type, false); +} + +void +yp_unescape_manipulate_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type) { + yp_unescape_manipulate_string_or_char_literal(parser, string, unescape_type, true); +} + // This function is similar to yp_unescape_manipulate_string, except it doesn't // actually perform any string manipulations. Instead, it calculates how long // the unescaped character is, and returns that value @@ -564,10 +587,11 @@ yp_unescape_calculate_difference(yp_parser_t *parser, const uint8_t *backslash, assert(unescape_type == YP_UNESCAPE_ALL); uint8_t flags = YP_UNESCAPE_FLAG_NONE; - if (expect_single_codepoint) + if (expect_single_codepoint) { flags |= YP_UNESCAPE_FLAG_EXPECT_SINGLE; + } - const uint8_t *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags); + const uint8_t *cursor = unescape(parser, NULL, 0, backslash, parser->end, flags, NULL); assert(cursor > backslash); return (size_t) (cursor - backslash); diff --git a/yarp/unescape.h b/yarp/unescape.h index bf8b7e83ec08e6..fb0df9fcb08145 100644 --- a/yarp/unescape.h +++ b/yarp/unescape.h @@ -29,9 +29,9 @@ typedef enum { YP_UNESCAPE_ALL } yp_unescape_type_t; -// Unescape the contents of the given token into the given string using the -// given unescape mode. +// Unescape the contents of the given token into the given string using the given unescape mode. YP_EXPORTED_FUNCTION void yp_unescape_manipulate_string(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type); +void yp_unescape_manipulate_char_literal(yp_parser_t *parser, yp_string_t *string, yp_unescape_type_t unescape_type); // Accepts a source string and a type of unescaping and returns the unescaped version. // The caller must yp_string_free(result); after calling this function. diff --git a/yarp/yarp.c b/yarp/yarp.c index 176e9f76b605ae..7e098daa48322c 100644 --- a/yarp/yarp.c +++ b/yarp/yarp.c @@ -7547,6 +7547,17 @@ yp_symbol_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *openin return node; } +static yp_string_node_t * +yp_char_literal_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *opening, const yp_token_t *content, const yp_token_t *closing, yp_unescape_type_t unescape_type) { + yp_string_node_t *node = yp_string_node_create(parser, opening, content, closing); + + assert((content->end - content->start) >= 0); + yp_string_shared_init(&node->unescaped, content->start, content->end); + + yp_unescape_manipulate_char_literal(parser, &node->unescaped, unescape_type); + return node; +} + static yp_string_node_t * yp_string_node_create_and_unescape(yp_parser_t *parser, const yp_token_t *opening, const yp_token_t *content, const yp_token_t *closing, yp_unescape_type_t unescape_type) { yp_string_node_t *node = yp_string_node_create(parser, opening, content, closing); @@ -10921,7 +10932,7 @@ parse_expression_prefix(yp_parser_t *parser, yp_binding_power_t binding_power) { yp_token_t closing = not_provided(parser); - return (yp_node_t *) yp_string_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_ALL); + return (yp_node_t *) yp_char_literal_node_create_and_unescape(parser, &opening, &content, &closing, YP_UNESCAPE_ALL); } case YP_TOKEN_CLASS_VARIABLE: { parser_lex(parser); From cfe1edddbf0512f029eaf8c6d68194d476120c16 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 1 Sep 2023 11:01:19 -0400 Subject: [PATCH 208/208] [ruby/yarp] fix: report syntax error for invalid hex escape Closes https://github.com/ruby/yarp/pull/1367 https://github.com/ruby/yarp/commit/b1ab54f526 --- test/yarp/errors_test.rb | 6 ++++++ yarp/unescape.c | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/yarp/errors_test.rb b/test/yarp/errors_test.rb index caf5db650aae77..2af3c605e4b4cf 100644 --- a/test/yarp/errors_test.rb +++ b/test/yarp/errors_test.rb @@ -603,6 +603,12 @@ def test_do_not_allow_multiple_codepoints_in_a_single_character_literal ] end + def test_invalid_hex_escape + assert_errors expression('"\\xx"'), '"\\xx"', [ + ["Invalid hex escape.", 1..3], + ] + end + def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_notation expected = StringNode(Location(), Location(), Location(), "\u0001") diff --git a/yarp/unescape.c b/yarp/unescape.c index 84f0095da177e7..830c5996ae380d 100644 --- a/yarp/unescape.c +++ b/yarp/unescape.c @@ -91,9 +91,10 @@ unescape_hexadecimal_digit(const uint8_t value) { // Scan the 1-2 digits of hexadecimal into the value. Returns the number of // digits scanned. static inline size_t -unescape_hexadecimal(const uint8_t *backslash, uint8_t *value, const uint8_t *end) { +unescape_hexadecimal(const uint8_t *backslash, uint8_t *value, const uint8_t *end, yp_list_t *error_list) { *value = 0; if (backslash + 2 >= end || !yp_char_is_hexadecimal_digit(backslash[2])) { + if (error_list) yp_diagnostic_list_append(error_list, backslash, backslash + 2, "Invalid hex escape."); return 2; } *value = unescape_hexadecimal_digit(backslash[2]); @@ -223,7 +224,7 @@ unescape( // \xnn hexadecimal bit pattern, where nn is 1-2 hexadecimal digits ([0-9a-fA-F]) case 'x': { uint8_t value; - const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value, end); + const uint8_t *cursor = backslash + unescape_hexadecimal(backslash, &value, end, error_list); if (dest) { dest[(*dest_length)++] = unescape_char(value, flags);