From 14ab698967cdaedc0a922a2bdf30dfc69bdba7eb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 08:32:58 -0400 Subject: [PATCH 01/58] [ruby/prism] Handle CLRF inside heredoc contents https://github.com/ruby/prism/commit/1fbac72485 --- prism/prism.c | 19 +++++++++++++++-- test/prism/ruby_parser_test.rb | 21 ++++--------------- test/prism/snapshots/dos_endings.txt | 4 ++-- test/prism/snapshots/heredoc_with_comment.txt | 2 +- .../heredoc__backslash_dos_format.txt | 2 +- ...c_with_carriage_return_escapes_windows.txt | 2 +- ...redoc_with_extra_carriage_horrible_mix.txt | 2 +- .../heredoc_with_extra_carriage_returns.txt | 2 +- ...oc_with_extra_carriage_returns_windows.txt | 2 +- ...on_and_carriage_return_escapes_windows.txt | 2 +- .../heredoc_with_only_carriage_returns.txt | 2 +- ...doc_with_only_carriage_returns_windows.txt | 2 +- 12 files changed, 32 insertions(+), 30 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 77cbcea2fe7a2d..a140dc734f0b5a 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -11267,11 +11267,11 @@ parser_lex(pm_parser_t *parser) { // Otherwise we'll be parsing string content. These are the places // where we need to split up the content of the heredoc. We'll use // strpbrk to find the first of these characters. - uint8_t breakpoints[] = "\n\\#"; + uint8_t breakpoints[] = "\r\n\\#"; pm_heredoc_quote_t quote = lex_mode->as.heredoc.quote; if (quote == PM_HEREDOC_QUOTE_SINGLE) { - breakpoints[2] = '\0'; + breakpoints[3] = '\0'; } const uint8_t *breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); @@ -11285,6 +11285,21 @@ parser_lex(pm_parser_t *parser) { parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; + case '\r': + parser->current.end = breakpoint + 1; + + if (peek_at(parser, breakpoint + 1) != '\n') { + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); + break; + } + + // If we hit a \r\n sequence, then we want to replace it + // with a single \n character in the final string. + pm_token_buffer_escape(parser, &token_buffer); + breakpoint++; + token_buffer.cursor = breakpoint; + + /* fallthrough */ case '\n': { if (parser->heredoc_end != NULL && (parser->heredoc_end > breakpoint)) { parser_flush_heredoc_end(parser); diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb index 952e493af9159c..e06b7ae4384bf4 100644 --- a/test/prism/ruby_parser_test.rb +++ b/test/prism/ruby_parser_test.rb @@ -52,25 +52,10 @@ class RubyParserTest < TestCase whitequark/string_concat.txt ] - # These files contain CRLF line endings, which ruby_parser translates into - # LF before it gets back to the node. This means the node actually has the - # wrong contents. - crlf = %w[ - dos_endings.txt - heredoc_with_comment.txt - seattlerb/heredoc__backslash_dos_format.txt - seattlerb/heredoc_with_carriage_return_escapes_windows.txt - seattlerb/heredoc_with_extra_carriage_horrible_mix.txt - seattlerb/heredoc_with_extra_carriage_returns_windows.txt - seattlerb/heredoc_with_extra_carriage_returns.txt - seattlerb/heredoc_with_interpolation_and_carriage_return_escapes_windows.txt - seattlerb/heredoc_with_only_carriage_returns_windows.txt - seattlerb/heredoc_with_only_carriage_returns.txt - ] - # https://github.com/seattlerb/ruby_parser/issues/344 - failures = crlf | %w[ + failures = %w[ alias.txt + dos_endings.txt heredocs_with_ignored_newlines.txt method_calls.txt methods.txt @@ -79,8 +64,10 @@ class RubyParserTest < TestCase patterns.txt regex.txt seattlerb/and_multi.txt + seattlerb/heredoc__backslash_dos_format.txt seattlerb/heredoc_bad_hex_escape.txt seattlerb/heredoc_bad_oct_escape.txt + seattlerb/heredoc_with_extra_carriage_horrible_mix.txt spanning_heredoc_newlines.txt spanning_heredoc.txt tilde_heredocs.txt diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index ed75b8a52fdaa7..c5b962f2181cf3 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -48,7 +48,7 @@ │ ├── opening_loc: (7,0)-(7,4) = "<<-E" │ ├── content_loc: (8,0)-(11,0) = " 1 \\\r\n 2\r\n 3\r\n" │ ├── closing_loc: (11,0)-(12,0) = "E\r\n" - │ └── unescaped: " 1 2\r\n 3\r\n" + │ └── unescaped: " 1 2\n 3\n" ├── @ LocalVariableWriteNode (location: (13,0)-(15,0)) │ ├── name: :x │ ├── depth: 0 @@ -94,7 +94,7 @@ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (19,0)-(20,0) = " baz\r\n" │ │ │ │ ├── closing_loc: ∅ - │ │ │ │ └── unescaped: "baz\r\n" + │ │ │ │ └── unescaped: "baz\n" │ │ │ └── closing_loc: (20,0)-(21,0) = " EOF\r\n" │ │ ├── call_operator_loc: (17,14)-(17,15) = "." │ │ ├── name: :chop diff --git a/test/prism/snapshots/heredoc_with_comment.txt b/test/prism/snapshots/heredoc_with_comment.txt index 117fdc117a3782..f2225ca981373d 100644 --- a/test/prism/snapshots/heredoc_with_comment.txt +++ b/test/prism/snapshots/heredoc_with_comment.txt @@ -11,7 +11,7 @@ │ ├── opening_loc: (1,0)-(1,9) = "<<-TARGET" │ ├── content_loc: (2,0)-(3,0) = " content makes for an obvious error\r\n" │ ├── closing_loc: (3,0)-(3,6) = "TARGET" - │ └── unescaped: " content makes for an obvious error\r\n" + │ └── unescaped: " content makes for an obvious error\n" ├── call_operator_loc: (1,9)-(1,10) = "." ├── name: :chomp ├── message_loc: (1,10)-(1,15) = "chomp" diff --git a/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt b/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt index 6ba437e36a907b..353e4c6964ceec 100644 --- a/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt +++ b/test/prism/snapshots/seattlerb/heredoc__backslash_dos_format.txt @@ -13,5 +13,5 @@ │ ├── opening_loc: (1,6)-(1,12) = "<<-XXX" │ ├── content_loc: (2,0)-(4,0) = "before\\\r\nafter\r\n" │ ├── closing_loc: (4,0)-(5,0) = "XXX\r\n" - │ └── unescaped: "beforeafter\r\n" + │ └── unescaped: "beforeafter\n" └── operator_loc: (1,4)-(1,5) = "=" diff --git a/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt b/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt index 21802c5707e6eb..2ef6763389d3c6 100644 --- a/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt +++ b/test/prism/snapshots/seattlerb/heredoc_with_carriage_return_escapes_windows.txt @@ -8,4 +8,4 @@ ├── opening_loc: (1,0)-(1,5) = "< Date: Mon, 25 Mar 2024 08:41:57 -0400 Subject: [PATCH 02/58] [ruby/prism] Handle CLRF inside string contents https://github.com/ruby/prism/commit/aac606301e --- prism/prism.c | 52 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index a140dc734f0b5a..9f0ecdb93845b5 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -340,10 +340,10 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed // These are the places where we need to split up the content of the // string. We'll use strpbrk to find the first of these characters. uint8_t *breakpoints = lex_mode.as.string.breakpoints; - memcpy(breakpoints, "\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); + memcpy(breakpoints, "\r\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); // Now add in the terminator. - size_t index = 2; + size_t index = 3; breakpoints[index++] = terminator; // If interpolation is allowed, then we're going to check for the # @@ -11042,29 +11042,43 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_END); } - // When we hit a newline, we need to flush any potential heredocs. Note - // that this has to happen after we check for the terminator in case the - // terminator is a newline character. - if (*breakpoint == '\n') { - if (parser->heredoc_end == NULL) { - pm_newline_list_append(&parser->newline_list, breakpoint); - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); - continue; - } else { - parser->current.end = breakpoint + 1; - parser_flush_heredoc_end(parser); - pm_token_buffer_flush(parser, &token_buffer); - LEX(PM_TOKEN_STRING_CONTENT); - } - } - switch (*breakpoint) { case '\0': // Skip directly past the null character. parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); break; + case '\r': + if (peek_at(parser, breakpoint + 1) != '\n') { + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); + break; + } + + // If we hit a \r\n sequence, then we need to treat it + // as a newline. + parser->current.end = breakpoint + 1; + pm_token_buffer_escape(parser, &token_buffer); + breakpoint++; + token_buffer.cursor = breakpoint; + + /* fallthrough */ + case '\n': + // When we hit a newline, we need to flush any potential + // heredocs. Note that this has to happen after we check + // for the terminator in case the terminator is a + // newline character. + if (parser->heredoc_end == NULL) { + pm_newline_list_append(&parser->newline_list, breakpoint); + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, true); + break; + } + + parser->current.end = breakpoint + 1; + parser_flush_heredoc_end(parser); + pm_token_buffer_flush(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); case '\\': { // Here we hit escapes. parser->current.end = breakpoint + 1; From 86077fbcde05f4abd6b306ad0fcc88ee891f8e8b Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 09:24:35 -0400 Subject: [PATCH 03/58] [ruby/prism] Refactor regexp lexing to make it easier to support CLRF https://github.com/ruby/prism/commit/60805d85ca --- prism/prism.c | 239 ++++++++++++++++++------------------ test/prism/encoding_test.rb | 2 +- 2 files changed, 120 insertions(+), 121 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 9f0ecdb93845b5..6aa611624a2336 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -10783,36 +10783,6 @@ parser_lex(pm_parser_t *parser) { pm_regexp_token_buffer_t token_buffer = { 0 }; while (breakpoint != NULL) { - // If we hit a null byte, skip directly past it. - if (*breakpoint == '\0') { - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; - } - - // If we've hit a newline, then we need to track that in the - // list of newlines. - if (*breakpoint == '\n') { - // For the special case of a newline-terminated regular expression, we will pass - // through this branch twice -- once with PM_TOKEN_REGEXP_BEGIN and then again - // with PM_TOKEN_STRING_CONTENT. Let's avoid tracking the newline twice, by - // tracking it only in the REGEXP_BEGIN case. - if ( - !(lex_mode->as.regexp.terminator == '\n' && parser->current.type != PM_TOKEN_REGEXP_BEGIN) - && parser->heredoc_end == NULL - ) { - pm_newline_list_append(&parser->newline_list, breakpoint); - } - - if (lex_mode->as.regexp.terminator != '\n') { - // If the terminator is not a newline, then we can set - // the next breakpoint and continue. - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; - } - } - // If we hit the terminator, we need to determine what kind of // token to return. if (*breakpoint == lex_mode->as.regexp.terminator) { @@ -10832,9 +10802,17 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_CONTENT); } + // Check here if we need to track the newline. + size_t eol_length = match_eol_at(parser, breakpoint); + if (eol_length) { + parser->current.end = breakpoint + eol_length; + pm_newline_list_append(&parser->newline_list, parser->current.end - 1); + } else { + parser->current.end = breakpoint + 1; + } + // Since we've hit the terminator of the regular expression, // we now need to parse the options. - parser->current.end = breakpoint + 1; parser->current.end += pm_strspn_regexp_option(parser->current.end, parser->end - parser->current.end); lex_mode_pop(parser); @@ -10842,114 +10820,135 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_REGEXP_END); } - // If we hit escapes, then we need to treat the next token - // literally. In this case we'll skip past the next character + // If we've hit the incrementor, then we need to skip past it // and find the next breakpoint. - if (*breakpoint == '\\') { + if (*breakpoint && *breakpoint == lex_mode->as.regexp.incrementor) { parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + lex_mode->as.regexp.nesting++; + continue; + } - // If we've hit the end of the file, then break out of the - // loop by setting the breakpoint to NULL. - if (parser->current.end == parser->end) { - breakpoint = NULL; - continue; - } - - pm_regexp_token_buffer_escape(parser, &token_buffer); - uint8_t peeked = peek(parser); + switch (*breakpoint) { + case '\0': + // If we hit a null byte, skip directly past it. + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + case '\n': + // If we've hit a newline, then we need to track that in + // the list of newlines. + if (parser->heredoc_end == NULL) { + pm_newline_list_append(&parser->newline_list, breakpoint); + } - switch (peeked) { - case '\r': - parser->current.end++; - if (peek(parser) != '\n') { - if (lex_mode->as.regexp.terminator != '\r') { - pm_token_buffer_push_byte(&token_buffer.base, '\\'); - } - pm_regexp_token_buffer_push_byte(&token_buffer, '\r'); - pm_token_buffer_push_byte(&token_buffer.base, '\r'); - break; - } - /* fallthrough */ - case '\n': - if (parser->heredoc_end) { - // ... if we are on the same line as a heredoc, - // flush the heredoc and continue parsing after - // heredoc_end. - parser_flush_heredoc_end(parser); - pm_regexp_token_buffer_copy(parser, &token_buffer); - LEX(PM_TOKEN_STRING_CONTENT); - } else { - // ... else track the newline. - pm_newline_list_append(&parser->newline_list, parser->current.end); - } + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + case '\\': { + // If we hit escapes, then we need to treat the next + // token literally. In this case we'll skip past the + // next character and find the next breakpoint. + parser->current.end = breakpoint + 1; - parser->current.end++; - break; - case 'c': - case 'C': - case 'M': - case 'u': - case 'x': - escape_read(parser, &token_buffer.regexp_buffer, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP); + // If we've hit the end of the file, then break out of + // the loop by setting the breakpoint to NULL. + if (parser->current.end == parser->end) { + breakpoint = NULL; break; - default: - if (lex_mode->as.regexp.terminator == peeked) { - // Some characters when they are used as the - // terminator also receive an escape. They are - // enumerated here. - switch (peeked) { - case '$': case ')': case '*': case '+': - case '.': case '>': case '?': case ']': - case '^': case '|': case '}': + } + + pm_regexp_token_buffer_escape(parser, &token_buffer); + uint8_t peeked = peek(parser); + + switch (peeked) { + case '\r': + parser->current.end++; + if (peek(parser) != '\n') { + if (lex_mode->as.regexp.terminator != '\r') { pm_token_buffer_push_byte(&token_buffer.base, '\\'); - break; - default: - break; + } + pm_regexp_token_buffer_push_byte(&token_buffer, '\r'); + pm_token_buffer_push_byte(&token_buffer.base, '\r'); + break; + } + /* fallthrough */ + case '\n': + if (parser->heredoc_end) { + // ... if we are on the same line as a heredoc, + // flush the heredoc and continue parsing after + // heredoc_end. + parser_flush_heredoc_end(parser); + pm_regexp_token_buffer_copy(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); + } else { + // ... else track the newline. + pm_newline_list_append(&parser->newline_list, parser->current.end); } - pm_regexp_token_buffer_push_byte(&token_buffer, peeked); - pm_token_buffer_push_byte(&token_buffer.base, peeked); parser->current.end++; break; - } - - if (peeked < 0x80) pm_token_buffer_push_byte(&token_buffer.base, '\\'); - pm_regexp_token_buffer_push_escaped(&token_buffer, parser); - break; - } + case 'c': + case 'C': + case 'M': + case 'u': + case 'x': + escape_read(parser, &token_buffer.regexp_buffer, &token_buffer.base.buffer, PM_ESCAPE_FLAG_REGEXP); + break; + default: + if (lex_mode->as.regexp.terminator == peeked) { + // Some characters when they are used as the + // terminator also receive an escape. They are + // enumerated here. + switch (peeked) { + case '$': case ')': case '*': case '+': + case '.': case '>': case '?': case ']': + case '^': case '|': case '}': + pm_token_buffer_push_byte(&token_buffer.base, '\\'); + break; + default: + break; + } - token_buffer.base.cursor = parser->current.end; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; - } + pm_regexp_token_buffer_push_byte(&token_buffer, peeked); + pm_token_buffer_push_byte(&token_buffer.base, peeked); + parser->current.end++; + break; + } - // If we hit a #, then we will attempt to lex interpolation. - if (*breakpoint == '#') { - pm_token_type_t type = lex_interpolation(parser, breakpoint); + if (peeked < 0x80) pm_token_buffer_push_byte(&token_buffer.base, '\\'); + pm_regexp_token_buffer_push_escaped(&token_buffer, parser); + break; + } - if (type == PM_TOKEN_NOT_PROVIDED) { - // If we haven't returned at this point then we had - // something that looked like an interpolated class or - // instance variable like "#@" but wasn't actually. In - // this case we'll just skip to the next breakpoint. + token_buffer.base.cursor = parser->current.end; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - continue; + break; } + case '#': { + // If we hit a #, then we will attempt to lex + // interpolation. + pm_token_type_t type = lex_interpolation(parser, breakpoint); - if (type == PM_TOKEN_STRING_CONTENT) { - pm_regexp_token_buffer_flush(parser, &token_buffer); - } + if (type == PM_TOKEN_NOT_PROVIDED) { + // If we haven't returned at this point then we had + // something that looked like an interpolated class or + // instance variable like "#@" but wasn't actually. In + // this case we'll just skip to the next breakpoint. + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + } - LEX(type); - } + if (type == PM_TOKEN_STRING_CONTENT) { + pm_regexp_token_buffer_flush(parser, &token_buffer); + } - // If we've hit the incrementor, then we need to skip past it - // and find the next breakpoint. - assert(*breakpoint == lex_mode->as.regexp.incrementor); - parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - lex_mode->as.regexp.nesting++; - continue; + LEX(type); + } + default: + assert(false && "unreachable"); + break; + } } if (parser->current.end > parser->current.start) { diff --git a/test/prism/encoding_test.rb b/test/prism/encoding_test.rb index 649d05b874164b..2aee473ddf9901 100644 --- a/test/prism/encoding_test.rb +++ b/test/prism/encoding_test.rb @@ -344,7 +344,7 @@ def assert_encoding(encoding, name, range) next if ["/", "{"].include?(character) source = "# encoding: #{name}\n/(?##{character})/\n" - assert Prism.parse(source).success? + assert Prism.parse(source).success?, "Expected #{source.inspect} to parse successfully." end rescue RangeError source = "# encoding: #{name}\n\\x#{codepoint.to_s(16)}" From 53cc2723877f7794807684e31a530daca1a72ed6 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 09:27:27 -0400 Subject: [PATCH 04/58] [ruby/prism] Handle CLRF in regexp https://github.com/ruby/prism/commit/b96bada9ae --- prism/parser.h | 4 ++-- prism/prism.c | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/prism/parser.h b/prism/parser.h index b685fa377d7eac..7e4bb99197259e 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -173,7 +173,7 @@ typedef struct pm_lex_mode { * This is the character set that should be used to delimit the * tokens within the regular expression. */ - uint8_t breakpoints[6]; + uint8_t breakpoints[7]; } regexp; struct { @@ -206,7 +206,7 @@ typedef struct pm_lex_mode { * This is the character set that should be used to delimit the * tokens within the string. */ - uint8_t breakpoints[6]; + uint8_t breakpoints[7]; } string; struct { diff --git a/prism/prism.c b/prism/prism.c index 6aa611624a2336..58c70dba69cdc0 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -308,14 +308,14 @@ lex_mode_push_regexp(pm_parser_t *parser, uint8_t incrementor, uint8_t terminato // regular expression. We'll use strpbrk to find the first of these // characters. uint8_t *breakpoints = lex_mode.as.regexp.breakpoints; - memcpy(breakpoints, "\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); + memcpy(breakpoints, "\r\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); // First we'll add the terminator. - breakpoints[3] = terminator; + breakpoints[4] = terminator; // Next, if there is an incrementor, then we'll check for that as well. if (incrementor != '\0') { - breakpoints[4] = incrementor; + breakpoints[5] = incrementor; } return lex_mode_push(parser, lex_mode); @@ -10835,6 +10835,19 @@ parser_lex(pm_parser_t *parser) { parser->current.end = breakpoint + 1; breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); break; + case '\r': + if (peek_at(parser, breakpoint + 1) != '\n') { + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; + } + + parser->current.end = breakpoint + 1; + pm_regexp_token_buffer_escape(parser, &token_buffer); + breakpoint++; + token_buffer.base.cursor = breakpoint; + + /* fallthrough */ case '\n': // If we've hit a newline, then we need to track that in // the list of newlines. From a08954569f197312db4d6b217f1b8ba3441fc078 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 10:08:21 -0400 Subject: [PATCH 05/58] [ruby/prism] Fix up minimal build setting https://github.com/ruby/prism/commit/98c85c4acb --- prism/api_pack.c | 9 +++++++++ prism/defines.h | 21 +++++++++++++++++++++ prism/extension.c | 20 +++++++++++++++++--- prism/pack.h | 4 ++-- prism/parser.h | 2 +- prism/prettyprint.h | 4 ++-- prism/prism.c | 10 +++++----- test/prism/ruby_parser_test.rb | 3 +++ 8 files changed, 60 insertions(+), 13 deletions(-) diff --git a/prism/api_pack.c b/prism/api_pack.c index c9f0b18a397bc7..98509ae65ccae0 100644 --- a/prism/api_pack.c +++ b/prism/api_pack.c @@ -1,5 +1,12 @@ #include "prism/extension.h" +#ifdef PRISM_EXCLUDE_PACK + +void +Init_prism_pack(void) {} + +#else + static VALUE rb_cPrism; static VALUE rb_cPrismPack; static VALUE rb_cPrismPackDirective; @@ -265,3 +272,5 @@ Init_prism_pack(void) { pack_symbol = ID2SYM(rb_intern("pack")); unpack_symbol = ID2SYM(rb_intern("unpack")); } + +#endif diff --git a/prism/defines.h b/prism/defines.h index aca3c6dc088fcf..2fe73fe3d8cbd9 100644 --- a/prism/defines.h +++ b/prism/defines.h @@ -182,4 +182,25 @@ #endif #endif +/** + * If PRISM_BUILD_MINIMAL is defined, then we're going to define every possible + * switch that will turn off certain features of prism. + */ +#ifdef PRISM_BUILD_MINIMAL + /** Exclude the serialization API. */ + #define PRISM_EXCLUDE_SERIALIZATION + + /** Exclude the JSON serialization API. */ + #define PRISM_EXCLUDE_JSON + + /** Exclude the Array#pack parser API. */ + #define PRISM_EXCLUDE_PACK + + /** Exclude the prettyprint API. */ + #define PRISM_EXCLUDE_PRETTYPRINT + + /** Exclude the full set of encodings, using the minimal only. */ + #define PRISM_ENCODING_EXCLUDE_FULL +#endif + #endif diff --git a/prism/extension.c b/prism/extension.c index 7c8636e3dfc15c..88ba006ac2ec30 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -270,6 +270,8 @@ file_options(int argc, VALUE *argv, pm_string_t *input, pm_options_t *options) { } } +#ifndef PRISM_EXCLUDE_SERIALIZATION + /******************************************************************************/ /* Serializing the AST */ /******************************************************************************/ @@ -351,6 +353,8 @@ dump_file(int argc, VALUE *argv, VALUE self) { return value; } +#endif + /******************************************************************************/ /* Extracting values for the parse result */ /******************************************************************************/ @@ -1129,6 +1133,8 @@ profile_file(VALUE self, VALUE filepath) { return Qnil; } +#ifndef PRISM_EXCLUDE_PRETTYPRINT + /** * call-seq: * Debug::inspect_node(source) -> inspected @@ -1159,6 +1165,8 @@ inspect_node(VALUE self, VALUE source) { return string; } +#endif + /** * call-seq: * Debug::format_errors(source, colorize) -> String @@ -1349,8 +1357,6 @@ Init_prism(void) { rb_define_const(rb_cPrism, "VERSION", rb_str_new2(EXPECTED_PRISM_VERSION)); // First, the functions that have to do with lexing and parsing. - rb_define_singleton_method(rb_cPrism, "dump", dump, -1); - rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1); rb_define_singleton_method(rb_cPrism, "lex", lex, -1); rb_define_singleton_method(rb_cPrism, "lex_file", lex_file, -1); rb_define_singleton_method(rb_cPrism, "parse", parse, -1); @@ -1363,6 +1369,11 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrism, "parse_success?", parse_success_p, -1); rb_define_singleton_method(rb_cPrism, "parse_file_success?", parse_file_success_p, -1); +#ifndef PRISM_EXCLUDE_SERIALIZATION + rb_define_singleton_method(rb_cPrism, "dump", dump, -1); + rb_define_singleton_method(rb_cPrism, "dump_file", dump_file, -1); +#endif + // Next, the functions that will be called by the parser to perform various // internal tasks. We expose these to make them easier to test. VALUE rb_cPrismDebug = rb_define_module_under(rb_cPrism, "Debug"); @@ -1370,10 +1381,13 @@ Init_prism(void) { rb_define_singleton_method(rb_cPrismDebug, "integer_parse", integer_parse, 1); rb_define_singleton_method(rb_cPrismDebug, "memsize", memsize, 1); rb_define_singleton_method(rb_cPrismDebug, "profile_file", profile_file, 1); - rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); rb_define_singleton_method(rb_cPrismDebug, "format_errors", format_errors, 2); rb_define_singleton_method(rb_cPrismDebug, "static_inspect", static_inspect, -1); +#ifndef PRISM_EXCLUDE_PRETTYPRINT + rb_define_singleton_method(rb_cPrismDebug, "inspect_node", inspect_node, 1); +#endif + // Next, define the functions that are exposed through the private // Debug::Encoding class. rb_cPrismDebugEncoding = rb_define_class_under(rb_cPrismDebug, "Encoding", rb_cObject); diff --git a/prism/pack.h b/prism/pack.h index cfdc251fe6d589..0b0b4b19cc85d4 100644 --- a/prism/pack.h +++ b/prism/pack.h @@ -6,6 +6,8 @@ #ifndef PRISM_PACK_H #define PRISM_PACK_H +#include "prism/defines.h" + // We optionally support parsing String#pack templates. For systems that don't // want or need this functionality, it can be turned off with the // PRISM_EXCLUDE_PACK define. @@ -15,8 +17,6 @@ void pm_pack_parse(void); #else -#include "prism/defines.h" - #include #include diff --git a/prism/parser.h b/prism/parser.h index 7e4bb99197259e..f706a67de7c75d 100644 --- a/prism/parser.h +++ b/prism/parser.h @@ -6,8 +6,8 @@ #ifndef PRISM_PARSER_H #define PRISM_PARSER_H -#include "prism/ast.h" #include "prism/defines.h" +#include "prism/ast.h" #include "prism/encoding.h" #include "prism/options.h" #include "prism/util/pm_constant_pool.h" diff --git a/prism/prettyprint.h b/prism/prettyprint.h index ea11b4a24687b3..5a52b2b6b8eb43 100644 --- a/prism/prettyprint.h +++ b/prism/prettyprint.h @@ -6,14 +6,14 @@ #ifndef PRISM_PRETTYPRINT_H #define PRISM_PRETTYPRINT_H +#include "prism/defines.h" + #ifdef PRISM_EXCLUDE_PRETTYPRINT void pm_prettyprint(void); #else -#include "prism/defines.h" - #include #include "prism/ast.h" diff --git a/prism/prism.c b/prism/prism.c index 58c70dba69cdc0..6f2ab81e9ae3f3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -10842,9 +10842,9 @@ parser_lex(pm_parser_t *parser) { break; } - parser->current.end = breakpoint + 1; - pm_regexp_token_buffer_escape(parser, &token_buffer); breakpoint++; + parser->current.end = breakpoint; + pm_regexp_token_buffer_escape(parser, &token_buffer); token_buffer.base.cursor = breakpoint; /* fallthrough */ @@ -11069,9 +11069,9 @@ parser_lex(pm_parser_t *parser) { // If we hit a \r\n sequence, then we need to treat it // as a newline. - parser->current.end = breakpoint + 1; - pm_token_buffer_escape(parser, &token_buffer); breakpoint++; + parser->current.end = breakpoint; + pm_token_buffer_escape(parser, &token_buffer); token_buffer.cursor = breakpoint; /* fallthrough */ @@ -11321,8 +11321,8 @@ parser_lex(pm_parser_t *parser) { // If we hit a \r\n sequence, then we want to replace it // with a single \n character in the final string. - pm_token_buffer_escape(parser, &token_buffer); breakpoint++; + pm_token_buffer_escape(parser, &token_buffer); token_buffer.cursor = breakpoint; /* fallthrough */ diff --git a/test/prism/ruby_parser_test.rb b/test/prism/ruby_parser_test.rb index e06b7ae4384bf4..c9883574a68b31 100644 --- a/test/prism/ruby_parser_test.rb +++ b/test/prism/ruby_parser_test.rb @@ -68,6 +68,9 @@ class RubyParserTest < TestCase seattlerb/heredoc_bad_hex_escape.txt seattlerb/heredoc_bad_oct_escape.txt seattlerb/heredoc_with_extra_carriage_horrible_mix.txt + seattlerb/heredoc_with_extra_carriage_returns_windows.txt + seattlerb/heredoc_with_only_carriage_returns_windows.txt + seattlerb/heredoc_with_only_carriage_returns.txt spanning_heredoc_newlines.txt spanning_heredoc.txt tilde_heredocs.txt From 46bf6ae886dc14d5e3a76d53eb4f97375f7c03c5 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Mon, 25 Mar 2024 09:06:47 -0700 Subject: [PATCH 06/58] YJIT: Propagate Array, Hash, and String classes (#10323) --- bootstraptest/test_yjit.rb | 28 ++++++++++++++++++ class.c | 2 ++ common.mk | 1 + yjit.h | 2 ++ yjit/src/codegen.rs | 57 ++++++++++++++++++++++++++----------- yjit/src/core.rs | 53 +++++++++++++++++++++------------- yjit/src/invariants.rs | 58 ++++++++++++++++++++++++++++++++++++++ yjit/src/stats.rs | 4 ++- 8 files changed, 168 insertions(+), 37 deletions(-) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 3eef86e0a9f4b7..24253a10c9aa2c 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4725,3 +4725,31 @@ def test_cases(file, chain) test_cases(File, Enumerator::Chain) } + +# singleton class should invalidate Type::CString assumption +assert_equal 'foo', %q{ + def define_singleton(str, define) + if define + # Wrap a C method frame to avoid exiting JIT code on defineclass + [nil].reverse_each do + class << str + def +(_) + "foo" + end + end + end + end + "bar" + end + + def entry(define) + str = "" + # When `define` is false, #+ compiles to rb_str_plus() without a class guard. + # When the code is reused with `define` is true, the class of `str` is changed + # to a singleton class, so the block should be invalidated. + str + define_singleton(str, define) + end + + entry(false) + entry(true) +} diff --git a/class.c b/class.c index 46b8c1f74a5a6b..879c0ead2af2ed 100644 --- a/class.c +++ b/class.c @@ -29,6 +29,7 @@ #include "internal/variable.h" #include "ruby/st.h" #include "vm_core.h" +#include "yjit.h" /* Flags of T_CLASS * @@ -805,6 +806,7 @@ make_singleton_class(VALUE obj) FL_SET(klass, FL_SINGLETON); RBASIC_SET_CLASS(obj, klass); rb_singleton_class_attached(klass, obj); + rb_yjit_invalidate_no_singleton_class(orig_class); SET_METACLASS_OF(klass, METACLASS_OF(rb_class_real(orig_class))); return klass; diff --git a/common.mk b/common.mk index 21ceaa3f01279b..9dbca5b731da3f 100644 --- a/common.mk +++ b/common.mk @@ -3106,6 +3106,7 @@ class.$(OBJEXT): {$(VPATH)}vm_core.h class.$(OBJEXT): {$(VPATH)}vm_debug.h class.$(OBJEXT): {$(VPATH)}vm_opts.h class.$(OBJEXT): {$(VPATH)}vm_sync.h +class.$(OBJEXT): {$(VPATH)}yjit.h compar.$(OBJEXT): $(hdrdir)/ruby/ruby.h compar.$(OBJEXT): $(hdrdir)/ruby/version.h compar.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h diff --git a/yjit.h b/yjit.h index 46218a47d72d44..2f5317ad977b6e 100644 --- a/yjit.h +++ b/yjit.h @@ -46,6 +46,7 @@ void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned ins void rb_yjit_tracing_invalidate_all(void); void rb_yjit_show_usage(int help, int highlight, unsigned int width, int columns); void rb_yjit_lazy_push_frame(const VALUE *pc); +void rb_yjit_invalidate_no_singleton_class(VALUE klass); #else // !USE_YJIT @@ -68,6 +69,7 @@ static inline void rb_yjit_before_ractor_spawn(void) {} static inline void rb_yjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx) {} static inline void rb_yjit_tracing_invalidate_all(void) {} static inline void rb_yjit_lazy_push_frame(const VALUE *pc) {} +static inline void rb_yjit_invalidate_no_singleton_class(VALUE klass) {} #endif // #if USE_YJIT diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 1d9955260b2322..7d9085d4b30d4e 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -97,6 +97,9 @@ pub struct JITState { /// not been written to for the block to be valid. pub stable_constant_names_assumption: Option<*const ID>, + /// A list of classes that are not supposed to have a singleton class. + pub no_singleton_class_assumptions: Vec, + /// When true, the block is valid only when there is a total of one ractor running pub block_assumes_single_ractor: bool, @@ -125,6 +128,7 @@ impl JITState { method_lookup_assumptions: vec![], bop_assumptions: vec![], stable_constant_names_assumption: None, + no_singleton_class_assumptions: vec![], block_assumes_single_ractor: false, perf_map: Rc::default(), perf_stack: vec![], @@ -231,6 +235,20 @@ impl JITState { Some(()) } + /// Assume that objects of a given class will have no singleton class. + /// Return true if there has been no such singleton class since boot + /// and we can safely invalidate it. + pub fn assume_no_singleton_class(&mut self, asm: &mut Assembler, ocb: &mut OutlinedCb, klass: VALUE) -> bool { + if jit_ensure_block_entry_exit(self, asm, ocb).is_none() { + return false; // out of space, give up + } + if has_singleton_class_of(klass) { + return false; // we've seen a singleton class. disable the optimization to avoid an invalidation loop. + } + self.no_singleton_class_assumptions.push(klass); + true + } + fn get_cfp(&self) -> *mut rb_control_frame_struct { unsafe { get_ec_cfp(self.ec) } } @@ -1504,7 +1522,7 @@ fn gen_newarray( ); asm.stack_pop(n.as_usize()); - let stack_ret = asm.stack_push(Type::TArray); + let stack_ret = asm.stack_push(Type::CArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1527,7 +1545,7 @@ fn gen_duparray( vec![ary.into()], ); - let stack_ret = asm.stack_push(Type::TArray); + let stack_ret = asm.stack_push(Type::CArray); asm.mov(stack_ret, new_ary); Some(KeepCompiling) @@ -1547,7 +1565,7 @@ fn gen_duphash( // call rb_hash_resurrect(VALUE hash); let hash = asm.ccall(rb_hash_resurrect as *const u8, vec![hash.into()]); - let stack_ret = asm.stack_push(Type::THash); + let stack_ret = asm.stack_push(Type::CHash); asm.mov(stack_ret, hash); Some(KeepCompiling) @@ -2303,12 +2321,12 @@ fn gen_newhash( asm.cpop_into(new_hash); // x86 alignment asm.stack_pop(num.try_into().unwrap()); - let stack_ret = asm.stack_push(Type::THash); + let stack_ret = asm.stack_push(Type::CHash); asm.mov(stack_ret, new_hash); } else { // val = rb_hash_new(); let new_hash = asm.ccall(rb_hash_new as *const u8, vec![]); - let stack_ret = asm.stack_push(Type::THash); + let stack_ret = asm.stack_push(Type::CHash); asm.mov(stack_ret, new_hash); } @@ -2330,7 +2348,7 @@ fn gen_putstring( vec![EC, put_val.into(), 0.into()] ); - let stack_top = asm.stack_push(Type::TString); + let stack_top = asm.stack_push(Type::CString); asm.mov(stack_top, str_opnd); Some(KeepCompiling) @@ -2351,7 +2369,7 @@ fn gen_putchilledstring( vec![EC, put_val.into(), 1.into()] ); - let stack_top = asm.stack_push(Type::TString); + let stack_top = asm.stack_push(Type::CString); asm.mov(stack_top, str_opnd); Some(KeepCompiling) @@ -4493,8 +4511,18 @@ fn jit_guard_known_klass( let val_type = asm.ctx.get_opnd_type(insn_opnd); if val_type.known_class() == Some(known_klass) { - // We already know from type information that this is a match - return; + // Unless frozen, Array, Hash, and String objects may change their RBASIC_CLASS + // when they get a singleton class. Those types need invalidations. + if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&known_klass) } { + if jit.assume_no_singleton_class(asm, ocb, known_klass) { + // Speculate that this object will not have a singleton class, + // and invalidate the block in case it does. + return; + } + } else { + // We already know from type information that this is a match + return; + } } if unsafe { known_klass == rb_cNilClass } { @@ -4613,14 +4641,11 @@ fn jit_guard_known_klass( jit_chain_guard(JCC_JNE, jit, asm, ocb, max_chain_depth, counter); if known_klass == unsafe { rb_cString } { - // Upgrading to Type::CString here is incorrect. - // The guard we put only checks RBASIC_CLASS(obj), - // which adding a singleton class can change. We - // additionally need to know the string is frozen - // to claim Type::CString. - asm.ctx.upgrade_opnd_type(insn_opnd, Type::TString); + asm.ctx.upgrade_opnd_type(insn_opnd, Type::CString); } else if known_klass == unsafe { rb_cArray } { - asm.ctx.upgrade_opnd_type(insn_opnd, Type::TArray); + asm.ctx.upgrade_opnd_type(insn_opnd, Type::CArray); + } else if known_klass == unsafe { rb_cHash } { + asm.ctx.upgrade_opnd_type(insn_opnd, Type::CHash); } } } diff --git a/yjit/src/core.rs b/yjit/src/core.rs index 000e0c0be03314..7f6e7d47f9defd 100644 --- a/yjit/src/core.rs +++ b/yjit/src/core.rs @@ -52,20 +52,18 @@ pub enum Type { Flonum, ImmSymbol, - #[allow(unused)] - HeapSymbol, - TString, // An object with the T_STRING flag set, possibly an rb_cString CString, // An un-subclassed string of type rb_cString (can have instance vars in some cases) TArray, // An object with the T_ARRAY flag set, possibly an rb_cArray + CArray, // An un-subclassed array of type rb_cArray (can have instance vars in some cases) THash, // An object with the T_HASH flag set, possibly an rb_cHash + CHash, // An un-subclassed hash of type rb_cHash (can have instance vars in some cases) BlockParamProxy, // A special sentinel value indicating the block parameter should be read from // the current surrounding cfp // The context currently relies on types taking at most 4 bits (max value 15) - // to encode, so if we add two more, we will need to refactor the context, - // or we could remove HeapSymbol, which is currently unused. + // to encode, so if we add any more, we will need to refactor the context. } // Default initialization @@ -98,8 +96,11 @@ impl Type { // Core.rs can't reference rb_cString because it's linked by Rust-only tests. // But CString vs TString is only an optimisation and shouldn't affect correctness. #[cfg(not(test))] - if val.class_of() == unsafe { rb_cString } && val.is_frozen() { - return Type::CString; + match val.class_of() { + class if class == unsafe { rb_cArray } => return Type::CArray, + class if class == unsafe { rb_cHash } => return Type::CHash, + class if class == unsafe { rb_cString } => return Type::CString, + _ => {} } // We likewise can't reference rb_block_param_proxy, but it's again an optimisation; // we can just treat it as a normal Object. @@ -150,8 +151,9 @@ impl Type { match self { Type::UnknownHeap => true, Type::TArray => true, + Type::CArray => true, Type::THash => true, - Type::HeapSymbol => true, + Type::CHash => true, Type::TString => true, Type::CString => true, Type::BlockParamProxy => true, @@ -161,21 +163,17 @@ impl Type { /// Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) pub fn is_array(&self) -> bool { - matches!(self, Type::TArray) + matches!(self, Type::TArray | Type::CArray) } - /// Check if it's a T_HASH object + /// Check if it's a T_HASH object (both THash and CHash are T_HASH) pub fn is_hash(&self) -> bool { - matches!(self, Type::THash) + matches!(self, Type::THash | Type::CHash) } /// Check if it's a T_STRING object (both TString and CString are T_STRING) pub fn is_string(&self) -> bool { - match self { - Type::TString => true, - Type::CString => true, - _ => false, - } + matches!(self, Type::TString | Type::CString) } /// Returns an Option with the T_ value type if it is known, otherwise None @@ -186,9 +184,9 @@ impl Type { Type::False => Some(RUBY_T_FALSE), Type::Fixnum => Some(RUBY_T_FIXNUM), Type::Flonum => Some(RUBY_T_FLOAT), - Type::TArray => Some(RUBY_T_ARRAY), - Type::THash => Some(RUBY_T_HASH), - Type::ImmSymbol | Type::HeapSymbol => Some(RUBY_T_SYMBOL), + Type::TArray | Type::CArray => Some(RUBY_T_ARRAY), + Type::THash | Type::CHash => Some(RUBY_T_HASH), + Type::ImmSymbol => Some(RUBY_T_SYMBOL), Type::TString | Type::CString => Some(RUBY_T_STRING), Type::Unknown | Type::UnknownImm | Type::UnknownHeap => None, Type::BlockParamProxy => None, @@ -204,7 +202,9 @@ impl Type { Type::False => Some(rb_cFalseClass), Type::Fixnum => Some(rb_cInteger), Type::Flonum => Some(rb_cFloat), - Type::ImmSymbol | Type::HeapSymbol => Some(rb_cSymbol), + Type::ImmSymbol => Some(rb_cSymbol), + Type::CArray => Some(rb_cArray), + Type::CHash => Some(rb_cHash), Type::CString => Some(rb_cString), _ => None, } @@ -255,6 +255,16 @@ impl Type { return TypeDiff::Compatible(1); } + // A CArray is also a TArray. + if self == Type::CArray && dst == Type::TArray { + return TypeDiff::Compatible(1); + } + + // A CHash is also a THash. + if self == Type::CHash && dst == Type::THash { + return TypeDiff::Compatible(1); + } + // A CString is also a TString. if self == Type::CString && dst == Type::TString { return TypeDiff::Compatible(1); @@ -1644,6 +1654,9 @@ impl JITState { if let Some(idlist) = self.stable_constant_names_assumption { track_stable_constant_names_assumption(blockref, idlist); } + for klass in self.no_singleton_class_assumptions { + track_no_singleton_class_assumption(blockref, klass); + } blockref } diff --git a/yjit/src/invariants.rs b/yjit/src/invariants.rs index 59f7b70e201d8f..c4facb31dd00d9 100644 --- a/yjit/src/invariants.rs +++ b/yjit/src/invariants.rs @@ -53,6 +53,12 @@ pub struct Invariants { /// A map from a block to a set of IDs that it is assuming have not been /// redefined. block_constant_states: HashMap>, + + /// A map from a class to a set of blocks that assume objects of the class + /// will have no singleton class. When the set is empty, it means that + /// there has been a singleton class for the class after boot, so you cannot + /// assume no singleton class going forward. + no_singleton_classes: HashMap>, } /// Private singleton instance of the invariants global struct. @@ -69,6 +75,7 @@ impl Invariants { single_ractor: HashSet::new(), constant_state_blocks: HashMap::new(), block_constant_states: HashMap::new(), + no_singleton_classes: HashMap::new(), }); } } @@ -130,6 +137,23 @@ pub fn track_method_lookup_stability_assumption( .insert(uninit_block); } +/// Track that a block will assume that `klass` objects will have no singleton class. +pub fn track_no_singleton_class_assumption(uninit_block: BlockRef, klass: VALUE) { + Invariants::get_instance() + .no_singleton_classes + .entry(klass) + .or_default() + .insert(uninit_block); +} + +/// Returns true if we've seen a singleton class of a given class since boot. +pub fn has_singleton_class_of(klass: VALUE) -> bool { + Invariants::get_instance() + .no_singleton_classes + .get(&klass) + .map_or(false, |blocks| blocks.is_empty()) +} + // Checks rb_method_basic_definition_p and registers the current block for invalidation if method // lookup changes. // A "basic method" is one defined during VM boot, so we can use this to check assumptions based on @@ -391,6 +415,11 @@ pub fn block_assumptions_free(blockref: BlockRef) { if invariants.constant_state_blocks.is_empty() { invariants.constant_state_blocks.shrink_to_fit(); } + + // Remove tracking for blocks assumping no singleton class + for (_, blocks) in invariants.no_singleton_classes.iter_mut() { + blocks.remove(&blockref); + } } /// Callback from the opt_setinlinecache instruction in the interpreter. @@ -457,6 +486,35 @@ pub extern "C" fn rb_yjit_constant_ic_update(iseq: *const rb_iseq_t, ic: IC, ins }); } +/// Invalidate blocks that assume objects of a given class will have no singleton class. +#[no_mangle] +pub extern "C" fn rb_yjit_invalidate_no_singleton_class(klass: VALUE) { + // Skip tracking singleton classes during boot. Such objects already have a singleton class + // before entering JIT code, so they get rejected when they're checked for the first time. + if unsafe { INVARIANTS.is_none() } { + return; + } + + // We apply this optimization only to Array, Hash, and String for now. + if unsafe { [rb_cArray, rb_cHash, rb_cString].contains(&klass) } { + let no_singleton_classes = &mut Invariants::get_instance().no_singleton_classes; + match no_singleton_classes.get_mut(&klass) { + Some(blocks) => { + // Invalidate existing blocks and let has_singleton_class_of() + // return true when they are compiled again + for block in mem::take(blocks) { + invalidate_block_version(&block); + incr_counter!(invalidate_no_singleton_class); + } + } + None => { + // Let has_singleton_class_of() return true for this class + no_singleton_classes.insert(klass, HashSet::new()); + } + } + } +} + // Invalidate all generated code and patch C method return code to contain // logic for firing the c_return TracePoint event. Once rb_vm_barrier() // returns, all other ractors are pausing inside RB_VM_LOCK_ENTER(), which diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index c1315aad90c4c9..621854d48793b2 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -261,7 +261,7 @@ macro_rules! make_counters { /// The list of counters that are available without --yjit-stats. /// They are incremented only by `incr_counter!` and don't use `gen_counter_incr`. -pub const DEFAULT_COUNTERS: [Counter; 15] = [ +pub const DEFAULT_COUNTERS: [Counter; 16] = [ Counter::code_gc_count, Counter::compiled_iseq_entry, Counter::cold_iseq_entry, @@ -278,6 +278,7 @@ pub const DEFAULT_COUNTERS: [Counter; 15] = [ Counter::invalidate_ractor_spawn, Counter::invalidate_constant_state_bump, Counter::invalidate_constant_ic_fill, + Counter::invalidate_no_singleton_class, ]; /// Macro to increase a counter by name and count @@ -559,6 +560,7 @@ make_counters! { invalidate_ractor_spawn, invalidate_constant_state_bump, invalidate_constant_ic_fill, + invalidate_no_singleton_class, // Currently, it's out of the ordinary (might be impossible) for YJIT to leave gaps in // executable memory, so this should be 0. From 9cf754b648bc04f0c1e8f9274e6047ff25c1b3e3 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 22 Mar 2024 10:48:01 -0400 Subject: [PATCH 07/58] Fix --debug=gc_stress flag ruby_env_debug_option gets called after Init_gc_stress, so the --debug=gc_stress flag never works. --- debug.c | 9 ++++++--- gc.c | 27 ++++++--------------------- inits.c | 1 - internal/gc.h | 3 ++- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/debug.c b/debug.c index 755f27531668ab..e194472c1aaf9d 100644 --- a/debug.c +++ b/debug.c @@ -185,9 +185,9 @@ ruby_env_debug_option(const char *str, int len, void *arg) int ov; size_t retlen; unsigned long n; +#define NAME_MATCH(name) (len == sizeof(name) - 1 && strncmp(str, (name), len) == 0) #define SET_WHEN(name, var, val) do { \ - if (len == sizeof(name) - 1 && \ - strncmp(str, (name), len) == 0) { \ + if (NAME_MATCH(name)) { \ (var) = (val); \ return 1; \ } \ @@ -221,7 +221,10 @@ ruby_env_debug_option(const char *str, int len, void *arg) #define SET_WHEN_UINT(name, vals, num, req) \ if (NAME_MATCH_VALUE(name)) SET_UINT_LIST(name, vals, num); - SET_WHEN("gc_stress", *ruby_initial_gc_stress_ptr, Qtrue); + if (NAME_MATCH("gc_stress")) { + rb_gc_stress_set(Qtrue); + return 1; + } SET_WHEN("core", ruby_enable_coredump, 1); SET_WHEN("ci", ruby_on_ci, 1); if (NAME_MATCH_VALUE("rgengc")) { diff --git a/gc.c b/gc.c index 282e9df0cfd653..4b30bb171d6fd3 100644 --- a/gc.c +++ b/gc.c @@ -439,8 +439,6 @@ typedef struct { size_t oldmalloc_limit_min; size_t oldmalloc_limit_max; double oldmalloc_limit_growth_factor; - - VALUE gc_stress; } ruby_gc_params_t; static ruby_gc_params_t gc_params = { @@ -462,8 +460,6 @@ static ruby_gc_params_t gc_params = { GC_OLDMALLOC_LIMIT_MIN, GC_OLDMALLOC_LIMIT_MAX, GC_OLDMALLOC_LIMIT_GROWTH_FACTOR, - - FALSE, }; /* GC_DEBUG: @@ -1135,10 +1131,6 @@ RVALUE_AGE_SET(VALUE obj, int age) if (unless_objspace_vm) objspace = unless_objspace_vm->objspace; \ else /* return; or objspace will be warned uninitialized */ -#define ruby_initial_gc_stress gc_params.gc_stress - -VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress; - #define malloc_limit objspace->malloc_params.limit #define malloc_increase objspace->malloc_params.increase #define malloc_allocated_size objspace->malloc_params.allocated_size @@ -1389,7 +1381,6 @@ NO_SANITIZE("memory", static inline int is_pointer_to_heap(rb_objspace_t *objspa static size_t obj_memsize_of(VALUE obj, int use_all_types); static void gc_verify_internal_consistency(rb_objspace_t *objspace); -static void gc_stress_set(rb_objspace_t *objspace, VALUE flag); static VALUE gc_disable_no_rest(rb_objspace_t *); static double getrusage_time(void); @@ -3552,14 +3543,6 @@ Init_heap(void) finalizer_table = st_init_numtable(); } -void -Init_gc_stress(void) -{ - rb_objspace_t *objspace = &rb_objspace; - - gc_stress_set(objspace, ruby_initial_gc_stress); -} - typedef int each_obj_callback(void *, void *, size_t, void *); typedef int each_page_callback(struct heap_page *, void *); @@ -11163,9 +11146,11 @@ gc_stress_get(rb_execution_context_t *ec, VALUE self) return ruby_gc_stress_mode; } -static void -gc_stress_set(rb_objspace_t *objspace, VALUE flag) +void +rb_gc_stress_set(VALUE flag) { + rb_objspace_t *objspace = &rb_objspace; + objspace->flags.gc_stressful = RTEST(flag); objspace->gc_stress_mode = flag; } @@ -11173,8 +11158,8 @@ gc_stress_set(rb_objspace_t *objspace, VALUE flag) static VALUE gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) { - rb_objspace_t *objspace = &rb_objspace; - gc_stress_set(objspace, flag); + + rb_gc_stress_set(flag); return flag; } diff --git a/inits.c b/inits.c index 9ed104f369e02f..677a384f9a37c5 100644 --- a/inits.c +++ b/inits.c @@ -73,7 +73,6 @@ rb_call_inits(void) CALL(vm_trace); CALL(vm_stack_canary); CALL(ast); - CALL(gc_stress); CALL(shape); CALL(Prism); diff --git a/internal/gc.h b/internal/gc.h index f86af2659456b2..595dbb9ef6064f 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -189,7 +189,6 @@ typedef struct ractor_newobj_cache { } rb_ractor_newobj_cache_t; /* gc.c */ -extern VALUE *ruby_initial_gc_stress_ptr; extern int ruby_disable_gc; RUBY_ATTR_MALLOC void *ruby_mimmalloc(size_t size); void ruby_mimfree(void *ptr); @@ -224,6 +223,8 @@ void rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr); void rb_gc_ref_update_table_values_only(st_table *tbl); +void rb_gc_stress_set(VALUE flag); + #define rb_gc_mark_and_move_ptr(ptr) do { \ VALUE _obj = (VALUE)*(ptr); \ rb_gc_mark_and_move(&_obj); \ From eef272f1548b92ca53867c94f7cc38b1f35ef656 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 13:18:39 -0400 Subject: [PATCH 08/58] [ruby/prism] Mark inner parts of interpolated* nodes as frozen https://github.com/ruby/prism/commit/58a127cd5d --- prism/prism.c | 17 +++++++++++++++++ test/prism/snapshots/dos_endings.txt | 4 ++-- test/prism/snapshots/dstring.txt | 4 ++-- test/prism/snapshots/regex.txt | 8 ++++---- .../seattlerb/parse_line_evstr_after_break.txt | 2 +- .../snapshots/seattlerb/qsymbols_interp.txt | 2 +- .../seattlerb/str_lit_concat_bad_encodings.txt | 4 ++-- test/prism/snapshots/seattlerb/words_interp.txt | 2 +- test/prism/snapshots/spanning_heredoc.txt | 16 ++++++++-------- test/prism/snapshots/strings.txt | 8 ++++---- test/prism/snapshots/symbols.txt | 8 ++++---- .../unparser/corpus/literal/literal.txt | 14 +++++++------- .../snapshots/unparser/corpus/semantic/dstr.txt | 16 ++++++++-------- .../unparser/corpus/semantic/literal.txt | 2 +- .../whitequark/array_symbols_interp.txt | 2 +- .../snapshots/whitequark/array_words_interp.txt | 2 +- .../whitequark/non_lvar_injecting_match.txt | 2 +- .../prism/snapshots/whitequark/regex_interp.txt | 4 ++-- .../snapshots/whitequark/ruby_bug_11990.txt | 4 ++-- .../snapshots/whitequark/string_concat.txt | 2 +- .../snapshots/whitequark/xstring_interp.txt | 4 ++-- test/prism/snapshots/xstring.txt | 4 ++-- 22 files changed, 74 insertions(+), 57 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 6f2ab81e9ae3f3..f1f7fc052d059b 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4302,6 +4302,11 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (node->base.location.end < part->location.end) { node->base.location.end = part->location.end; } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); } @@ -4348,6 +4353,10 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ node->base.location.start = part->location.start; } + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4394,6 +4403,10 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.start = part->location.start; } + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4423,6 +4436,10 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index c5b962f2181cf3..5ac946b207e135 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -18,13 +18,13 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,9)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: (1,5)-(1,6) = "\"" │ │ │ │ ├── content_loc: (1,6)-(1,8) = "hi" │ │ │ │ ├── closing_loc: (1,8)-(1,9) = "\"" │ │ │ │ └── unescaped: "hi" │ │ │ └── @ StringNode (location: (2,5)-(2,12)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (2,5)-(2,6) = "\"" │ │ │ ├── content_loc: (2,6)-(2,11) = "there" │ │ │ ├── closing_loc: (2,11)-(2,12) = "\"" diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index ad395f8a8eb94b..64ab6e8a823bbe 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -39,13 +39,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,0)-(8,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" │ │ │ └── unescaped: "fo\no" │ │ └── @ StringNode (location: (8,3)-(9,2)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (8,3)-(8,4) = "\"" │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" │ │ ├── closing_loc: (9,1)-(9,2) = "\"" diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index 9e19bbb18d273c..d07ab8c5e71900 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -39,7 +39,7 @@ │ ├── opening_loc: (7,0)-(7,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,1)-(7,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (7,1)-(7,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -55,7 +55,7 @@ │ ├── opening_loc: (9,0)-(9,1) = "/" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (9,1)-(9,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (9,1)-(9,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -77,7 +77,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (9,10)-(9,11) = "}" │ │ └── @ StringNode (location: (9,11)-(9,15)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (9,11)-(9,15) = " ccc" │ │ ├── closing_loc: ∅ @@ -192,7 +192,7 @@ │ ├── opening_loc: (30,0)-(30,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (30,1)-(30,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (30,1)-(30,5) = "aaa " │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt index 9cb200e94bce5b..d8acbc05c49f07 100644 --- a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt +++ b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt @@ -7,7 +7,7 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,3)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,2) = "a" │ │ ├── closing_loc: (1,2)-(1,3) = "\"" diff --git a/test/prism/snapshots/seattlerb/qsymbols_interp.txt b/test/prism/snapshots/seattlerb/qsymbols_interp.txt index 23c39302475630..97bc6754ff5088 100644 --- a/test/prism/snapshots/seattlerb/qsymbols_interp.txt +++ b/test/prism/snapshots/seattlerb/qsymbols_interp.txt @@ -16,7 +16,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,6)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (1,5)-(1,6) = "b" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt index b841407cd8df2a..b31245a7f4f664 100644 --- a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt +++ b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt @@ -7,13 +7,13 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,62)) - │ │ ├── flags: forced_utf8_encoding + │ │ ├── flags: forced_utf8_encoding, frozen │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,61) = "\\xE3\\xD3\\x8B\\xE3\\x83\\xBC\\x83\\xE3\\x83\\xE3\\x82\\xB3\\xA3\\x82\\x99" │ │ ├── closing_loc: (1,61)-(1,62) = "\"" │ │ └── unescaped: "\xE3Ӌー\x83\xE3\x83コ\xA3\x82\x99" │ └── @ StringNode (location: (2,8)-(2,66)) - │ ├── flags: forced_utf8_encoding + │ ├── flags: forced_utf8_encoding, frozen │ ├── opening_loc: (2,8)-(2,9) = "\"" │ ├── content_loc: (2,9)-(2,65) = "\\xE3\\x83\\xB3\\xE3\\x83\\x8F\\xE3\\x82\\x9A\\xC3\\xBD;foo@bar.com" │ ├── closing_loc: (2,65)-(2,66) = "\"" diff --git a/test/prism/snapshots/seattlerb/words_interp.txt b/test/prism/snapshots/seattlerb/words_interp.txt index dfead7d353404b..e0b08bc075639a 100644 --- a/test/prism/snapshots/seattlerb/words_interp.txt +++ b/test/prism/snapshots/seattlerb/words_interp.txt @@ -19,7 +19,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,6)-(1,7) = "}" │ │ └── @ StringNode (location: (1,7)-(1,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,7)-(1,8) = "b" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/spanning_heredoc.txt b/test/prism/snapshots/spanning_heredoc.txt index 90297d2282c753..7c3db913333712 100644 --- a/test/prism/snapshots/spanning_heredoc.txt +++ b/test/prism/snapshots/spanning_heredoc.txt @@ -36,13 +36,13 @@ │ │ │ │ ├── opening_loc: (4,13)-(4,14) = "/" │ │ │ │ ├── parts: (length: 2) │ │ │ │ │ ├── @ StringNode (location: (4,14)-(4,16)) - │ │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ │ ├── flags: frozen │ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ │ ├── content_loc: (4,14)-(4,16) = "b\\" │ │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ │ └── unescaped: "b" │ │ │ │ │ └── @ StringNode (location: (7,0)-(7,1)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (7,0)-(7,1) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -223,13 +223,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (35,12)-(35,14)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (35,12)-(35,14) = "l\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "l\n" │ │ │ │ └── @ StringNode (location: (38,0)-(38,1)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (38,0)-(38,1) = "l" │ │ │ │ ├── closing_loc: ∅ @@ -299,13 +299,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "p\n" │ │ │ │ └── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ ├── closing_loc: ∅ @@ -331,13 +331,13 @@ │ │ │ ├── opening_loc: (53,5)-(53,6) = "/" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (53,6)-(53,7)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (53,6)-(53,7) = "\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "" │ │ │ │ └── @ StringNode (location: (55,0)-(55,6)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (55,0)-(55,6) = "(?)" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/strings.txt b/test/prism/snapshots/strings.txt index 40a05ce09d5bbf..471b371332c957 100644 --- a/test/prism/snapshots/strings.txt +++ b/test/prism/snapshots/strings.txt @@ -337,7 +337,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 3) │ │ │ │ ├── @ StringNode (location: (67,5)-(67,6)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (67,5)-(67,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -359,7 +359,7 @@ │ │ │ │ │ │ └── block: ∅ │ │ │ │ │ └── closing_loc: (67,9)-(67,10) = "}" │ │ │ │ └── @ StringNode (location: (67,10)-(67,11)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (67,10)-(67,11) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -495,13 +495,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (99,0)-(99,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (99,0)-(99,1) = "?" │ │ │ ├── content_loc: (99,1)-(99,2) = "a" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "a" │ │ └── @ StringNode (location: (99,3)-(99,6)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (99,3)-(99,4) = "\"" │ │ ├── content_loc: (99,4)-(99,5) = "a" │ │ ├── closing_loc: (99,5)-(99,6) = "\"" diff --git a/test/prism/snapshots/symbols.txt b/test/prism/snapshots/symbols.txt index c34be74b37d506..b6a854db0b2069 100644 --- a/test/prism/snapshots/symbols.txt +++ b/test/prism/snapshots/symbols.txt @@ -234,7 +234,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (39,5)-(39,6)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (39,5)-(39,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -262,7 +262,7 @@ │ │ │ │ │ │ └── value: 2 │ │ │ │ │ └── closing_loc: (39,14)-(39,15) = "}" │ │ │ │ └── @ StringNode (location: (39,15)-(39,16)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,15)-(39,16) = "c" │ │ │ │ ├── closing_loc: ∅ @@ -272,7 +272,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (39,17)-(39,18)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,17)-(39,18) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -287,7 +287,7 @@ │ │ │ │ │ └── value: 3 │ │ │ │ └── closing_loc: (39,21)-(39,22) = "}" │ │ │ └── @ StringNode (location: (39,22)-(39,23)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,22)-(39,23) = "f" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt index ba7dd70b5b717c..8ecf613e7ce9ad 100644 --- a/test/prism/snapshots/unparser/corpus/literal/literal.txt +++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt @@ -358,13 +358,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (28,0)-(28,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (28,0)-(28,1) = "\"" │ │ │ ├── content_loc: (28,1)-(28,4) = "foo" │ │ │ ├── closing_loc: (28,4)-(28,5) = "\"" │ │ │ └── unescaped: "foo" │ │ └── @ StringNode (location: (28,6)-(28,11)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (28,6)-(28,7) = "\"" │ │ ├── content_loc: (28,7)-(28,10) = "bar" │ │ ├── closing_loc: (28,10)-(28,11) = "\"" @@ -497,7 +497,7 @@ │ ├── opening_loc: (39,0)-(39,1) = "`" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (39,1)-(39,4)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,1)-(39,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -582,7 +582,7 @@ │ ├── opening_loc: (51,0)-(51,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (51,1)-(51,4)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (51,1)-(51,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -601,7 +601,7 @@ │ ├── opening_loc: (52,0)-(52,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (52,1)-(52,4)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (52,1)-(52,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -1156,7 +1156,7 @@ ├── opening_loc: (89,0)-(89,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (89,1)-(90,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (89,1)-(90,0) = " x\n" │ │ ├── closing_loc: ∅ @@ -1178,7 +1178,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (90,5)-(90,6) = "}" │ └── @ StringNode (location: (90,6)-(91,1)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (90,6)-(91,1) = "\n#" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt index 5ab954b6d4119d..c5337729692b09 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt @@ -429,7 +429,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (119,0)-(119,3)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (119,0)-(119,1) = "'" │ │ │ ├── content_loc: (119,1)-(119,2) = "a" │ │ │ ├── closing_loc: (119,2)-(119,3) = "'" @@ -447,19 +447,19 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (122,0)-(122,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (122,0)-(122,1) = "\"" │ │ │ ├── content_loc: (122,1)-(122,1) = "" │ │ │ ├── closing_loc: (122,1)-(122,2) = "\"" │ │ │ └── unescaped: "" │ │ ├── @ StringNode (location: (122,3)-(122,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (122,3)-(122,4) = "\"" │ │ │ ├── content_loc: (122,4)-(122,4) = "" │ │ │ ├── closing_loc: (122,4)-(122,5) = "\"" │ │ │ └── unescaped: "" │ │ └── @ StringNode (location: (122,6)-(122,8)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (122,6)-(122,7) = "\"" │ │ ├── content_loc: (122,7)-(122,7) = "" │ │ ├── closing_loc: (122,7)-(122,8) = "\"" @@ -487,7 +487,7 @@ │ │ │ │ └── closing_loc: (124,6)-(124,7) = "}" │ │ │ └── closing_loc: (124,7)-(124,8) = "\"" │ │ └── @ StringNode (location: (124,9)-(124,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (124,9)-(124,10) = "\"" │ │ ├── content_loc: (124,10)-(124,11) = "b" │ │ ├── closing_loc: (124,11)-(124,12) = "\"" @@ -512,7 +512,7 @@ │ │ │ │ └── name: :@a │ │ │ └── closing_loc: (125,5)-(125,6) = "\"" │ │ └── @ StringNode (location: (125,7)-(125,10)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (125,7)-(125,8) = "\"" │ │ ├── content_loc: (125,8)-(125,9) = "b" │ │ ├── closing_loc: (125,9)-(125,10) = "\"" @@ -537,7 +537,7 @@ │ │ │ │ └── name: :$a │ │ │ └── closing_loc: (126,5)-(126,6) = "\"" │ │ └── @ StringNode (location: (126,7)-(126,10)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (126,7)-(126,8) = "\"" │ │ ├── content_loc: (126,8)-(126,9) = "b" │ │ ├── closing_loc: (126,9)-(126,10) = "\"" @@ -562,7 +562,7 @@ │ │ │ └── name: :@@a │ │ └── closing_loc: (127,6)-(127,7) = "\"" │ └── @ StringNode (location: (127,8)-(127,11)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: (127,8)-(127,9) = "\"" │ ├── content_loc: (127,9)-(127,10) = "b" │ ├── closing_loc: (127,10)-(127,11) = "\"" diff --git a/test/prism/snapshots/unparser/corpus/semantic/literal.txt b/test/prism/snapshots/unparser/corpus/semantic/literal.txt index 59e02be64fcfd2..ef666890be7d34 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/literal.txt @@ -55,7 +55,7 @@ │ │ │ │ └── name: :@bar │ │ │ └── closing_loc: (11,9)-(11,10) = "}" │ │ └── @ StringNode (location: (11,10)-(11,13)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (11,10)-(11,13) = "baz" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_symbols_interp.txt b/test/prism/snapshots/whitequark/array_symbols_interp.txt index 7583ad8833e17d..2437486b43e250 100644 --- a/test/prism/snapshots/whitequark/array_symbols_interp.txt +++ b/test/prism/snapshots/whitequark/array_symbols_interp.txt @@ -41,7 +41,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (3,3)-(3,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,3)-(3,6) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_words_interp.txt b/test/prism/snapshots/whitequark/array_words_interp.txt index c2f23d2bf19d78..00fd05e7f7befc 100644 --- a/test/prism/snapshots/whitequark/array_words_interp.txt +++ b/test/prism/snapshots/whitequark/array_words_interp.txt @@ -63,7 +63,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,12)-(3,13) = "}" │ │ ├── @ StringNode (location: (3,13)-(3,16)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,13)-(3,16) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt index 2df571e8dc1890..584e997df2c6da 100644 --- a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt +++ b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt @@ -20,7 +20,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,4)-(1,5) = "}" │ │ └── @ StringNode (location: (1,5)-(1,18)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,5)-(1,18) = "(?bar)" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/regex_interp.txt b/test/prism/snapshots/whitequark/regex_interp.txt index ee8caf22b9ee55..0a6db4cfdf4b7b 100644 --- a/test/prism/snapshots/whitequark/regex_interp.txt +++ b/test/prism/snapshots/whitequark/regex_interp.txt @@ -8,7 +8,7 @@ ├── opening_loc: (1,0)-(1,1) = "/" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -30,7 +30,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/ruby_bug_11990.txt b/test/prism/snapshots/whitequark/ruby_bug_11990.txt index 43c04e55a23a15..abdb8d9ddd6ca6 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11990.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11990.txt @@ -18,13 +18,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (1,2)-(1,6)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (1,2)-(1,6) = "<<~E" │ │ │ ├── content_loc: (2,0)-(3,0) = " x\n" │ │ │ ├── closing_loc: (3,0)-(4,0) = "E\n" │ │ │ └── unescaped: "x\n" │ │ └── @ StringNode (location: (1,7)-(1,12)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (1,7)-(1,8) = "\"" │ │ ├── content_loc: (1,8)-(1,11) = " y" │ │ ├── closing_loc: (1,11)-(1,12) = "\"" diff --git a/test/prism/snapshots/whitequark/string_concat.txt b/test/prism/snapshots/whitequark/string_concat.txt index 38ea0c745dd664..1739b2f65bfed4 100644 --- a/test/prism/snapshots/whitequark/string_concat.txt +++ b/test/prism/snapshots/whitequark/string_concat.txt @@ -22,7 +22,7 @@ │ │ │ └── name: :@a │ │ └── closing_loc: (1,7)-(1,8) = "\"" │ └── @ StringNode (location: (1,9)-(1,14)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: (1,9)-(1,10) = "\"" │ ├── content_loc: (1,10)-(1,13) = "bar" │ ├── closing_loc: (1,13)-(1,14) = "\"" diff --git a/test/prism/snapshots/whitequark/xstring_interp.txt b/test/prism/snapshots/whitequark/xstring_interp.txt index e9543098c2b232..0676ea96837f72 100644 --- a/test/prism/snapshots/whitequark/xstring_interp.txt +++ b/test/prism/snapshots/whitequark/xstring_interp.txt @@ -7,7 +7,7 @@ ├── opening_loc: (1,0)-(1,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -29,7 +29,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt index 04b4cbf6ea7151..1a177026db6bc4 100644 --- a/test/prism/snapshots/xstring.txt +++ b/test/prism/snapshots/xstring.txt @@ -13,7 +13,7 @@ │ ├── opening_loc: (3,0)-(3,1) = "`" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (3,1)-(3,5)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,1)-(3,5) = "foo " │ │ │ ├── closing_loc: ∅ @@ -35,7 +35,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,10)-(3,11) = "}" │ │ └── @ StringNode (location: (3,11)-(3,15)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (3,11)-(3,15) = " baz" │ │ ├── closing_loc: ∅ From ff8f98f5c26febea9ca61278f167ad6582381855 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 13:21:13 -0400 Subject: [PATCH 09/58] [ruby/prism] Mark interpolated nodes as static literal https://github.com/ruby/prism/commit/d00977a9bd --- prism/prism.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/prism/prism.c b/prism/prism.c index f1f7fc052d059b..223a0a92dede62 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4281,6 +4281,7 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { { .type = PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = NULL, @@ -4307,6 +4308,10 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); } @@ -4327,6 +4332,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4357,6 +4363,10 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4380,6 +4390,7 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { { .type = PM_INTERPOLATED_SYMBOL_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4407,6 +4418,10 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4421,6 +4436,7 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { { .type = PM_INTERPOLATED_X_STRING_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end @@ -4440,6 +4456,10 @@ pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_no pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); } + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } From 453de8c2bc1392be86058510630fab1a55c2a265 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 15:08:21 -0400 Subject: [PATCH 10/58] [ruby/prism] Revert "Frozen parts" https://github.com/ruby/prism/commit/48f2e8c169 --- prism/prism.c | 37 ------------------- test/prism/snapshots/dos_endings.txt | 4 +- test/prism/snapshots/dstring.txt | 4 +- test/prism/snapshots/regex.txt | 8 ++-- .../parse_line_evstr_after_break.txt | 2 +- .../snapshots/seattlerb/qsymbols_interp.txt | 2 +- .../str_lit_concat_bad_encodings.txt | 4 +- .../snapshots/seattlerb/words_interp.txt | 2 +- test/prism/snapshots/spanning_heredoc.txt | 16 ++++---- test/prism/snapshots/strings.txt | 8 ++-- test/prism/snapshots/symbols.txt | 8 ++-- .../unparser/corpus/literal/literal.txt | 14 +++---- .../unparser/corpus/semantic/dstr.txt | 16 ++++---- .../unparser/corpus/semantic/literal.txt | 2 +- .../whitequark/array_symbols_interp.txt | 2 +- .../whitequark/array_words_interp.txt | 2 +- .../whitequark/non_lvar_injecting_match.txt | 2 +- .../snapshots/whitequark/regex_interp.txt | 4 +- .../snapshots/whitequark/ruby_bug_11990.txt | 4 +- .../snapshots/whitequark/string_concat.txt | 2 +- .../snapshots/whitequark/xstring_interp.txt | 4 +- test/prism/snapshots/xstring.txt | 4 +- 22 files changed, 57 insertions(+), 94 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 223a0a92dede62..6f2ab81e9ae3f3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4281,7 +4281,6 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { { .type = PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = NULL, @@ -4303,15 +4302,6 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (node->base.location.end < part->location.end) { node->base.location.end = part->location.end; } - - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); } @@ -4332,7 +4322,6 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4359,14 +4348,6 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_ node->base.location.start = part->location.start; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4390,7 +4371,6 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { { .type = PM_INTERPOLATED_SYMBOL_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4414,14 +4394,6 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.start = part->location.start; } - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -4436,7 +4408,6 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi *node = (pm_interpolated_x_string_node_t) { { .type = PM_INTERPOLATED_X_STRING_NODE, - .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end @@ -4452,14 +4423,6 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); - } - - if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { - pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); - } - pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index 5ac946b207e135..c5b962f2181cf3 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -18,13 +18,13 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,9)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: (1,5)-(1,6) = "\"" │ │ │ │ ├── content_loc: (1,6)-(1,8) = "hi" │ │ │ │ ├── closing_loc: (1,8)-(1,9) = "\"" │ │ │ │ └── unescaped: "hi" │ │ │ └── @ StringNode (location: (2,5)-(2,12)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (2,5)-(2,6) = "\"" │ │ │ ├── content_loc: (2,6)-(2,11) = "there" │ │ │ ├── closing_loc: (2,11)-(2,12) = "\"" diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index 64ab6e8a823bbe..ad395f8a8eb94b 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -39,13 +39,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,0)-(8,2)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" │ │ │ └── unescaped: "fo\no" │ │ └── @ StringNode (location: (8,3)-(9,2)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (8,3)-(8,4) = "\"" │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" │ │ ├── closing_loc: (9,1)-(9,2) = "\"" diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index d07ab8c5e71900..9e19bbb18d273c 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -39,7 +39,7 @@ │ ├── opening_loc: (7,0)-(7,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,1)-(7,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (7,1)-(7,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -55,7 +55,7 @@ │ ├── opening_loc: (9,0)-(9,1) = "/" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (9,1)-(9,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (9,1)-(9,5) = "aaa " │ │ │ ├── closing_loc: ∅ @@ -77,7 +77,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (9,10)-(9,11) = "}" │ │ └── @ StringNode (location: (9,11)-(9,15)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (9,11)-(9,15) = " ccc" │ │ ├── closing_loc: ∅ @@ -192,7 +192,7 @@ │ ├── opening_loc: (30,0)-(30,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (30,1)-(30,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (30,1)-(30,5) = "aaa " │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt index d8acbc05c49f07..9cb200e94bce5b 100644 --- a/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt +++ b/test/prism/snapshots/seattlerb/parse_line_evstr_after_break.txt @@ -7,7 +7,7 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,3)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,2) = "a" │ │ ├── closing_loc: (1,2)-(1,3) = "\"" diff --git a/test/prism/snapshots/seattlerb/qsymbols_interp.txt b/test/prism/snapshots/seattlerb/qsymbols_interp.txt index 97bc6754ff5088..23c39302475630 100644 --- a/test/prism/snapshots/seattlerb/qsymbols_interp.txt +++ b/test/prism/snapshots/seattlerb/qsymbols_interp.txt @@ -16,7 +16,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,6)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (1,5)-(1,6) = "b" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt index b31245a7f4f664..b841407cd8df2a 100644 --- a/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt +++ b/test/prism/snapshots/seattlerb/str_lit_concat_bad_encodings.txt @@ -7,13 +7,13 @@ ├── opening_loc: ∅ ├── parts: (length: 2) │ ├── @ StringNode (location: (1,0)-(1,62)) - │ │ ├── flags: forced_utf8_encoding, frozen + │ │ ├── flags: forced_utf8_encoding │ │ ├── opening_loc: (1,0)-(1,1) = "\"" │ │ ├── content_loc: (1,1)-(1,61) = "\\xE3\\xD3\\x8B\\xE3\\x83\\xBC\\x83\\xE3\\x83\\xE3\\x82\\xB3\\xA3\\x82\\x99" │ │ ├── closing_loc: (1,61)-(1,62) = "\"" │ │ └── unescaped: "\xE3Ӌー\x83\xE3\x83コ\xA3\x82\x99" │ └── @ StringNode (location: (2,8)-(2,66)) - │ ├── flags: forced_utf8_encoding, frozen + │ ├── flags: forced_utf8_encoding │ ├── opening_loc: (2,8)-(2,9) = "\"" │ ├── content_loc: (2,9)-(2,65) = "\\xE3\\x83\\xB3\\xE3\\x83\\x8F\\xE3\\x82\\x9A\\xC3\\xBD;foo@bar.com" │ ├── closing_loc: (2,65)-(2,66) = "\"" diff --git a/test/prism/snapshots/seattlerb/words_interp.txt b/test/prism/snapshots/seattlerb/words_interp.txt index e0b08bc075639a..dfead7d353404b 100644 --- a/test/prism/snapshots/seattlerb/words_interp.txt +++ b/test/prism/snapshots/seattlerb/words_interp.txt @@ -19,7 +19,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,6)-(1,7) = "}" │ │ └── @ StringNode (location: (1,7)-(1,8)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,7)-(1,8) = "b" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/spanning_heredoc.txt b/test/prism/snapshots/spanning_heredoc.txt index 7c3db913333712..90297d2282c753 100644 --- a/test/prism/snapshots/spanning_heredoc.txt +++ b/test/prism/snapshots/spanning_heredoc.txt @@ -36,13 +36,13 @@ │ │ │ │ ├── opening_loc: (4,13)-(4,14) = "/" │ │ │ │ ├── parts: (length: 2) │ │ │ │ │ ├── @ StringNode (location: (4,14)-(4,16)) - │ │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ │ ├── content_loc: (4,14)-(4,16) = "b\\" │ │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ │ └── unescaped: "b" │ │ │ │ │ └── @ StringNode (location: (7,0)-(7,1)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (7,0)-(7,1) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -223,13 +223,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (35,12)-(35,14)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (35,12)-(35,14) = "l\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "l\n" │ │ │ │ └── @ StringNode (location: (38,0)-(38,1)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (38,0)-(38,1) = "l" │ │ │ │ ├── closing_loc: ∅ @@ -299,13 +299,13 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "p\n" │ │ │ │ └── @ StringNode (location: (48,12)-(48,14)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (48,12)-(48,14) = "p\\" │ │ │ │ ├── closing_loc: ∅ @@ -331,13 +331,13 @@ │ │ │ ├── opening_loc: (53,5)-(53,6) = "/" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (53,6)-(53,7)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (53,6)-(53,7) = "\\" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "" │ │ │ │ └── @ StringNode (location: (55,0)-(55,6)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (55,0)-(55,6) = "(?)" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/strings.txt b/test/prism/snapshots/strings.txt index 471b371332c957..40a05ce09d5bbf 100644 --- a/test/prism/snapshots/strings.txt +++ b/test/prism/snapshots/strings.txt @@ -337,7 +337,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 3) │ │ │ │ ├── @ StringNode (location: (67,5)-(67,6)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (67,5)-(67,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -359,7 +359,7 @@ │ │ │ │ │ │ └── block: ∅ │ │ │ │ │ └── closing_loc: (67,9)-(67,10) = "}" │ │ │ │ └── @ StringNode (location: (67,10)-(67,11)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (67,10)-(67,11) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -495,13 +495,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (99,0)-(99,2)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (99,0)-(99,1) = "?" │ │ │ ├── content_loc: (99,1)-(99,2) = "a" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "a" │ │ └── @ StringNode (location: (99,3)-(99,6)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (99,3)-(99,4) = "\"" │ │ ├── content_loc: (99,4)-(99,5) = "a" │ │ ├── closing_loc: (99,5)-(99,6) = "\"" diff --git a/test/prism/snapshots/symbols.txt b/test/prism/snapshots/symbols.txt index b6a854db0b2069..c34be74b37d506 100644 --- a/test/prism/snapshots/symbols.txt +++ b/test/prism/snapshots/symbols.txt @@ -234,7 +234,7 @@ │ │ │ ├── opening_loc: ∅ │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (39,5)-(39,6)) - │ │ │ │ │ ├── flags: frozen + │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (39,5)-(39,6) = "b" │ │ │ │ │ ├── closing_loc: ∅ @@ -262,7 +262,7 @@ │ │ │ │ │ │ └── value: 2 │ │ │ │ │ └── closing_loc: (39,14)-(39,15) = "}" │ │ │ │ └── @ StringNode (location: (39,15)-(39,16)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,15)-(39,16) = "c" │ │ │ │ ├── closing_loc: ∅ @@ -272,7 +272,7 @@ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (39,17)-(39,18)) - │ │ │ │ ├── flags: frozen + │ │ │ │ ├── flags: ∅ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (39,17)-(39,18) = "d" │ │ │ │ ├── closing_loc: ∅ @@ -287,7 +287,7 @@ │ │ │ │ │ └── value: 3 │ │ │ │ └── closing_loc: (39,21)-(39,22) = "}" │ │ │ └── @ StringNode (location: (39,22)-(39,23)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,22)-(39,23) = "f" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/literal/literal.txt b/test/prism/snapshots/unparser/corpus/literal/literal.txt index 8ecf613e7ce9ad..ba7dd70b5b717c 100644 --- a/test/prism/snapshots/unparser/corpus/literal/literal.txt +++ b/test/prism/snapshots/unparser/corpus/literal/literal.txt @@ -358,13 +358,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (28,0)-(28,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (28,0)-(28,1) = "\"" │ │ │ ├── content_loc: (28,1)-(28,4) = "foo" │ │ │ ├── closing_loc: (28,4)-(28,5) = "\"" │ │ │ └── unescaped: "foo" │ │ └── @ StringNode (location: (28,6)-(28,11)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (28,6)-(28,7) = "\"" │ │ ├── content_loc: (28,7)-(28,10) = "bar" │ │ ├── closing_loc: (28,10)-(28,11) = "\"" @@ -497,7 +497,7 @@ │ ├── opening_loc: (39,0)-(39,1) = "`" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (39,1)-(39,4)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (39,1)-(39,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -582,7 +582,7 @@ │ ├── opening_loc: (51,0)-(51,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (51,1)-(51,4)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (51,1)-(51,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -601,7 +601,7 @@ │ ├── opening_loc: (52,0)-(52,1) = "/" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (52,1)-(52,4)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (52,1)-(52,4) = "foo" │ │ │ ├── closing_loc: ∅ @@ -1156,7 +1156,7 @@ ├── opening_loc: (89,0)-(89,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (89,1)-(90,0)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (89,1)-(90,0) = " x\n" │ │ ├── closing_loc: ∅ @@ -1178,7 +1178,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (90,5)-(90,6) = "}" │ └── @ StringNode (location: (90,6)-(91,1)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── content_loc: (90,6)-(91,1) = "\n#" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt index c5337729692b09..5ab954b6d4119d 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/dstr.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/dstr.txt @@ -429,7 +429,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (119,0)-(119,3)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (119,0)-(119,1) = "'" │ │ │ ├── content_loc: (119,1)-(119,2) = "a" │ │ │ ├── closing_loc: (119,2)-(119,3) = "'" @@ -447,19 +447,19 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (122,0)-(122,2)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (122,0)-(122,1) = "\"" │ │ │ ├── content_loc: (122,1)-(122,1) = "" │ │ │ ├── closing_loc: (122,1)-(122,2) = "\"" │ │ │ └── unescaped: "" │ │ ├── @ StringNode (location: (122,3)-(122,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (122,3)-(122,4) = "\"" │ │ │ ├── content_loc: (122,4)-(122,4) = "" │ │ │ ├── closing_loc: (122,4)-(122,5) = "\"" │ │ │ └── unescaped: "" │ │ └── @ StringNode (location: (122,6)-(122,8)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (122,6)-(122,7) = "\"" │ │ ├── content_loc: (122,7)-(122,7) = "" │ │ ├── closing_loc: (122,7)-(122,8) = "\"" @@ -487,7 +487,7 @@ │ │ │ │ └── closing_loc: (124,6)-(124,7) = "}" │ │ │ └── closing_loc: (124,7)-(124,8) = "\"" │ │ └── @ StringNode (location: (124,9)-(124,12)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (124,9)-(124,10) = "\"" │ │ ├── content_loc: (124,10)-(124,11) = "b" │ │ ├── closing_loc: (124,11)-(124,12) = "\"" @@ -512,7 +512,7 @@ │ │ │ │ └── name: :@a │ │ │ └── closing_loc: (125,5)-(125,6) = "\"" │ │ └── @ StringNode (location: (125,7)-(125,10)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (125,7)-(125,8) = "\"" │ │ ├── content_loc: (125,8)-(125,9) = "b" │ │ ├── closing_loc: (125,9)-(125,10) = "\"" @@ -537,7 +537,7 @@ │ │ │ │ └── name: :$a │ │ │ └── closing_loc: (126,5)-(126,6) = "\"" │ │ └── @ StringNode (location: (126,7)-(126,10)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (126,7)-(126,8) = "\"" │ │ ├── content_loc: (126,8)-(126,9) = "b" │ │ ├── closing_loc: (126,9)-(126,10) = "\"" @@ -562,7 +562,7 @@ │ │ │ └── name: :@@a │ │ └── closing_loc: (127,6)-(127,7) = "\"" │ └── @ StringNode (location: (127,8)-(127,11)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: (127,8)-(127,9) = "\"" │ ├── content_loc: (127,9)-(127,10) = "b" │ ├── closing_loc: (127,10)-(127,11) = "\"" diff --git a/test/prism/snapshots/unparser/corpus/semantic/literal.txt b/test/prism/snapshots/unparser/corpus/semantic/literal.txt index ef666890be7d34..59e02be64fcfd2 100644 --- a/test/prism/snapshots/unparser/corpus/semantic/literal.txt +++ b/test/prism/snapshots/unparser/corpus/semantic/literal.txt @@ -55,7 +55,7 @@ │ │ │ │ └── name: :@bar │ │ │ └── closing_loc: (11,9)-(11,10) = "}" │ │ └── @ StringNode (location: (11,10)-(11,13)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (11,10)-(11,13) = "baz" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_symbols_interp.txt b/test/prism/snapshots/whitequark/array_symbols_interp.txt index 2437486b43e250..7583ad8833e17d 100644 --- a/test/prism/snapshots/whitequark/array_symbols_interp.txt +++ b/test/prism/snapshots/whitequark/array_symbols_interp.txt @@ -41,7 +41,7 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (3,3)-(3,6)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,3)-(3,6) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/array_words_interp.txt b/test/prism/snapshots/whitequark/array_words_interp.txt index 00fd05e7f7befc..c2f23d2bf19d78 100644 --- a/test/prism/snapshots/whitequark/array_words_interp.txt +++ b/test/prism/snapshots/whitequark/array_words_interp.txt @@ -63,7 +63,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,12)-(3,13) = "}" │ │ ├── @ StringNode (location: (3,13)-(3,16)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,13)-(3,16) = "foo" │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt index 584e997df2c6da..2df571e8dc1890 100644 --- a/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt +++ b/test/prism/snapshots/whitequark/non_lvar_injecting_match.txt @@ -20,7 +20,7 @@ │ │ │ │ └── value: 1 │ │ │ └── closing_loc: (1,4)-(1,5) = "}" │ │ └── @ StringNode (location: (1,5)-(1,18)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,5)-(1,18) = "(?bar)" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/regex_interp.txt b/test/prism/snapshots/whitequark/regex_interp.txt index 0a6db4cfdf4b7b..ee8caf22b9ee55 100644 --- a/test/prism/snapshots/whitequark/regex_interp.txt +++ b/test/prism/snapshots/whitequark/regex_interp.txt @@ -8,7 +8,7 @@ ├── opening_loc: (1,0)-(1,1) = "/" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -30,7 +30,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/whitequark/ruby_bug_11990.txt b/test/prism/snapshots/whitequark/ruby_bug_11990.txt index abdb8d9ddd6ca6..43c04e55a23a15 100644 --- a/test/prism/snapshots/whitequark/ruby_bug_11990.txt +++ b/test/prism/snapshots/whitequark/ruby_bug_11990.txt @@ -18,13 +18,13 @@ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (1,2)-(1,6)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (1,2)-(1,6) = "<<~E" │ │ │ ├── content_loc: (2,0)-(3,0) = " x\n" │ │ │ ├── closing_loc: (3,0)-(4,0) = "E\n" │ │ │ └── unescaped: "x\n" │ │ └── @ StringNode (location: (1,7)-(1,12)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: (1,7)-(1,8) = "\"" │ │ ├── content_loc: (1,8)-(1,11) = " y" │ │ ├── closing_loc: (1,11)-(1,12) = "\"" diff --git a/test/prism/snapshots/whitequark/string_concat.txt b/test/prism/snapshots/whitequark/string_concat.txt index 1739b2f65bfed4..38ea0c745dd664 100644 --- a/test/prism/snapshots/whitequark/string_concat.txt +++ b/test/prism/snapshots/whitequark/string_concat.txt @@ -22,7 +22,7 @@ │ │ │ └── name: :@a │ │ └── closing_loc: (1,7)-(1,8) = "\"" │ └── @ StringNode (location: (1,9)-(1,14)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: (1,9)-(1,10) = "\"" │ ├── content_loc: (1,10)-(1,13) = "bar" │ ├── closing_loc: (1,13)-(1,14) = "\"" diff --git a/test/prism/snapshots/whitequark/xstring_interp.txt b/test/prism/snapshots/whitequark/xstring_interp.txt index 0676ea96837f72..e9543098c2b232 100644 --- a/test/prism/snapshots/whitequark/xstring_interp.txt +++ b/test/prism/snapshots/whitequark/xstring_interp.txt @@ -7,7 +7,7 @@ ├── opening_loc: (1,0)-(1,1) = "`" ├── parts: (length: 3) │ ├── @ StringNode (location: (1,1)-(1,4)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (1,1)-(1,4) = "foo" │ │ ├── closing_loc: ∅ @@ -29,7 +29,7 @@ │ │ │ └── block: ∅ │ │ └── closing_loc: (1,9)-(1,10) = "}" │ └── @ StringNode (location: (1,10)-(1,13)) - │ ├── flags: frozen + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── content_loc: (1,10)-(1,13) = "baz" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/xstring.txt b/test/prism/snapshots/xstring.txt index 1a177026db6bc4..04b4cbf6ea7151 100644 --- a/test/prism/snapshots/xstring.txt +++ b/test/prism/snapshots/xstring.txt @@ -13,7 +13,7 @@ │ ├── opening_loc: (3,0)-(3,1) = "`" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (3,1)-(3,5)) - │ │ │ ├── flags: frozen + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (3,1)-(3,5) = "foo " │ │ │ ├── closing_loc: ∅ @@ -35,7 +35,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (3,10)-(3,11) = "}" │ │ └── @ StringNode (location: (3,11)-(3,15)) - │ │ ├── flags: frozen + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (3,11)-(3,15) = " baz" │ │ ├── closing_loc: ∅ From 06d5d4f1d0f58bdf73a0978221fe13e17f36712e Mon Sep 17 00:00:00 2001 From: David Rodriguez Date: Mon, 25 Mar 2024 18:02:28 +0100 Subject: [PATCH 11/58] [rubygems/rubygems] Fix resolver bug where ActivationRequest objects were not properly compared They were delegating their `#hash` value to a class not overriding that method, and so were returning inconsistent results. https://github.com/rubygems/rubygems/commit/723e4ee0fc --- lib/rubygems/resolver/spec_specification.rb | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/rubygems/resolver/spec_specification.rb b/lib/rubygems/resolver/spec_specification.rb index 79a34d8063f6fa..00ef9fdba05bc0 100644 --- a/lib/rubygems/resolver/spec_specification.rb +++ b/lib/rubygems/resolver/spec_specification.rb @@ -66,4 +66,11 @@ def platform def version spec.version end + + ## + # The hash value for this specification. + + def hash + spec.hash + end end From aa90013829d9394fa92a423a818fb0d6e2ab89cb Mon Sep 17 00:00:00 2001 From: Martin Emde Date: Thu, 29 Feb 2024 17:05:18 -0800 Subject: [PATCH 12/58] [rubygems/rubygems] Fix: vendor_gem takes a block https://github.com/rubygems/rubygems/commit/50cda56fc3 --- test/rubygems/helper.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index ed9e34a906faea..82fb50b70492c7 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -1367,12 +1367,12 @@ def v(string) # # Yields the +specification+ to the block, if given - def vendor_gem(name = "a", version = 1) + def vendor_gem(name = "a", version = 1, &block) directory = File.join "vendor", name FileUtils.mkdir_p directory - save_gemspec name, version, directory + save_gemspec name, version, directory, &block end ## From de742b425fde96283e6a5ac60da8cbdb12291bb9 Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Mon, 25 Mar 2024 15:34:26 -0400 Subject: [PATCH 13/58] YJIT: Inline simple getlocal+leave iseqs This mainly targets things like `T.unsafe()` from Sorbet, which is just an identity function at runtime and only a hint for the static checker. Only deal with simple caller and callees (no keywords and splat etc.). Co-authored-by: Takashi Kokubun (k0kubun) --- bootstraptest/test_yjit.rb | 16 ++++++++++++++++ yjit/src/codegen.rs | 35 ++++++++++++++++++++++++++++++++--- yjit/src/cruby.rs | 1 + 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index 24253a10c9aa2c..e5a160fa9fa342 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4753,3 +4753,19 @@ def entry(define) entry(false) entry(true) } + +assert_equal '[:ok, :ok, :ok]', %q{ + def identity(x) = x + def foo(x, _) = x + def bar(_, _, _, _, x) = x + + def tests + [ + identity(:ok), + foo(:ok, 2), + bar(1, 2, 3, 4, :ok), + ] + end + + tests +} diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 7d9085d4b30d4e..0d536907c1abad 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -6754,11 +6754,16 @@ fn gen_send_bmethod( /// The kind of a value an ISEQ returns enum IseqReturn { Value(VALUE), + LocalVariable(u32), Receiver, } -/// Return the ISEQ's return value if it consists of only putnil/putobject and leave. -fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option) -> Option { +extern { + fn rb_simple_iseq_p(iseq: IseqPtr) -> bool; +} + +/// Return the ISEQ's return value if it consists of one simple instruction and leave. +fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option, ci_flags: u32) -> Option { // Expect only two instructions and one possible operand let iseq_size = unsafe { get_iseq_encoded_size(iseq) }; if !(2..=3).contains(&iseq_size) { @@ -6774,6 +6779,17 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option) -> Option { + // Only accept simple positional only cases for both the caller and the callee. + // Reject block ISEQs to avoid autosplat and other block parameter complications. + if captured_opnd.is_none() && unsafe { rb_simple_iseq_p(iseq) } && ci_flags & VM_CALL_ARGS_SIMPLE != 0 { + let ep_offset = unsafe { *rb_iseq_pc_at_idx(iseq, 1) }.as_u32(); + let local_idx = ep_offset_to_local_idx(iseq, ep_offset); + Some(IseqReturn::LocalVariable(local_idx)) + } else { + None + } + } YARVINSN_putnil => Some(IseqReturn::Value(Qnil)), YARVINSN_putobject => Some(IseqReturn::Value(unsafe { *rb_iseq_pc_at_idx(iseq, 1) })), YARVINSN_putobject_INT2FIX_0_ => Some(IseqReturn::Value(VALUE::fixnum_from_usize(0))), @@ -7055,11 +7071,24 @@ fn gen_send_iseq( } // Inline simple ISEQs whose return value is known at compile time - if let (Some(value), None, false) = (iseq_get_return_value(iseq, captured_opnd), block_arg_type, opt_send_call) { + if let (Some(value), None, false) = (iseq_get_return_value(iseq, captured_opnd, flags), block_arg_type, opt_send_call) { asm_comment!(asm, "inlined simple ISEQ"); gen_counter_incr(asm, Counter::num_send_iseq_inline); match value { + IseqReturn::LocalVariable(local_idx) => { + // Put the local variable at the return slot + let stack_local = asm.stack_opnd(argc - 1 - local_idx as i32); + let stack_return = asm.stack_opnd(argc); + asm.mov(stack_return, stack_local); + + // Update the mapping for the return value + let mapping = asm.ctx.get_opnd_mapping(stack_local.into()); + asm.ctx.set_opnd_mapping(stack_return.into(), mapping); + + // Pop everything but the return value + asm.stack_pop(argc as usize); + } IseqReturn::Value(value) => { // Pop receiver and arguments asm.stack_pop(argc as usize + if captured_opnd.is_some() { 0 } else { 1 }); diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index 31d842531f5515..9547e3fa2cc67b 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -711,6 +711,7 @@ mod manual_defs { pub const RUBY_FIXNUM_MAX: isize = RUBY_LONG_MAX / 2; // From vm_callinfo.h - uses calculation that seems to confuse bindgen + pub const VM_CALL_ARGS_SIMPLE: u32 = 1 << VM_CALL_ARGS_SIMPLE_bit; pub const VM_CALL_ARGS_SPLAT: u32 = 1 << VM_CALL_ARGS_SPLAT_bit; pub const VM_CALL_ARGS_BLOCKARG: u32 = 1 << VM_CALL_ARGS_BLOCKARG_bit; pub const VM_CALL_FCALL: u32 = 1 << VM_CALL_FCALL_bit; From b39057f32cdcf15d8586bd1629d5fc78038320b7 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 Mar 2024 11:10:20 +0900 Subject: [PATCH 14/58] Fix extension installer for out-of-place build https://github.com/ruby/ruby/pull/9673#issuecomment-2019028293 --- tool/rbinstall.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tool/rbinstall.rb b/tool/rbinstall.rb index cdaeff666832b4..63f4beb9434861 100755 --- a/tool/rbinstall.rb +++ b/tool/rbinstall.rb @@ -588,7 +588,12 @@ def ext_features end def makefile_path - "#{makefile_dir}/Makefile" + if File.exist?("#{makefile_dir}/Makefile") + "#{makefile_dir}/Makefile" + else + # for out-of-place build + "#{$ext_build_dir}/#{relative_base}/Makefile" + end end def makefile_dir From 3680981c7b71df8c3a426164787ccefe5296bb25 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Tue, 26 Mar 2024 16:43:14 +0900 Subject: [PATCH 15/58] skip `test_gc_stress_at_startup` (maybe) from 9cf754b the test fails on some environments: https://rubyci.s3.amazonaws.com/icc-x64/ruby-master/log/20240325T200004Z.fail.html.gz ``` 1) Failure: TestGc#test_gc_stress_at_startup [/home/chkbuild/chkbuild/tmp/build/20240325T200004Z/ruby/test/ruby/test_gc.rb:771]: [Bug #15784] pid 1087168 killed by SIGSEGV (signal 11) (core dumped). 1. [3/3] Assertion for "success?" | Expected # to be success?. ``` https://rubyci.s3.amazonaws.com/freebsd14/ruby-master/log/20240325T203002Z.fail.html.gz https://rubyci.s3.amazonaws.com/osx1200arm-no-yjit/ruby-master/log/20240325T195003Z.fail.html.gz https://rubyci.s3.amazonaws.com/s390x/ruby-master/log/20240325T210003Z.fail.html.gz ... so just skipt it until it works. --- test/ruby/test_gc.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 39b001c3d0fee6..470e53fc1fb3f0 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -768,6 +768,7 @@ def initialize end def test_gc_stress_at_startup + omit 'TODO: fixme later' assert_in_out_err([{"RUBY_DEBUG"=>"gc_stress"}], '', [], [], '[Bug #15784]', success: true, timeout: 60) end From a850cd1a87bef738c40d9c550fb8823699083f2e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 26 Mar 2024 10:04:54 +0900 Subject: [PATCH 16/58] [Bug #20392] Block arguments duplication check at `super` --- parse.y | 3 ++- test/ruby/test_syntax.rb | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/parse.y b/parse.y index 28b03f8cd149b8..585130c3465ed6 100644 --- a/parse.y +++ b/parse.y @@ -2063,8 +2063,9 @@ get_nd_args(struct parser_params *p, NODE *node) return RNODE_FCALL(node)->nd_args; case NODE_QCALL: return RNODE_QCALL(node)->nd_args; - case NODE_VCALL: case NODE_SUPER: + return RNODE_SUPER(node)->nd_args; + case NODE_VCALL: case NODE_ZSUPER: case NODE_YIELD: case NODE_RETURN: diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 42108f955f0241..371c41fe37beda 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -2154,6 +2154,13 @@ def obj.bar(*args, **kws, &block) assert_equal 0...1, exp.call(a: 0) end + def test_argument_forwarding_with_super + assert_valid_syntax('def foo(...) super {}; end') + assert_valid_syntax('def foo(...) super() {}; end') + assert_syntax_error('def foo(...) super(...) {}; end', /both block arg and actual block/) + assert_syntax_error('def foo(...) super(1, ...) {}; end', /both block arg and actual block/) + end + def test_class_module_Object_ancestors assert_separately([], <<-RUBY) m = Module.new From 52cf6ec46b744a2ac34d9f6531520bf2aee017f5 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Sun, 24 Mar 2024 20:06:35 +0900 Subject: [PATCH 17/58] [ruby/prism] Fix typos After finding the "if if" typo, some additional typos identified by running `codespell` are also being corrected: https://github.com/codespell-project/codespell https://github.com/ruby/prism/commit/e6a34cfeeb --- prism/defines.h | 8 ++++---- prism/extension.c | 2 +- prism/options.h | 4 ++-- prism/prism.c | 14 +++++++------- test/prism/parse_test.rb | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/prism/defines.h b/prism/defines.h index 2fe73fe3d8cbd9..849ca6d05160ff 100644 --- a/prism/defines.h +++ b/prism/defines.h @@ -151,7 +151,7 @@ #else #ifndef xmalloc /** - * The malloc function that should be used. This can be overriden with + * The malloc function that should be used. This can be overridden with * the PRISM_XALLOCATOR define. */ #define xmalloc malloc @@ -159,7 +159,7 @@ #ifndef xrealloc /** - * The realloc function that should be used. This can be overriden with + * The realloc function that should be used. This can be overridden with * the PRISM_XALLOCATOR define. */ #define xrealloc realloc @@ -167,7 +167,7 @@ #ifndef xcalloc /** - * The calloc function that should be used. This can be overriden with + * The calloc function that should be used. This can be overridden with * the PRISM_XALLOCATOR define. */ #define xcalloc calloc @@ -175,7 +175,7 @@ #ifndef xfree /** - * The free function that should be used. This can be overriden with the + * The free function that should be used. This can be overridden with the * PRISM_XALLOCATOR define. */ #define xfree free diff --git a/prism/extension.c b/prism/extension.c index 88ba006ac2ec30..5b5c0593c91719 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -730,7 +730,7 @@ parse_input(pm_string_t *input, const pm_options_t *options) { * parsed. This should be an array of arrays of symbols or nil. Scopes are * ordered from the outermost scope to the innermost one. * * `version` - the version of Ruby syntax that prism should used to parse Ruby - * code. By default prism assumes you want to parse with the latest vesion + * code. By default prism assumes you want to parse with the latest version * of Ruby syntax (which you can trigger with `nil` or `"latest"`). You * may also restrict the syntax to a specific version of Ruby. The * supported values are `"3.3.0"` and `"3.4.0"`. diff --git a/prism/options.h b/prism/options.h index d07f5aa4fa7e72..9a4d6969c3c67b 100644 --- a/prism/options.h +++ b/prism/options.h @@ -286,14 +286,14 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); * | `0` | use the latest version of prism | * | `1` | use the version of prism that is vendored in CRuby 3.3.0 | * - * Each scope is layed out as follows: + * Each scope is laid out as follows: * * | # bytes | field | * | ------- | -------------------------- | * | `4` | the number of locals | * | ... | the locals | * - * Each local is layed out as follows: + * Each local is laid out as follows: * * | # bytes | field | * | ------- | -------------------------- | diff --git a/prism/prism.c b/prism/prism.c index 6f2ab81e9ae3f3..169f9c26d64028 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -5602,7 +5602,7 @@ pm_rescue_modifier_node_create(pm_parser_t *parser, pm_node_t *expression, const } /** - * Allocate and initiliaze a new RescueNode node. + * Allocate and initialize a new RescueNode node. */ static pm_rescue_node_t * pm_rescue_node_create(pm_parser_t *parser, const pm_token_t *keyword) { @@ -10416,7 +10416,7 @@ parser_lex(pm_parser_t *parser) { } default: // If we get to this point, then we have a % that is completely - // unparseable. In this case we'll just drop it from the parser + // unparsable. In this case we'll just drop it from the parser // and skip past it and hope that the next token is something // that we can parse. pm_parser_err_current(parser, PM_ERR_INVALID_PERCENT); @@ -14426,7 +14426,7 @@ parse_heredoc_dedent_string(pm_string_t *string, size_t common_whitespace) { static void parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_whitespace) { // The next node should be dedented if it's the first node in the list or if - // if follows a string node. + // it follows a string node. bool dedent_next = true; // Iterate over all nodes, and trim whitespace accordingly. We're going to @@ -17949,7 +17949,7 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * if (pm_regexp_named_capture_group_names(pm_string_source(content), pm_string_length(content), &named_captures, parser->encoding_changed, parser->encoding) && (named_captures.length > 0)) { // Since we should not create a MatchWriteNode when all capture names - // are invalid, creating a MatchWriteNode is delayed here. + // are invalid, creating a MatchWriteNode is delaid here. pm_match_write_node_t *match = NULL; pm_constant_id_list_t names = { 0 }; @@ -19616,7 +19616,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // Now we're going to determine how we're going to format line numbers and // blank lines based on the maximum number of digits in the line numbers - // that are going to be displayed. + // that are going to be displaid. pm_error_format_t error_format; int32_t max_line_number = errors[error_list->size - 1].line - start_line; @@ -19707,7 +19707,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col pm_error_t *error = &errors[index]; // Here we determine how many lines of padding of the source to display, - // based on the difference from the last line that was displayed. + // based on the difference from the last line that was displaid. if (error->line - last_line > 1) { if (error->line - last_line > 2) { if ((index != 0) && (error->line - last_line > 3)) { @@ -19739,7 +19739,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // the width of the error, then the error message itself. // // Note that this doesn't take into account the width of the actual - // character when displayed in the terminal. For some east-asian + // character when displaid in the terminal. For some east-asian // languages or emoji, this means it can be thrown off pretty badly. We // will need to solve this eventually. pm_buffer_append_string(buffer, " ", 2); diff --git a/test/prism/parse_test.rb b/test/prism/parse_test.rb index db66b431ba7ad0..4255553f516afd 100644 --- a/test/prism/parse_test.rb +++ b/test/prism/parse_test.rb @@ -204,7 +204,7 @@ def test_parse_file_comments # Additionally, Ripper cannot parse the %w[] fixture in this file, so set ripper_should_parse to false. ripper_should_parse = false if relative == "spanning_heredoc.txt" - # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace charactes in the heredoc start. + # Ruby < 3.3.0 cannot parse heredocs where there are leading whitespace characters in the heredoc start. # Example: <<~' EOF' or <<-' EOF' # https://bugs.ruby-lang.org/issues/19539 ripper_should_parse = false if relative == "heredocs_leading_whitespace.txt" && RUBY_VERSION < "3.3.0" From 8cfa8e87b2705fb356bbbb9ef719b5c5a54f9862 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 26 Mar 2024 20:30:29 +0900 Subject: [PATCH 18/58] [ruby/irb] Fix a typo (https://github.com/ruby/irb/pull/912) https://github.com/ruby/irb/commit/2057248e40 --- lib/irb/cmd/nop.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/irb/cmd/nop.rb b/lib/irb/cmd/nop.rb index 49f89bac95eed8..9d2e3c4d47cd6e 100644 --- a/lib/irb/cmd/nop.rb +++ b/lib/irb/cmd/nop.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true # This file is just a placeholder for backward-compatibility. -# Please require 'irb' and inheirt your command from `IRB::Command::Base` instead. +# Please require 'irb' and inherit your command from `IRB::Command::Base` instead. From 2b08406cd0db0042520fb0346544660e10a4d93c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89tienne=20Barri=C3=A9?= Date: Mon, 25 Mar 2024 11:18:26 +0100 Subject: [PATCH 19/58] Expose rb_str_chilled_p Some extensions (like stringio) may need to differentiate between chilled strings and frozen strings. They can now use rb_str_chilled_p but must check for its presence since the function will be removed when chilled strings are removed. [Bug #20389] [Feature #20205] Co-authored-by: Jean Boussier --- ext/-test-/string/chilled.c | 13 +++ ext/-test-/string/depend | 159 ++++++++++++++++++++++++++ include/ruby/internal/intern/string.h | 15 +++ string.c | 6 + test/-ext-/string/test_chilled.rb | 19 +++ 5 files changed, 212 insertions(+) create mode 100644 ext/-test-/string/chilled.c create mode 100644 test/-ext-/string/test_chilled.rb diff --git a/ext/-test-/string/chilled.c b/ext/-test-/string/chilled.c new file mode 100644 index 00000000000000..c98fc72c477256 --- /dev/null +++ b/ext/-test-/string/chilled.c @@ -0,0 +1,13 @@ +#include "ruby.h" + +static VALUE +bug_s_rb_str_chilled_p(VALUE self, VALUE str) +{ + return rb_str_chilled_p(str) ? Qtrue : Qfalse; +} + +void +Init_string_chilled(VALUE klass) +{ + rb_define_singleton_method(klass, "rb_str_chilled_p", bug_s_rb_str_chilled_p, 1); +} diff --git a/ext/-test-/string/depend b/ext/-test-/string/depend index eeb4914346bd91..f8f58e7d441ee2 100644 --- a/ext/-test-/string/depend +++ b/ext/-test-/string/depend @@ -173,6 +173,165 @@ capacity.o: $(hdrdir)/ruby/subst.h capacity.o: $(top_srcdir)/internal/compilers.h capacity.o: $(top_srcdir)/internal/string.h capacity.o: capacity.c +chilled.o: $(RUBY_EXTCONF_H) +chilled.o: $(arch_hdrdir)/ruby/config.h +chilled.o: $(hdrdir)/ruby.h +chilled.o: $(hdrdir)/ruby/assert.h +chilled.o: $(hdrdir)/ruby/backward.h +chilled.o: $(hdrdir)/ruby/backward/2/assume.h +chilled.o: $(hdrdir)/ruby/backward/2/attributes.h +chilled.o: $(hdrdir)/ruby/backward/2/bool.h +chilled.o: $(hdrdir)/ruby/backward/2/inttypes.h +chilled.o: $(hdrdir)/ruby/backward/2/limits.h +chilled.o: $(hdrdir)/ruby/backward/2/long_long.h +chilled.o: $(hdrdir)/ruby/backward/2/stdalign.h +chilled.o: $(hdrdir)/ruby/backward/2/stdarg.h +chilled.o: $(hdrdir)/ruby/defines.h +chilled.o: $(hdrdir)/ruby/intern.h +chilled.o: $(hdrdir)/ruby/internal/abi.h +chilled.o: $(hdrdir)/ruby/internal/anyargs.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/char.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/double.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/fixnum.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/gid_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/int.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/intptr_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/long.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/long_long.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/mode_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/off_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/pid_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/short.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/size_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/st_data_t.h +chilled.o: $(hdrdir)/ruby/internal/arithmetic/uid_t.h +chilled.o: $(hdrdir)/ruby/internal/assume.h +chilled.o: $(hdrdir)/ruby/internal/attr/alloc_size.h +chilled.o: $(hdrdir)/ruby/internal/attr/artificial.h +chilled.o: $(hdrdir)/ruby/internal/attr/cold.h +chilled.o: $(hdrdir)/ruby/internal/attr/const.h +chilled.o: $(hdrdir)/ruby/internal/attr/constexpr.h +chilled.o: $(hdrdir)/ruby/internal/attr/deprecated.h +chilled.o: $(hdrdir)/ruby/internal/attr/diagnose_if.h +chilled.o: $(hdrdir)/ruby/internal/attr/enum_extensibility.h +chilled.o: $(hdrdir)/ruby/internal/attr/error.h +chilled.o: $(hdrdir)/ruby/internal/attr/flag_enum.h +chilled.o: $(hdrdir)/ruby/internal/attr/forceinline.h +chilled.o: $(hdrdir)/ruby/internal/attr/format.h +chilled.o: $(hdrdir)/ruby/internal/attr/maybe_unused.h +chilled.o: $(hdrdir)/ruby/internal/attr/noalias.h +chilled.o: $(hdrdir)/ruby/internal/attr/nodiscard.h +chilled.o: $(hdrdir)/ruby/internal/attr/noexcept.h +chilled.o: $(hdrdir)/ruby/internal/attr/noinline.h +chilled.o: $(hdrdir)/ruby/internal/attr/nonnull.h +chilled.o: $(hdrdir)/ruby/internal/attr/noreturn.h +chilled.o: $(hdrdir)/ruby/internal/attr/packed_struct.h +chilled.o: $(hdrdir)/ruby/internal/attr/pure.h +chilled.o: $(hdrdir)/ruby/internal/attr/restrict.h +chilled.o: $(hdrdir)/ruby/internal/attr/returns_nonnull.h +chilled.o: $(hdrdir)/ruby/internal/attr/warning.h +chilled.o: $(hdrdir)/ruby/internal/attr/weakref.h +chilled.o: $(hdrdir)/ruby/internal/cast.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/apple.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/clang.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/gcc.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/intel.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/msvc.h +chilled.o: $(hdrdir)/ruby/internal/compiler_is/sunpro.h +chilled.o: $(hdrdir)/ruby/internal/compiler_since.h +chilled.o: $(hdrdir)/ruby/internal/config.h +chilled.o: $(hdrdir)/ruby/internal/constant_p.h +chilled.o: $(hdrdir)/ruby/internal/core.h +chilled.o: $(hdrdir)/ruby/internal/core/rarray.h +chilled.o: $(hdrdir)/ruby/internal/core/rbasic.h +chilled.o: $(hdrdir)/ruby/internal/core/rbignum.h +chilled.o: $(hdrdir)/ruby/internal/core/rclass.h +chilled.o: $(hdrdir)/ruby/internal/core/rdata.h +chilled.o: $(hdrdir)/ruby/internal/core/rfile.h +chilled.o: $(hdrdir)/ruby/internal/core/rhash.h +chilled.o: $(hdrdir)/ruby/internal/core/robject.h +chilled.o: $(hdrdir)/ruby/internal/core/rregexp.h +chilled.o: $(hdrdir)/ruby/internal/core/rstring.h +chilled.o: $(hdrdir)/ruby/internal/core/rstruct.h +chilled.o: $(hdrdir)/ruby/internal/core/rtypeddata.h +chilled.o: $(hdrdir)/ruby/internal/ctype.h +chilled.o: $(hdrdir)/ruby/internal/dllexport.h +chilled.o: $(hdrdir)/ruby/internal/dosish.h +chilled.o: $(hdrdir)/ruby/internal/error.h +chilled.o: $(hdrdir)/ruby/internal/eval.h +chilled.o: $(hdrdir)/ruby/internal/event.h +chilled.o: $(hdrdir)/ruby/internal/fl_type.h +chilled.o: $(hdrdir)/ruby/internal/gc.h +chilled.o: $(hdrdir)/ruby/internal/glob.h +chilled.o: $(hdrdir)/ruby/internal/globals.h +chilled.o: $(hdrdir)/ruby/internal/has/attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/builtin.h +chilled.o: $(hdrdir)/ruby/internal/has/c_attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/cpp_attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/declspec_attribute.h +chilled.o: $(hdrdir)/ruby/internal/has/extension.h +chilled.o: $(hdrdir)/ruby/internal/has/feature.h +chilled.o: $(hdrdir)/ruby/internal/has/warning.h +chilled.o: $(hdrdir)/ruby/internal/intern/array.h +chilled.o: $(hdrdir)/ruby/internal/intern/bignum.h +chilled.o: $(hdrdir)/ruby/internal/intern/class.h +chilled.o: $(hdrdir)/ruby/internal/intern/compar.h +chilled.o: $(hdrdir)/ruby/internal/intern/complex.h +chilled.o: $(hdrdir)/ruby/internal/intern/cont.h +chilled.o: $(hdrdir)/ruby/internal/intern/dir.h +chilled.o: $(hdrdir)/ruby/internal/intern/enum.h +chilled.o: $(hdrdir)/ruby/internal/intern/enumerator.h +chilled.o: $(hdrdir)/ruby/internal/intern/error.h +chilled.o: $(hdrdir)/ruby/internal/intern/eval.h +chilled.o: $(hdrdir)/ruby/internal/intern/file.h +chilled.o: $(hdrdir)/ruby/internal/intern/hash.h +chilled.o: $(hdrdir)/ruby/internal/intern/io.h +chilled.o: $(hdrdir)/ruby/internal/intern/load.h +chilled.o: $(hdrdir)/ruby/internal/intern/marshal.h +chilled.o: $(hdrdir)/ruby/internal/intern/numeric.h +chilled.o: $(hdrdir)/ruby/internal/intern/object.h +chilled.o: $(hdrdir)/ruby/internal/intern/parse.h +chilled.o: $(hdrdir)/ruby/internal/intern/proc.h +chilled.o: $(hdrdir)/ruby/internal/intern/process.h +chilled.o: $(hdrdir)/ruby/internal/intern/random.h +chilled.o: $(hdrdir)/ruby/internal/intern/range.h +chilled.o: $(hdrdir)/ruby/internal/intern/rational.h +chilled.o: $(hdrdir)/ruby/internal/intern/re.h +chilled.o: $(hdrdir)/ruby/internal/intern/ruby.h +chilled.o: $(hdrdir)/ruby/internal/intern/select.h +chilled.o: $(hdrdir)/ruby/internal/intern/select/largesize.h +chilled.o: $(hdrdir)/ruby/internal/intern/signal.h +chilled.o: $(hdrdir)/ruby/internal/intern/sprintf.h +chilled.o: $(hdrdir)/ruby/internal/intern/string.h +chilled.o: $(hdrdir)/ruby/internal/intern/struct.h +chilled.o: $(hdrdir)/ruby/internal/intern/thread.h +chilled.o: $(hdrdir)/ruby/internal/intern/time.h +chilled.o: $(hdrdir)/ruby/internal/intern/variable.h +chilled.o: $(hdrdir)/ruby/internal/intern/vm.h +chilled.o: $(hdrdir)/ruby/internal/interpreter.h +chilled.o: $(hdrdir)/ruby/internal/iterator.h +chilled.o: $(hdrdir)/ruby/internal/memory.h +chilled.o: $(hdrdir)/ruby/internal/method.h +chilled.o: $(hdrdir)/ruby/internal/module.h +chilled.o: $(hdrdir)/ruby/internal/newobj.h +chilled.o: $(hdrdir)/ruby/internal/scan_args.h +chilled.o: $(hdrdir)/ruby/internal/special_consts.h +chilled.o: $(hdrdir)/ruby/internal/static_assert.h +chilled.o: $(hdrdir)/ruby/internal/stdalign.h +chilled.o: $(hdrdir)/ruby/internal/stdbool.h +chilled.o: $(hdrdir)/ruby/internal/symbol.h +chilled.o: $(hdrdir)/ruby/internal/value.h +chilled.o: $(hdrdir)/ruby/internal/value_type.h +chilled.o: $(hdrdir)/ruby/internal/variable.h +chilled.o: $(hdrdir)/ruby/internal/warning_push.h +chilled.o: $(hdrdir)/ruby/internal/xmalloc.h +chilled.o: $(hdrdir)/ruby/missing.h +chilled.o: $(hdrdir)/ruby/ruby.h +chilled.o: $(hdrdir)/ruby/st.h +chilled.o: $(hdrdir)/ruby/subst.h +chilled.o: chilled.c coderange.o: $(RUBY_EXTCONF_H) coderange.o: $(arch_hdrdir)/ruby/config.h coderange.o: $(hdrdir)/ruby/assert.h diff --git a/include/ruby/internal/intern/string.h b/include/ruby/internal/intern/string.h index 952dc508c24a67..cfe0454ee88562 100644 --- a/include/ruby/internal/intern/string.h +++ b/include/ruby/internal/intern/string.h @@ -601,6 +601,21 @@ VALUE rb_str_dup(VALUE str); */ VALUE rb_str_resurrect(VALUE str); +/** + * Returns whether a string is chilled or not. + * + * This function is temporary and users must check for its presence using + * #ifdef HAVE_RB_STR_CHILLED_P. If HAVE_RB_STR_CHILLED_P is not defined, then + * strings can't be chilled. + * + * @param[in] str A string. + * @retval 1 The string is chilled. + * @retval 0 Otherwise. + */ +bool rb_str_chilled_p(VALUE str); + +#define HAVE_RB_STR_CHILLED_P 1 + /** * Obtains a "temporary lock" of the string. This advisory locking mechanism * prevents other cooperating threads from tampering the receiver. The same diff --git a/string.c b/string.c index 9d84b16a073ec8..5c29718dff8647 100644 --- a/string.c +++ b/string.c @@ -1833,6 +1833,12 @@ rb_ec_str_resurrect(struct rb_execution_context_struct *ec, VALUE str, bool chil return new_str; } +bool +rb_str_chilled_p(VALUE str) +{ + return CHILLED_STRING_P(str); +} + /* * * call-seq: diff --git a/test/-ext-/string/test_chilled.rb b/test/-ext-/string/test_chilled.rb new file mode 100644 index 00000000000000..dccab61cede766 --- /dev/null +++ b/test/-ext-/string/test_chilled.rb @@ -0,0 +1,19 @@ +require 'test/unit' +require '-test-/string' + +class Test_String_ChilledString < Test::Unit::TestCase + def test_rb_str_chilled_p + str = "" + assert_equal true, Bug::String.rb_str_chilled_p(str) + end + + def test_rb_str_chilled_p_frozen + str = "".freeze + assert_equal false, Bug::String.rb_str_chilled_p(str) + end + + def test_rb_str_chilled_p_mutable + str = "".dup + assert_equal false, Bug::String.rb_str_chilled_p(str) + end +end From e9152bc9da39ad864b755e7bcc682610f8035a98 Mon Sep 17 00:00:00 2001 From: Andrew Konchin Date: Mon, 19 Feb 2024 12:58:20 +0200 Subject: [PATCH 20/58] [ruby/prism] Enable ParametersSignatureTest on TruffleRuby https://github.com/ruby/prism/commit/c7a7af1eac --- test/prism/parameters_signature_test.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/prism/parameters_signature_test.rb b/test/prism/parameters_signature_test.rb index 788ce7b90712e4..0eed8d993d52c9 100644 --- a/test/prism/parameters_signature_test.rb +++ b/test/prism/parameters_signature_test.rb @@ -2,7 +2,7 @@ require_relative "test_helper" -return if RUBY_VERSION < "3.2" || RUBY_ENGINE == "truffleruby" +return if RUBY_VERSION < "3.2" module Prism class ParametersSignatureTest < TestCase @@ -55,6 +55,8 @@ def test_keyrest_anonymous end def test_key_ordering + omit("TruffleRuby returns keys in order they were declared") if RUBY_ENGINE == "truffleruby" + assert_parameters([[:keyreq, :a], [:keyreq, :b], [:key, :c], [:key, :d]], "a:, c: 1, b:, d: 2") end From 19752cf4aa227106212be129507ac1bf339b26c0 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 21 Mar 2024 16:22:46 -0400 Subject: [PATCH 21/58] Use macro SET_WHEN_UINT --- debug.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/debug.c b/debug.c index e194472c1aaf9d..546ffc440f155b 100644 --- a/debug.c +++ b/debug.c @@ -219,7 +219,11 @@ ruby_env_debug_option(const char *str, int len, void *arg) } \ } while (0) #define SET_WHEN_UINT(name, vals, num, req) \ - if (NAME_MATCH_VALUE(name)) SET_UINT_LIST(name, vals, num); + if (NAME_MATCH_VALUE(name)) { \ + if (!len) req; \ + else SET_UINT_LIST(name, vals, num); \ + return 1; \ + } if (NAME_MATCH("gc_stress")) { rb_gc_stress_set(Qtrue); @@ -227,22 +231,15 @@ ruby_env_debug_option(const char *str, int len, void *arg) } SET_WHEN("core", ruby_enable_coredump, 1); SET_WHEN("ci", ruby_on_ci, 1); - if (NAME_MATCH_VALUE("rgengc")) { - if (!len) ruby_rgengc_debug = 1; - else SET_UINT_LIST("rgengc", &ruby_rgengc_debug, 1); - return 1; - } + SET_WHEN_UINT("rgengc", &ruby_rgengc_debug, 1, ruby_rgengc_debug = 1); #if defined _WIN32 # if RUBY_MSVCRT_VERSION >= 80 SET_WHEN("rtc_error", ruby_w32_rtc_error, 1); # endif #endif #if defined _WIN32 || defined __CYGWIN__ - if (NAME_MATCH_VALUE("codepage")) { - if (!len) fprintf(stderr, "missing codepage argument"); - else SET_UINT_LIST("codepage", ruby_w32_codepage, numberof(ruby_w32_codepage)); - return 1; - } + SET_WHEN_UINT("codepage", ruby_w32_codepage, numberof(ruby_w32_codepage), + fprintf(stderr, "missing codepage argument")); #endif return 0; } From 240fb3957b02cb9a1cc0d1e01ae40db803390bed Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 25 Mar 2024 15:32:54 -0400 Subject: [PATCH 22/58] [ruby/prism] Freeze internal parts, again https://github.com/ruby/prism/commit/50372fee5c --- lib/prism/node_ext.rb | 1 + prism/config.yml | 10 ++ prism/prism.c | 115 ++++++++++++------ test/prism/snapshots/alias.txt | 2 +- test/prism/snapshots/dash_heredocs.txt | 20 +-- test/prism/snapshots/dos_endings.txt | 10 +- test/prism/snapshots/dstring.txt | 8 +- .../snapshots/heredocs_leading_whitespace.txt | 10 +- test/prism/snapshots/heredocs_nested.txt | 16 ++- .../heredocs_with_ignored_newlines.txt | 19 +-- test/prism/snapshots/lambda.txt | 3 +- test/prism/snapshots/method_calls.txt | 5 +- test/prism/snapshots/methods.txt | 3 +- test/prism/snapshots/modules.txt | 5 +- test/prism/snapshots/regex.txt | 8 +- .../prism/snapshots/seattlerb/difficult0_.txt | 5 +- test/prism/snapshots/seattlerb/dstr_evstr.txt | 1 + .../snapshots/seattlerb/dstr_lex_state.txt | 1 + test/prism/snapshots/seattlerb/dstr_str.txt | 3 +- .../prism/snapshots/seattlerb/evstr_evstr.txt | 1 + test/prism/snapshots/seattlerb/evstr_str.txt | 3 +- .../snapshots/seattlerb/heredoc_nested.txt | 5 +- .../snapshots/seattlerb/heredoc_squiggly.txt | 7 +- ...squiggly_blank_line_plus_interpolation.txt | 5 +- .../heredoc_squiggly_blank_lines.txt | 7 +- .../seattlerb/heredoc_squiggly_interp.txt | 9 +- .../seattlerb/heredoc_squiggly_tabs.txt | 5 +- .../seattlerb/heredoc_squiggly_tabs_extra.txt | 5 +- .../heredoc_squiggly_visually_blank_lines.txt | 7 +- ...erpolation_and_carriage_return_escapes.txt | 5 +- ...on_and_carriage_return_escapes_windows.txt | 5 +- .../parse_line_dstr_escaped_newline.txt | 3 +- .../parse_line_dstr_soft_newline.txt | 3 +- .../parse_line_evstr_after_break.txt | 4 +- .../seattlerb/parse_line_heredoc_evstr.txt | 5 +- .../seattlerb/pct_w_heredoc_interp_nested.txt | 1 + .../snapshots/seattlerb/qsymbols_interp.txt | 2 +- test/prism/snapshots/seattlerb/str_evstr.txt | 3 +- .../snapshots/seattlerb/str_evstr_escape.txt | 5 +- .../seattlerb/str_heredoc_interp.txt | 3 +- .../seattlerb/str_interp_ternary_or_label.txt | 1 + .../str_lit_concat_bad_encodings.txt | 5 +- .../snapshots/seattlerb/str_pct_Q_nested.txt | 5 +- .../seattlerb/str_pct_nested_nested.txt | 6 +- test/prism/snapshots/seattlerb/str_str.txt | 3 +- .../prism/snapshots/seattlerb/str_str_str.txt | 5 +- .../snapshots/seattlerb/words_interp.txt | 3 +- test/prism/snapshots/spanning_heredoc.txt | 40 +++--- test/prism/snapshots/strings.txt | 23 ++-- test/prism/snapshots/symbols.txt | 10 +- test/prism/snapshots/tilde_heredocs.txt | 97 +++++++++------ test/prism/snapshots/undef.txt | 2 +- .../unparser/corpus/literal/assignment.txt | 25 ++-- .../snapshots/unparser/corpus/literal/def.txt | 5 +- .../unparser/corpus/literal/dstr.txt | 47 ++++--- .../unparser/corpus/literal/literal.txt | 66 ++++++---- .../unparser/corpus/semantic/dstr.txt | 114 ++++++++++------- .../unparser/corpus/semantic/literal.txt | 2 +- .../whitequark/array_symbols_interp.txt | 2 +- .../whitequark/array_words_interp.txt | 4 +- test/prism/snapshots/whitequark/bug_435.txt | 1 + test/prism/snapshots/whitequark/bug_466.txt | 1 + test/prism/snapshots/whitequark/bug_473.txt | 1 + test/prism/snapshots/whitequark/bug_480.txt | 1 + .../whitequark/bug_interp_single.txt | 2 + .../whitequark/dedenting_heredoc.txt | 59 +++++---- ...olating_heredoc_fake_line_continuation.txt | 5 +- ...nterpolating_heredoc_line_continuation.txt | 5 +- .../endless_method_command_syntax.txt | 6 +- .../whitequark/non_lvar_injecting_match.txt | 2 +- .../snapshots/whitequark/parser_bug_640.txt | 5 +- ...ps_truncated_parts_of_squiggly_heredoc.txt | 3 +- .../snapshots/whitequark/regex_interp.txt | 4 +- .../snapshots/whitequark/ruby_bug_11990.txt | 5 +- .../whitequark/slash_newline_in_heredocs.txt | 7 +- .../snapshots/whitequark/string_concat.txt | 6 +- .../snapshots/whitequark/string_dvar.txt | 5 +- .../snapshots/whitequark/string_interp.txt | 5 +- .../snapshots/whitequark/symbol_interp.txt | 4 +- test/prism/snapshots/whitequark/undef.txt | 2 +- .../snapshots/whitequark/xstring_interp.txt | 4 +- test/prism/snapshots/xstring.txt | 4 +- 82 files changed, 584 insertions(+), 356 deletions(-) diff --git a/lib/prism/node_ext.rb b/lib/prism/node_ext.rb index 4ec7c3014cb464..86745440659680 100644 --- a/lib/prism/node_ext.rb +++ b/lib/prism/node_ext.rb @@ -55,6 +55,7 @@ class StringNode < Node def to_interpolated InterpolatedStringNode.new( source, + frozen? ? InterpolatedStringNodeFlags::FROZEN : 0, opening_loc, [copy(opening_loc: nil, closing_loc: nil, location: content_loc)], closing_loc, diff --git a/prism/config.yml b/prism/config.yml index d9e39460d12f2c..bb1dd6e6b02ec0 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -626,6 +626,13 @@ flags: - name: HEXADECIMAL comment: "0x prefix" comment: Flags for integer nodes that correspond to the base of the integer. + - name: InterpolatedStringNodeFlags + values: + - name: FROZEN + comment: "frozen by virtue of a `frozen_string_literal: true` comment or `--enable-frozen-string-literal`" + - name: MUTABLE + comment: "mutable by virtue of a `frozen_string_literal: false` comment or `--disable-frozen-string-literal`" + comment: Flags for interpolated string nodes that indicated mutability if they are also marked as literals. - name: KeywordHashNodeFlags values: - name: SYMBOL_KEYS @@ -2260,6 +2267,9 @@ nodes: ^^^^^^^^^^^^^^^^ - name: InterpolatedStringNode fields: + - name: flags + type: flags + kind: InterpolatedStringNodeFlags - name: opening_loc type: location? - name: parts diff --git a/prism/prism.c b/prism/prism.c index 169f9c26d64028..e97af236769156 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4281,6 +4281,7 @@ pm_interpolated_regular_expression_node_create(pm_parser_t *parser, const pm_tok *node = (pm_interpolated_regular_expression_node_t) { { .type = PM_INTERPOLATED_REGULAR_EXPRESSION_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = NULL, @@ -4302,6 +4303,15 @@ pm_interpolated_regular_expression_node_append(pm_interpolated_regular_expressio if (node->base.location.end < part->location.end) { node->base.location.end = part->location.end; } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + pm_node_list_append(&node->parts, part); } @@ -4312,6 +4322,38 @@ pm_interpolated_regular_expression_node_closing_set(pm_parser_t *parser, pm_inte pm_node_flag_set((pm_node_t *)node, pm_regular_expression_flags_create(parser, closing)); } +/** + * Append a part to an InterpolatedStringNode node. + */ +static inline void +pm_interpolated_string_node_append(pm_parser_t *parser, pm_interpolated_string_node_t *node, pm_node_t *part) { + if (node->parts.size == 0 && node->opening_loc.start == NULL) { + node->base.location.start = part->location.start; + } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL | PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN | PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + } + + pm_node_list_append(&node->parts, part); + node->base.location.end = MAX(node->base.location.end, part->location.end); + + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + switch (parser->frozen_string_literal) { + case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED: + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); + break; + case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED: + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + break; + } + } +} + /** * Allocate and initialize a new InterpolatedStringNode node. */ @@ -4322,6 +4364,7 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_string_node_t) { { .type = PM_INTERPOLATED_STRING_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4333,25 +4376,14 @@ pm_interpolated_string_node_create(pm_parser_t *parser, const pm_token_t *openin }; if (parts != NULL) { - node->parts = *parts; + for (size_t index = 0; index < parts->size; index++) { + pm_interpolated_string_node_append(parser, node, parts->nodes[index]); + } } return node; } -/** - * Append a part to an InterpolatedStringNode node. - */ -static inline void -pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_t *part) { - if (node->parts.size == 0 && node->opening_loc.start == NULL) { - node->base.location.start = part->location.start; - } - - pm_node_list_append(&node->parts, part); - node->base.location.end = part->location.end; -} - /** * Set the closing token of the given InterpolatedStringNode node. */ @@ -4361,6 +4393,24 @@ pm_interpolated_string_node_closing_set(pm_interpolated_string_node_t *node, con node->base.location.end = closing->end; } +static void +pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { + if (node->parts.size == 0 && node->opening_loc.start == NULL) { + node->base.location.start = part->location.start; + } + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + + if (!PM_NODE_FLAG_P(part, PM_NODE_FLAG_STATIC_LITERAL)) { + pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL); + } + + pm_node_list_append(&node->parts, part); + node->base.location.end = MAX(node->base.location.end, part->location.end); +} + /** * Allocate and initialize a new InterpolatedSymbolNode node. */ @@ -4371,6 +4421,7 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin *node = (pm_interpolated_symbol_node_t) { { .type = PM_INTERPOLATED_SYMBOL_NODE, + .flags = PM_NODE_FLAG_STATIC_LITERAL, .location = { .start = opening->start, .end = closing->end, @@ -4382,22 +4433,14 @@ pm_interpolated_symbol_node_create(pm_parser_t *parser, const pm_token_t *openin }; if (parts != NULL) { - node->parts = *parts; + for (size_t index = 0; index < parts->size; index++) { + pm_interpolated_symbol_node_append(node, parts->nodes[index]); + } } return node; } -static inline void -pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_t *part) { - if (node->parts.size == 0 && node->opening_loc.start == NULL) { - node->base.location.start = part->location.start; - } - - pm_node_list_append(&node->parts, part); - node->base.location.end = part->location.end; -} - /** * Allocate a new InterpolatedXStringNode node. */ @@ -4423,6 +4466,10 @@ pm_interpolated_xstring_node_create(pm_parser_t *parser, const pm_token_t *openi static inline void pm_interpolated_xstring_node_append(pm_interpolated_x_string_node_t *node, pm_node_t *part) { + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + pm_node_flag_set(part, PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN); + } + pm_node_list_append(&node->parts, part); node->base.location.end = part->location.end; } @@ -15427,11 +15474,11 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { pm_token_t bounds = not_provided(parser); pm_interpolated_string_node_t *container = pm_interpolated_string_node_create(parser, &bounds, NULL, &bounds); - pm_interpolated_string_node_append(container, current); + pm_interpolated_string_node_append(parser, container, current); current = (pm_node_t *) container; } - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, node); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, node); } } @@ -17365,15 +17412,15 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b // If we hit string content and the current node is // an interpolated string, then we need to append // the string content to the list of child nodes. - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, string); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, string); } else if (PM_NODE_TYPE_P(current, PM_STRING_NODE)) { // If we hit string content and the current node is // a string node, then we need to convert the // current node into an interpolated string and add // the string content to the list of child nodes. pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(interpolated, current); - pm_interpolated_string_node_append(interpolated, string); + pm_interpolated_string_node_append(parser, interpolated, current); + pm_interpolated_string_node_append(parser, interpolated, string); current = (pm_node_t *) interpolated; } else { assert(false && "unreachable"); @@ -17398,7 +17445,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(interpolated, current); + pm_interpolated_string_node_append(parser, interpolated, current); current = (pm_node_t *) interpolated; } else { // If we hit an embedded variable and the current @@ -17407,7 +17454,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *part = parse_string_part(parser); - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, part); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, part); break; } case PM_TOKEN_EMBEXPR_BEGIN: { @@ -17427,7 +17474,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_token_t opening = not_provided(parser); pm_token_t closing = not_provided(parser); pm_interpolated_string_node_t *interpolated = pm_interpolated_string_node_create(parser, &opening, NULL, &closing); - pm_interpolated_string_node_append(interpolated, current); + pm_interpolated_string_node_append(parser, interpolated, current); current = (pm_node_t *) interpolated; } else if (PM_NODE_TYPE_P(current, PM_INTERPOLATED_STRING_NODE)) { // If we hit an embedded expression and the current @@ -17438,7 +17485,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b } pm_node_t *part = parse_string_part(parser); - pm_interpolated_string_node_append((pm_interpolated_string_node_t *) current, part); + pm_interpolated_string_node_append(parser, (pm_interpolated_string_node_t *) current, part); break; } default: diff --git a/test/prism/snapshots/alias.txt b/test/prism/snapshots/alias.txt index 687082d85e2e31..a952e96f67721f 100644 --- a/test/prism/snapshots/alias.txt +++ b/test/prism/snapshots/alias.txt @@ -57,7 +57,7 @@ │ │ ├── opening_loc: (7,6)-(7,8) = ":\"" │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (7,8)-(7,11)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (7,8)-(7,11) = "abc" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/dash_heredocs.txt b/test/prism/snapshots/dash_heredocs.txt index 9af3acf9c2e6e8..bd2b05ea0d0717 100644 --- a/test/prism/snapshots/dash_heredocs.txt +++ b/test/prism/snapshots/dash_heredocs.txt @@ -79,10 +79,11 @@ │ ├── closing_loc: (23,0)-(24,0) = " EOF\n" │ └── unescaped: " a\n b\n" ├── @ InterpolatedStringNode (location: (25,0)-(25,8)) + │ ├── flags: ∅ │ ├── opening_loc: (25,0)-(25,8) = "<<-\"EOF\"" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (26,0)-(27,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (26,0)-(27,0) = " a\n" │ │ │ ├── closing_loc: ∅ @@ -104,17 +105,18 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (27,3)-(27,4) = "}" │ │ └── @ StringNode (location: (27,4)-(28,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (27,4)-(28,0) = "\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "\n" │ └── closing_loc: (28,0)-(29,0) = "EOF\n" ├── @ InterpolatedStringNode (location: (30,0)-(30,6)) + │ ├── flags: ∅ │ ├── opening_loc: (30,0)-(30,6) = "<<-EOF" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (31,0)-(32,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (31,0)-(32,0) = " a\n" │ │ │ ├── closing_loc: ∅ @@ -136,7 +138,7 @@ │ │ │ │ └── block: ∅ │ │ │ └── closing_loc: (32,3)-(32,4) = "}" │ │ └── @ StringNode (location: (32,4)-(33,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (32,4)-(33,0) = "\n" │ │ ├── closing_loc: ∅ @@ -184,10 +186,11 @@ │ │ ├── flags: ∅ │ │ └── arguments: (length: 1) │ │ └── @ InterpolatedStringNode (location: (49,7)-(49,11)) + │ │ ├── flags: ∅ │ │ ├── opening_loc: (49,7)-(49,11) = "<<-B" │ │ ├── parts: (length: 3) │ │ │ ├── @ StringNode (location: (52,0)-(53,2)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (52,0)-(53,2) = " b\n " │ │ │ │ ├── closing_loc: ∅ @@ -202,7 +205,7 @@ │ │ │ │ │ └── value: 2 │ │ │ │ └── closing_loc: (54,2)-(54,3) = "}" │ │ │ └── @ StringNode (location: (54,3)-(55,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (54,3)-(55,0) = "\n" │ │ │ ├── closing_loc: ∅ @@ -228,10 +231,11 @@ │ ├── flags: ∅ │ └── arguments: (length: 1) │ └── @ InterpolatedStringNode (location: (57,7)-(57,11)) + │ ├── flags: ∅ │ ├── opening_loc: (57,7)-(57,11) = "<<-B" │ ├── parts: (length: 3) │ │ ├── @ StringNode (location: (60,0)-(61,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (60,0)-(61,2) = " b\n " │ │ │ ├── closing_loc: ∅ @@ -246,7 +250,7 @@ │ │ │ │ └── value: 2 │ │ │ └── closing_loc: (62,3)-(62,4) = "}" │ │ └── @ StringNode (location: (62,4)-(63,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (62,4)-(63,0) = "\n" │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/dos_endings.txt b/test/prism/snapshots/dos_endings.txt index c5b962f2181cf3..1ae15e1e87c081 100644 --- a/test/prism/snapshots/dos_endings.txt +++ b/test/prism/snapshots/dos_endings.txt @@ -15,16 +15,17 @@ │ │ ├── flags: ∅ │ │ └── arguments: (length: 1) │ │ └── @ InterpolatedStringNode (location: (1,5)-(2,12)) + │ │ ├── flags: ∅ │ │ ├── opening_loc: ∅ │ │ ├── parts: (length: 2) │ │ │ ├── @ StringNode (location: (1,5)-(1,9)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: (1,5)-(1,6) = "\"" │ │ │ │ ├── content_loc: (1,6)-(1,8) = "hi" │ │ │ │ ├── closing_loc: (1,8)-(1,9) = "\"" │ │ │ │ └── unescaped: "hi" │ │ │ └── @ StringNode (location: (2,5)-(2,12)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (2,5)-(2,6) = "\"" │ │ │ ├── content_loc: (2,6)-(2,11) = "there" │ │ │ ├── closing_loc: (2,11)-(2,12) = "\"" @@ -81,16 +82,17 @@ │ │ ├── flags: ∅ │ │ ├── receiver: │ │ │ @ InterpolatedStringNode (location: (17,8)-(17,14)) + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (17,8)-(17,14) = "<<~EOF" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ StringNode (location: (18,0)-(19,0)) - │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ ├── flags: frozen │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ ├── content_loc: (18,0)-(19,0) = "\r\n" │ │ │ │ │ ├── closing_loc: ∅ │ │ │ │ │ └── unescaped: "\n" │ │ │ │ └── @ StringNode (location: (19,0)-(20,0)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (19,0)-(20,0) = " baz\r\n" │ │ │ │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/dstring.txt b/test/prism/snapshots/dstring.txt index ad395f8a8eb94b..3978a2f0879e50 100644 --- a/test/prism/snapshots/dstring.txt +++ b/test/prism/snapshots/dstring.txt @@ -10,10 +10,11 @@ │ ├── closing_loc: (2,5)-(2,6) = "\"" │ └── unescaped: "foo\n bar" ├── @ InterpolatedStringNode (location: (4,0)-(5,9)) + │ ├── flags: ∅ │ ├── opening_loc: (4,0)-(4,1) = "\"" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (4,1)-(5,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (4,1)-(5,2) = "foo\n " │ │ │ ├── closing_loc: ∅ @@ -36,16 +37,17 @@ │ │ └── closing_loc: (5,7)-(5,8) = "}" │ └── closing_loc: (5,8)-(5,9) = "\"" ├── @ InterpolatedStringNode (location: (7,0)-(9,2)) + │ ├── flags: ∅ │ ├── opening_loc: ∅ │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (7,0)-(8,2)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: (7,0)-(7,1) = "\"" │ │ │ ├── content_loc: (7,1)-(8,1) = "fo\no" │ │ │ ├── closing_loc: (8,1)-(8,2) = "\"" │ │ │ └── unescaped: "fo\no" │ │ └── @ StringNode (location: (8,3)-(9,2)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: (8,3)-(8,4) = "\"" │ │ ├── content_loc: (8,4)-(9,1) = "ba\nr" │ │ ├── closing_loc: (9,1)-(9,2) = "\"" diff --git a/test/prism/snapshots/heredocs_leading_whitespace.txt b/test/prism/snapshots/heredocs_leading_whitespace.txt index 332dfa29862d0d..dbcb0d5a19aed9 100644 --- a/test/prism/snapshots/heredocs_leading_whitespace.txt +++ b/test/prism/snapshots/heredocs_leading_whitespace.txt @@ -28,32 +28,34 @@ │ ├── closing_loc: (19,0)-(20,0) = " FOO\n" │ └── unescaped: "a\nb\n" ├── @ InterpolatedStringNode (location: (21,0)-(21,10)) + │ ├── flags: ∅ │ ├── opening_loc: (21,0)-(21,10) = "<<~' FOO'" │ ├── parts: (length: 2) │ │ ├── @ StringNode (location: (22,0)-(23,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (22,0)-(23,0) = "a\n" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "a\n" │ │ └── @ StringNode (location: (23,0)-(24,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (23,0)-(24,0) = "b\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "b\n" │ └── closing_loc: (24,0)-(25,0) = " FOO\n" └── @ InterpolatedStringNode (location: (26,0)-(26,10)) + ├── flags: ∅ ├── opening_loc: (26,0)-(26,10) = "<<~' FOO'" ├── parts: (length: 2) │ ├── @ StringNode (location: (27,0)-(28,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (27,0)-(28,0) = "a\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "a\n" │ └── @ StringNode (location: (28,0)-(29,0)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (28,0)-(29,0) = "b\n" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/heredocs_nested.txt b/test/prism/snapshots/heredocs_nested.txt index 1717aadbbce9c8..f830b028c78622 100644 --- a/test/prism/snapshots/heredocs_nested.txt +++ b/test/prism/snapshots/heredocs_nested.txt @@ -4,10 +4,11 @@ @ StatementsNode (location: (1,0)-(12,4)) └── body: (length: 2) ├── @ InterpolatedStringNode (location: (1,0)-(1,7)) + │ ├── flags: ∅ │ ├── opening_loc: (1,0)-(1,7) = "<<~RUBY" │ ├── parts: (length: 4) │ │ ├── @ StringNode (location: (2,0)-(3,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (2,0)-(3,0) = "pre\n" │ │ │ ├── closing_loc: ∅ @@ -25,19 +26,20 @@ │ │ │ │ └── unescaped: " hello\n" │ │ │ └── closing_loc: (7,0)-(7,1) = "}" │ │ ├── @ StringNode (location: (7,1)-(8,0)) - │ │ │ ├── flags: ∅ + │ │ │ ├── flags: frozen │ │ │ ├── opening_loc: ∅ │ │ │ ├── content_loc: (7,1)-(8,0) = "\n" │ │ │ ├── closing_loc: ∅ │ │ │ └── unescaped: "\n" │ │ └── @ StringNode (location: (8,0)-(9,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (8,0)-(9,0) = "post\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "post\n" │ └── closing_loc: (9,0)-(10,0) = "RUBY\n" └── @ InterpolatedStringNode (location: (12,0)-(12,4)) + ├── flags: ∅ ├── opening_loc: (12,0)-(12,4) = "<<-A" ├── parts: (length: 2) │ ├── @ EmbeddedStatementsNode (location: (13,0)-(21,1)) @@ -46,6 +48,7 @@ │ │ │ @ StatementsNode (location: (14,0)-(14,4)) │ │ │ └── body: (length: 1) │ │ │ └── @ InterpolatedStringNode (location: (14,0)-(14,4)) + │ │ │ ├── flags: ∅ │ │ │ ├── opening_loc: (14,0)-(14,4) = "<<-B" │ │ │ ├── parts: (length: 2) │ │ │ │ ├── @ EmbeddedStatementsNode (location: (15,0)-(19,1)) @@ -54,6 +57,7 @@ │ │ │ │ │ │ @ StatementsNode (location: (16,0)-(16,4)) │ │ │ │ │ │ └── body: (length: 1) │ │ │ │ │ │ └── @ InterpolatedStringNode (location: (16,0)-(16,4)) + │ │ │ │ │ │ ├── flags: ∅ │ │ │ │ │ │ ├── opening_loc: (16,0)-(16,4) = "<<-C" │ │ │ │ │ │ ├── parts: (length: 2) │ │ │ │ │ │ │ ├── @ EmbeddedStatementsNode (location: (17,0)-(17,4)) @@ -66,7 +70,7 @@ │ │ │ │ │ │ │ │ │ └── value: 3 │ │ │ │ │ │ │ │ └── closing_loc: (17,3)-(17,4) = "}" │ │ │ │ │ │ │ └── @ StringNode (location: (17,4)-(18,0)) - │ │ │ │ │ │ │ ├── flags: ∅ + │ │ │ │ │ │ │ ├── flags: frozen │ │ │ │ │ │ │ ├── opening_loc: ∅ │ │ │ │ │ │ │ ├── content_loc: (17,4)-(18,0) = "\n" │ │ │ │ │ │ │ ├── closing_loc: ∅ @@ -74,7 +78,7 @@ │ │ │ │ │ │ └── closing_loc: (18,0)-(19,0) = "C\n" │ │ │ │ │ └── closing_loc: (19,0)-(19,1) = "}" │ │ │ │ └── @ StringNode (location: (19,1)-(20,0)) - │ │ │ │ ├── flags: ∅ + │ │ │ │ ├── flags: frozen │ │ │ │ ├── opening_loc: ∅ │ │ │ │ ├── content_loc: (19,1)-(20,0) = "\n" │ │ │ │ ├── closing_loc: ∅ @@ -82,7 +86,7 @@ │ │ │ └── closing_loc: (20,0)-(21,0) = "B\n" │ │ └── closing_loc: (21,0)-(21,1) = "}" │ └── @ StringNode (location: (21,1)-(22,0)) - │ ├── flags: ∅ + │ ├── flags: frozen │ ├── opening_loc: ∅ │ ├── content_loc: (21,1)-(22,0) = "\n" │ ├── closing_loc: ∅ diff --git a/test/prism/snapshots/heredocs_with_ignored_newlines.txt b/test/prism/snapshots/heredocs_with_ignored_newlines.txt index cdc0b4faab9279..0f964ec294185b 100644 --- a/test/prism/snapshots/heredocs_with_ignored_newlines.txt +++ b/test/prism/snapshots/heredocs_with_ignored_newlines.txt @@ -10,58 +10,59 @@ │ ├── closing_loc: (2,0)-(3,0) = "HERE\n" │ └── unescaped: "" └── @ InterpolatedStringNode (location: (4,0)-(4,8)) + ├── flags: ∅ ├── opening_loc: (4,0)-(4,8) = "<<~THERE" ├── parts: (length: 9) │ ├── @ StringNode (location: (5,0)-(6,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (5,0)-(6,0) = " way over\n" │ │ ├── closing_loc: ∅ │ │ └── unescaped: "way over\n" │ ├── @ StringNode (location: (6,0)-(7,0)) - │ │ ├── flags: ∅ + │ │ ├── flags: frozen │ │ ├── opening_loc: ∅ │ │ ├── content_loc: (6,0)-(7,0) = " < Date: Tue, 26 Mar 2024 09:33:03 -0400 Subject: [PATCH 23/58] [ruby/prism] Handle regexp split between heredocs https://github.com/ruby/prism/commit/c1400d8aed --- prism/prism.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index e97af236769156..e02bceed657ad3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4345,10 +4345,10 @@ pm_interpolated_string_node_append(pm_parser_t *parser, pm_interpolated_string_n if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { switch (parser->frozen_string_literal) { case PM_OPTIONS_FROZEN_STRING_LITERAL_DISABLED: - pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); break; case PM_OPTIONS_FROZEN_STRING_LITERAL_ENABLED: - pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE); + pm_node_flag_set((pm_node_t *) node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN); break; } } @@ -10900,11 +10900,15 @@ parser_lex(pm_parser_t *parser) { // the list of newlines. if (parser->heredoc_end == NULL) { pm_newline_list_append(&parser->newline_list, breakpoint); + parser->current.end = breakpoint + 1; + breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); + break; } parser->current.end = breakpoint + 1; - breakpoint = pm_strpbrk(parser, parser->current.end, breakpoints, parser->end - parser->current.end, false); - break; + parser_flush_heredoc_end(parser); + pm_regexp_token_buffer_flush(parser, &token_buffer); + LEX(PM_TOKEN_STRING_CONTENT); case '\\': { // If we hit escapes, then we need to treat the next // token literally. In this case we'll skip past the From 8ec7c3ce3069177a9e8eb9ca93cced9aa0533085 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 10:31:51 -0400 Subject: [PATCH 24/58] [ruby/prism] Properly handle freeing ephemeral node lists https://github.com/ruby/prism/commit/f49261a9b9 --- prism/prism.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index e02bceed657ad3..4f65e1e0173e33 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -4411,6 +4411,12 @@ pm_interpolated_symbol_node_append(pm_interpolated_symbol_node_t *node, pm_node_ node->base.location.end = MAX(node->base.location.end, part->location.end); } +static void +pm_interpolated_symbol_node_closing_loc_set(pm_interpolated_symbol_node_t *node, const pm_token_t *closing) { + node->closing_loc = PM_OPTIONAL_LOCATION_TOKEN_VALUE(closing); + node->base.location.end = closing->end; +} + /** * Allocate and initialize a new InterpolatedSymbolNode node. */ @@ -14151,14 +14157,12 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s return (pm_node_t *) pm_string_node_to_symbol_node(parser, (pm_string_node_t *) part, &opening, &parser->previous); } - // Create a node_list first. We'll use this to check if it should be an - // InterpolatedSymbolNode or a SymbolNode. - pm_node_list_t node_list = { 0 }; - if (part) pm_node_list_append(&node_list, part); + pm_interpolated_symbol_node_t *symbol = pm_interpolated_symbol_node_create(parser, &opening, NULL, &opening); + if (part) pm_interpolated_symbol_node_append(symbol, part); while (!match2(parser, PM_TOKEN_STRING_END, PM_TOKEN_EOF)) { if ((part = parse_string_part(parser)) != NULL) { - pm_node_list_append(&node_list, part); + pm_interpolated_symbol_node_append(symbol, part); } } @@ -14169,7 +14173,8 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_INTERPOLATED); } - return (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &node_list, &parser->previous); + pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + return (pm_node_t *) symbol; } pm_token_t content; @@ -14190,14 +14195,14 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s // In this case, the best way we have to represent this is as an // interpolated string node, so that's what we'll do here. if (match1(parser, PM_TOKEN_STRING_CONTENT)) { - pm_node_list_t parts = { 0 }; + pm_interpolated_symbol_node_t *symbol = pm_interpolated_symbol_node_create(parser, &opening, NULL, &opening); pm_token_t bounds = not_provided(parser); pm_node_t *part = (pm_node_t *) pm_string_node_create_unescaped(parser, &bounds, &content, &bounds, &unescaped); - pm_node_list_append(&parts, part); + pm_interpolated_symbol_node_append(symbol, part); part = (pm_node_t *) pm_string_node_create_unescaped(parser, &bounds, &parser->current, &bounds, &parser->current_string); - pm_node_list_append(&parts, part); + pm_interpolated_symbol_node_append(symbol, part); if (next_state != PM_LEX_STATE_NONE) { lex_state_set(parser, next_state); @@ -14205,7 +14210,9 @@ parse_symbol(pm_parser_t *parser, pm_lex_mode_t *lex_mode, pm_lex_state_t next_s parser_lex(parser); expect1(parser, PM_TOKEN_STRING_END, PM_ERR_SYMBOL_TERM_DYNAMIC); - return (pm_node_t *) pm_interpolated_symbol_node_create(parser, &opening, &parts, &parser->previous); + + pm_interpolated_symbol_node_closing_loc_set(symbol, &parser->previous); + return (pm_node_t *) symbol; } } else { content = (pm_token_t) { .type = PM_TOKEN_STRING_CONTENT, .start = parser->previous.end, .end = parser->previous.end }; @@ -15373,6 +15380,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_LITERAL_EOF); node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); + + pm_node_list_free(&parts); } else if (accept1(parser, PM_TOKEN_LABEL_END) && !state_is_arg_labeled) { node = (pm_node_t *) pm_symbol_node_create_unescaped(parser, &opening, &content, &parser->previous, &unescaped, parse_symbol_encoding(parser, &unescaped)); } else if (match1(parser, PM_TOKEN_EOF)) { @@ -15427,6 +15436,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); } + + pm_node_list_free(&parts); } } else { // If we get here, then the first part of the string is not plain @@ -15450,6 +15461,8 @@ parse_strings(pm_parser_t *parser, pm_node_t *current) { expect1(parser, PM_TOKEN_STRING_END, PM_ERR_STRING_INTERPOLATED_TERM); node = (pm_node_t *) pm_interpolated_string_node_create(parser, &opening, &parts, &parser->previous); } + + pm_node_list_free(&parts); } if (current == NULL) { @@ -16058,6 +16071,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b node = (pm_node_t *) cast; } else { pm_interpolated_string_node_t *cast = pm_interpolated_string_node_create(parser, &opening, &parts, &opening); + pm_node_list_free(&parts); lex_mode_pop(parser); expect1(parser, PM_TOKEN_HEREDOC_END, PM_ERR_HEREDOC_TERM); From 4300c42a7eb12a0aa81e4ec79891cdba1cfdf3aa Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 09:58:24 -0400 Subject: [PATCH 25/58] [PRISM] Better handle interpolated* nodes with inner frozen parts --- prism_compile.c | 367 ++++++++++++++++++++++++++++++------------------ 1 file changed, 234 insertions(+), 133 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index e919a44c0885ae..db032e2a0530a5 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -129,7 +129,8 @@ parse_integer(const pm_integer_node_t *node) if (integer->values == NULL) { result = UINT2NUM(integer->value); - } else { + } + else { VALUE string = rb_str_new(NULL, integer->length * 8); unsigned char *bytes = (unsigned char *) RSTRING_PTR(string); @@ -251,9 +252,11 @@ parse_string_encoded(const pm_scope_node_t *scope_node, const pm_node_t *node, c if (node->flags & PM_ENCODING_FLAGS_FORCED_BINARY_ENCODING) { encoding = rb_ascii8bit_encoding(); - } else if (node->flags & PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING) { + } + else if (node->flags & PM_ENCODING_FLAGS_FORCED_UTF8_ENCODING) { encoding = rb_utf8_encoding(); - } else { + } + else { encoding = scope_node->encoding; } @@ -335,21 +338,21 @@ pm_reg_flags(const pm_node_t *node) { } static rb_encoding * -pm_reg_enc(const pm_scope_node_t *scope_node, const pm_regular_expression_node_t *node) +pm_reg_enc(const pm_scope_node_t *scope_node, const pm_node_t *node) { - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT)) { return rb_ascii8bit_encoding(); } - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { return rb_enc_get_from_index(ENCINDEX_EUC_JP); } - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { return rb_enc_get_from_index(ENCINDEX_Windows_31J); } - if (node->base.flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { return rb_utf8_encoding(); } @@ -368,12 +371,12 @@ pm_static_literal_p(const pm_node_t *node) } static VALUE -pm_new_regex(const pm_scope_node_t *scope_node, const pm_regular_expression_node_t *node) +pm_new_regex(const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) { - VALUE regex_str = parse_string(scope_node, &node->unescaped); + VALUE regex_str = parse_string(scope_node, unescaped); rb_encoding *enc = pm_reg_enc(scope_node, node); - VALUE regex = rb_enc_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), enc, pm_reg_flags((const pm_node_t *) node)); + VALUE regex = rb_enc_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), enc, pm_reg_flags(node)); RB_GC_GUARD(regex_str); rb_obj_freeze(regex); @@ -381,12 +384,44 @@ pm_new_regex(const pm_scope_node_t *scope_node, const pm_regular_expression_node return regex; } +static VALUE +pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +{ + VALUE current = Qnil; + + for (size_t index = 0; index < nodes->size; index++) { + const pm_node_t *part = nodes->nodes[index]; + VALUE string; + + switch (PM_NODE_TYPE(part)) { + case PM_STRING_NODE: + string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); + break; + case PM_INTERPOLATED_STRING_NODE: + string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); + break; + default: + RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); + return Qnil; + } + + if (current != Qnil) { + current = rb_str_concat(current, string); + } + else { + current = string; + } + } + + return top ? rb_fstring(current) : current; +} + /** * Certain nodes can be compiled literally. This function returns the literal * value described by the given node. For example, an array node with all static * literal values can be compiled into a literal array. */ -static inline VALUE +static VALUE pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node) { // Every node that comes into this function should already be marked as @@ -433,12 +468,40 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node return parse_imaginary((pm_imaginary_node_t *) node); case PM_INTEGER_NODE: return parse_integer((const pm_integer_node_t *) node); + case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: { + const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; + VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + + rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); + return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + } + case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { + const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; + VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + + rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); + return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + } + case PM_INTERPOLATED_STRING_NODE: + return pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, true); + case PM_INTERPOLATED_SYMBOL_NODE: { + const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; + VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); + + return ID2SYM(rb_intern_str(string)); + } + case PM_MATCH_LAST_LINE_NODE: { + const pm_match_last_line_node_t *cast = (const pm_match_last_line_node_t *) node; + return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + } case PM_NIL_NODE: return Qnil; case PM_RATIONAL_NODE: return parse_rational((const pm_rational_node_t *) node); - case PM_REGULAR_EXPRESSION_NODE: - return pm_new_regex(scope_node, (const pm_regular_expression_node_t *) node); + case PM_REGULAR_EXPRESSION_NODE: { + const pm_regular_expression_node_t *cast = (const pm_regular_expression_node_t *) node; + return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + } case PM_SOURCE_ENCODING_NODE: return rb_enc_from_encoding(scope_node->encoding); case PM_SOURCE_FILE_NODE: { @@ -757,7 +820,8 @@ pm_compile_loop(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_node_fl PUSH_LABEL(ret, next_label); if (type == PM_WHILE_NODE) { pm_compile_branch_condition(iseq, ret, predicate, redo_label, end_label, popped, scope_node); - } else if (type == PM_UNTIL_NODE) { + } + else if (type == PM_UNTIL_NODE) { pm_compile_branch_condition(iseq, ret, predicate, end_label, redo_label, popped, scope_node); } @@ -779,7 +843,7 @@ pm_compile_loop(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_node_fl } static int -pm_interpolated_node_compile(pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +pm_interpolated_node_compile(const pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) { int number_of_items_pushed = 0; size_t parts_size = parts->size; @@ -787,44 +851,6 @@ pm_interpolated_node_compile(pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_ if (parts_size > 0) { VALUE current_string = Qnil; - bool literal = true; - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - - if (!PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - literal = false; - break; - } - } - - if (literal) { - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - const pm_string_node_t *string_node = (const pm_string_node_t *)part; - VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - - const pm_node_t *part = parts->nodes[0]; - current_string = rb_fstring(current_string); - if (PM_NODE_FLAG_P(part, PM_STRING_FLAGS_FROZEN)) { - ADD_INSN1(ret, &dummy_line_node, putobject, current_string); - } - else if (PM_NODE_FLAG_P(part, PM_STRING_FLAGS_MUTABLE)) { - ADD_INSN1(ret, &dummy_line_node, putstring, current_string); - } - else { - ADD_INSN1(ret, &dummy_line_node, putchilledstring, current_string); - } - return 1; - } - for (size_t index = 0; index < parts_size; index++) { const pm_node_t *part = parts->nodes[index]; @@ -882,6 +908,7 @@ pm_interpolated_node_compile(pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_ else { PM_PUTNIL; } + return number_of_items_pushed; } @@ -904,7 +931,8 @@ pm_lookup_local_index(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, pm_con if (scope_node->previous) { scope_node = scope_node->previous; - } else { + } + else { // We have recursed up all scope nodes // and have not found the local yet rb_bug("Local with constant_id %u does not exist", (unsigned int) constant_id); @@ -1213,7 +1241,8 @@ pm_setup_args_core(const pm_arguments_node_t *arguments_node, const pm_node_t *b } RUBY_ASSERT(keyword_index == size); - } else { + } + else { // If they aren't all symbol keys then we need to // construct a new hash and pass that as an argument. orig_argc++; @@ -1819,7 +1848,8 @@ pm_compile_pattern_deconstruct(rb_iseq_t *iseq, pm_scope_node_t *scope_node, con ADD_INSN(ret, &line.node, pop); ADD_INSN1(ret, &line.node, topn, INT2FIX(base_index + PM_PATTERN_BASE_INDEX_OFFSET_DECONSTRUCTED_CACHE - 1)); ADD_INSNL(ret, &line.node, jump, deconstructed_label); - } else { + } + else { ADD_INSNL(ret, &line.node, jump, deconstruct_label); } @@ -2006,7 +2036,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t ADD_INSN1(ret, &line.node, setn, INT2FIX(4)); ADD_SEND(ret, &line.node, idAREF, INT2FIX(2)); CHECK(pm_compile_pattern_match(iseq, scope_node, ((const pm_splat_node_t *) cast->rest)->expression, ret, match_failed_label, in_single_pattern, in_alternation_pattern, false, base_index + 1)); - } else if (posts_size > 0) { + } + else if (posts_size > 0) { ADD_INSN(ret, &line.node, dup); ADD_SEND(ret, &line.node, idLength, INT2FIX(0)); ADD_INSN1(ret, &line.node, putobject, INT2FIX(minimum_size)); @@ -2237,7 +2268,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t if (NIL_P(keys)) { ADD_INSN(ret, &line.node, putnil); - } else { + } + else { ADD_INSN1(ret, &line.node, duparray, keys); RB_OBJ_WRITTEN(iseq, Qundef, rb_obj_hide(keys)); } @@ -2301,7 +2333,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t } ADD_SEQ(ret, match_values); - } else { + } + else { ADD_INSN(ret, &line.node, dup); ADD_SEND(ret, &line.node, idEmptyP, INT2FIX(0)); if (in_single_pattern) { @@ -2516,7 +2549,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t RUBY_ASSERT(cast->statements != NULL && cast->statements->body.size == 1); statement = cast->statements->body.nodes[0]; - } else { + } + else { const pm_unless_node_t *cast = (const pm_unless_node_t *) node; predicate = cast->predicate; @@ -2533,7 +2567,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t ADD_INSN(ret, &line.node, dup); if (PM_NODE_TYPE_P(node, PM_IF_NODE)) { ADD_INSNL(ret, &line.node, branchif, match_succeeded_label); - } else { + } + else { ADD_INSNL(ret, &line.node, branchunless, match_succeeded_label); } @@ -2550,7 +2585,8 @@ pm_compile_pattern(rb_iseq_t *iseq, pm_scope_node_t *scope_node, const pm_node_t if (PM_NODE_TYPE_P(node, PM_IF_NODE)) { ADD_INSNL(ret, &line.node, branchunless, unmatched_label); - } else { + } + else { ADD_INSNL(ret, &line.node, branchif, unmatched_label); } @@ -3254,7 +3290,8 @@ pm_compile_destructured_param_locals(const pm_multi_target_node_t *node, st_tabl pm_insert_local_index(((const pm_required_parameter_node_t *) left)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); local_index++; } - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(left, PM_MULTI_TARGET_NODE)); local_index = pm_compile_destructured_param_locals((const pm_multi_target_node_t *) left, index_lookup_table, local_table_for_iseq, scope_node, local_index); } @@ -3276,7 +3313,8 @@ pm_compile_destructured_param_locals(const pm_multi_target_node_t *node, st_tabl if (PM_NODE_TYPE_P(right, PM_REQUIRED_PARAMETER_NODE)) { pm_insert_local_index(((const pm_required_parameter_node_t *) right)->name, local_index, index_lookup_table, local_table_for_iseq, scope_node); local_index++; - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(right, PM_MULTI_TARGET_NODE)); local_index = pm_compile_destructured_param_locals((const pm_multi_target_node_t *) right, index_lookup_table, local_table_for_iseq, scope_node, local_index); } @@ -3324,7 +3362,8 @@ pm_compile_destructured_param_writes(rb_iseq_t *iseq, const pm_multi_target_node if (PM_NODE_TYPE_P(left, PM_REQUIRED_PARAMETER_NODE)) { pm_compile_destructured_param_write(iseq, (const pm_required_parameter_node_t *) left, ret, scope_node); - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(left, PM_MULTI_TARGET_NODE)); pm_compile_destructured_param_writes(iseq, (const pm_multi_target_node_t *) left, ret, scope_node); } @@ -3351,7 +3390,8 @@ pm_compile_destructured_param_writes(rb_iseq_t *iseq, const pm_multi_target_node if (PM_NODE_TYPE_P(right, PM_REQUIRED_PARAMETER_NODE)) { pm_compile_destructured_param_write(iseq, (const pm_required_parameter_node_t *) right, ret, scope_node); - } else { + } + else { RUBY_ASSERT(PM_NODE_TYPE_P(right, PM_MULTI_TARGET_NODE)); pm_compile_destructured_param_writes(iseq, (const pm_multi_target_node_t *) right, ret, scope_node); } @@ -3420,7 +3460,8 @@ pm_multi_target_state_push(pm_multi_target_state_t *state, INSN *topn, size_t st if (state->head == NULL) { state->head = node; state->tail = node; - } else { + } + else { state->tail->next = node; state->tail = node; } @@ -3585,13 +3626,15 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons if (cast->parent != NULL) { pm_compile_node(iseq, cast->parent, parents, false, scope_node); - } else { + } + else { ADD_INSN1(parents, &dummy_line_node, putobject, rb_cObject); } if (state == NULL) { ADD_INSN(writes, &dummy_line_node, swap); - } else { + } + else { ADD_INSN1(writes, &dummy_line_node, topn, INT2FIX(1)); pm_multi_target_state_push(state, (INSN *) LAST_ELEMENT(writes), 1); } @@ -3660,7 +3703,8 @@ pm_compile_target_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *cons if (argc == 0) { ADD_INSN(writes, &dummy_line_node, swap); - } else { + } + else { for (int index = 0; index < argc; index++) { ADD_INSN1(writes, &dummy_line_node, topn, INT2FIX(argc + 1)); } @@ -4529,7 +4573,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ADD_ADJUST_RESTORE(ret, splabel); PM_PUTNIL_UNLESS_POPPED; - } else { + } + else { const rb_iseq_t *ip = iseq; while (ip) { @@ -4985,7 +5030,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, branch_id++; if (in_node->statements != NULL) { PM_COMPILE_INTO_ANCHOR(body_seq, (const pm_node_t *) in_node->statements); - } else if (!popped) { + } + else if (!popped) { ADD_INSN(body_seq, &in_line.node, putnil); } @@ -5013,7 +5059,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (else_node->statements != NULL) { PM_COMPILE_INTO_ANCHOR(cond_seq, (const pm_node_t *) else_node->statements); - } else if (!popped) { + } + else if (!popped) { ADD_INSN(cond_seq, &dummy_line_node, putnil); } @@ -5022,7 +5069,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (popped) { ADD_INSN(cond_seq, &dummy_line_node, putnil); } - } else { + } + else { // Otherwise, if we do not have an `else` clause, we will compile in // the code to handle raising an appropriate error. ADD_LABEL(cond_seq, else_label); @@ -5032,7 +5080,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (in_single_pattern) { pm_compile_pattern_error_handler(iseq, scope_node, node, cond_seq, end_label, popped); - } else { + } + else { ADD_INSN1(cond_seq, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); ADD_INSN1(cond_seq, &dummy_line_node, putobject, rb_eNoMatchingPatternError); ADD_INSN1(cond_seq, &dummy_line_node, topn, INT2FIX(2)); @@ -5846,7 +5895,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ADD_INSN1(ret, &dummy_line_node, duphash, value); RB_OBJ_WRITTEN(iseq, Qundef, value); } - } else { + } + else { // Here since we know there are possible side-effects inside the // hash contents, we're going to build it entirely at runtime. We'll // do this by pushing all of the key-value pairs onto the stack and @@ -6018,71 +6068,114 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: { - pm_interpolated_match_last_line_node_t *cast = (pm_interpolated_match_last_line_node_t *) node; - - int parts_size = (int)cast->parts.size; - - pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - - ADD_INSN2(ret, &dummy_line_node, toregexp, INT2FIX(pm_reg_flags(node)), INT2FIX(parts_size)); + // if /foo #{bar}/ then end + // ^^^^^^^^^^^^ + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE regexp = pm_static_literal_value(node, scope_node); + PUSH_INSN1(ret, location, putobject, regexp); + } + } + else { + const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; + int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); + PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + } - ADD_INSN1(ret, &dummy_line_node, getglobal, rb_id2sym(idLASTLINE)); - ADD_SEND(ret, &dummy_line_node, idEqTilde, INT2NUM(1)); - PM_POP_IF_POPPED; + PUSH_INSN1(ret, location, getglobal, rb_id2sym(idLASTLINE)); + PUSH_SEND(ret, location, idEqTilde, INT2NUM(1)); + if (popped) PUSH_INSN(ret, location, pop); return; } case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_ONCE) { + // /foo #{bar}/ + // ^^^^^^^^^^^^ + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ONCE)) { const rb_iseq_t *prevblock = ISEQ_COMPILE_DATA(iseq)->current_block; const rb_iseq_t *block_iseq = NULL; - int ic_index = ISEQ_BODY(iseq)->ise_size++; + int ise_index = ISEQ_BODY(iseq)->ise_size++; pm_scope_node_t next_scope_node; - pm_scope_node_init((pm_node_t*)node, &next_scope_node, scope_node); - block_iseq = NEW_CHILD_ISEQ(&next_scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, lineno); + pm_scope_node_init(node, &next_scope_node, scope_node); + + block_iseq = NEW_CHILD_ISEQ(&next_scope_node, make_name_for_block(iseq), ISEQ_TYPE_BLOCK, location.line); pm_scope_node_destroy(&next_scope_node); ISEQ_COMPILE_DATA(iseq)->current_block = block_iseq; - - ADD_INSN2(ret, &dummy_line_node, once, block_iseq, INT2FIX(ic_index)); - + PUSH_INSN2(ret, location, once, block_iseq, INT2FIX(ise_index)); ISEQ_COMPILE_DATA(iseq)->current_block = prevblock; + return; } - pm_interpolated_regular_expression_node_t *cast = (pm_interpolated_regular_expression_node_t *) node; + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE regexp = pm_static_literal_value(node, scope_node); + PUSH_INSN1(ret, location, putobject, regexp); + } + } + else { + const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; + int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - int parts_size = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); + PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + if (popped) PUSH_INSN(ret, location, pop); + } - ADD_INSN2(ret, &dummy_line_node, toregexp, INT2FIX(pm_reg_flags(node)), INT2FIX(parts_size)); - PM_POP_IF_POPPED; return; } case PM_INTERPOLATED_STRING_NODE: { - pm_interpolated_string_node_t *interp_string_node = (pm_interpolated_string_node_t *) node; - int number_of_items_pushed = pm_interpolated_node_compile(&interp_string_node->parts, iseq, dummy_line_node, ret, popped, scope_node); + // "foo #{bar}" + // ^^^^^^^^^^^^ + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE string = pm_static_literal_value(node, scope_node); - if (number_of_items_pushed > 1) { - ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX(number_of_items_pushed)); + if (PM_NODE_FLAG_P(node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN)) { + PUSH_INSN1(ret, location, putobject, string); + } + else if (PM_NODE_FLAG_P(node, PM_INTERPOLATED_STRING_NODE_FLAGS_MUTABLE)) { + PUSH_INSN1(ret, location, putstring, string); + } + else { + PUSH_INSN1(ret, location, putchilledstring, string); + } + } + } + else { + const pm_interpolated_string_node_t *cast = (const pm_interpolated_string_node_t *) node; + int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); + + if (length > 1) PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); + if (popped) PUSH_INSN(ret, location, pop); } - PM_POP_IF_POPPED; return; } case PM_INTERPOLATED_SYMBOL_NODE: { - pm_interpolated_symbol_node_t *interp_symbol_node = (pm_interpolated_symbol_node_t *) node; - int number_of_items_pushed = pm_interpolated_node_compile(&interp_symbol_node->parts, iseq, dummy_line_node, ret, popped, scope_node); - - if (number_of_items_pushed > 1) { - ADD_INSN1(ret, &dummy_line_node, concatstrings, INT2FIX(number_of_items_pushed)); - } + // :"foo #{bar}" + // ^^^^^^^^^^^^^ + const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; + int length; - if (!popped) { - ADD_INSN(ret, &dummy_line_node, intern); + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { + if (!popped) { + VALUE symbol = pm_static_literal_value(node, scope_node); + PUSH_INSN1(ret, location, putobject, symbol); + } } else { - PM_POP; + if ((length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node)) > 1) { + PUSH_INSN1(ret, location, concatstrings, INT2FIX(length)); + } + + if (!popped) { + PUSH_INSN(ret, location, intern); + } + else { + PUSH_INSN(ret, location, pop); + } } return; @@ -6228,17 +6321,14 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, return; } case PM_MATCH_LAST_LINE_NODE: { - if (!popped) { - pm_match_last_line_node_t *cast = (pm_match_last_line_node_t *) node; - - VALUE regex_str = parse_string(scope_node, &cast->unescaped); - VALUE regex = rb_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), pm_reg_flags(node)); - RB_GC_GUARD(regex_str); + // if /foo/ then end + // ^^^^^ + VALUE regexp = pm_static_literal_value(node, scope_node); - ADD_INSN1(ret, &dummy_line_node, putobject, regex); - ADD_INSN2(ret, &dummy_line_node, getspecial, INT2FIX(0), INT2FIX(0)); - ADD_SEND(ret, &dummy_line_node, idEqTilde, INT2NUM(1)); - } + PUSH_INSN1(ret, location, putobject, regexp); + PUSH_INSN2(ret, location, getspecial, INT2FIX(0), INT2FIX(0)); + PUSH_SEND(ret, location, idEqTilde, INT2NUM(1)); + if (popped) PUSH_INSN(ret, location, pop); return; } @@ -6634,7 +6724,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (cast->body != NULL) { PM_COMPILE(cast->body); - } else if (!popped) { + } + else if (!popped) { PUSH_INSN(ret, location, putnil); } @@ -6714,13 +6805,15 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, else { if (cast->left == NULL) { PUSH_INSN(ret, location, putnil); - } else { + } + else { PM_COMPILE(cast->left); } if (cast->right == NULL) { PUSH_INSN(ret, location, putnil); - } else { + } + else { PM_COMPILE(cast->right); } @@ -6837,7 +6930,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN1(ret, location, checkmatch, INT2FIX(checkmatch_flags)); PUSH_INSNL(ret, location, branchif, exception_match_label); } - } else { + } + else { ADD_GETLOCAL(ret, &dummy_line_node, LVAR_ERRINFO, 0); PUSH_INSN1(ret, location, putobject, rb_eStandardError); PUSH_INSN1(ret, location, checkmatch, INT2FIX(VM_CHECKMATCH_TYPE_RESCUE)); @@ -6885,7 +6979,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // Now restore the end_label ISEQ_COMPILE_DATA(iseq)->end_label = prev_end; - } else { + } + else { PUSH_INSN(ret, location, putnil); } @@ -6898,7 +6993,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_LABEL(ret, rescue_end_label); if (cast->consequent) { PM_COMPILE((pm_node_t *) cast->consequent); - } else { + } + else { ADD_GETLOCAL(ret, &dummy_line_node, 1, 0); } @@ -7005,7 +7101,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN(ret, location, putnil); PUSH_INSN1(ret, location, throw, INT2FIX(TAG_RETRY)); if (popped) PUSH_INSN(ret, location, pop); - } else { + } + else { COMPILE_ERROR(ERROR_ARGS "Invalid retry"); return; } @@ -7595,7 +7692,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (PM_NODE_TYPE_P(scope_node->ast_node, PM_FOR_NODE)) { if (PM_NODE_TYPE_P(((const pm_for_node_t *) scope_node->ast_node)->index, PM_LOCAL_VARIABLE_TARGET_NODE)) { body->param.lead_num++; - } else { + } + else { body->param.rest_start = local_index; body->param.flags.has_rest = true; } @@ -7823,7 +7921,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_compile_node(iseq, scope_node->body, ret, popped, scope_node); break; } - } else { + } + else { PM_PUTNIL; } @@ -7851,7 +7950,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ADD_TRACE(ret, RUBY_EVENT_CALL); if (scope_node->body) { PM_COMPILE((pm_node_t *)scope_node->body); - } else { + } + else { PM_PUTNIL; } @@ -7886,7 +7986,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, default: if (scope_node->body) { PM_COMPILE((pm_node_t *)scope_node->body); - } else { + } + else { PM_PUTNIL; } break; From 4bdb79618b33422551e96723827e50ab20e9abb1 Mon Sep 17 00:00:00 2001 From: Gannon McGibbon Date: Fri, 22 Mar 2024 15:40:42 -0500 Subject: [PATCH 26/58] Mark frame info structs with rb_gc_mark_movable Using rb_gc_mark_movable and a reference update function, we can make frame infos movable in memory, and avoid pinning frame info backtraces. ``` require "objspace" exceptions = [] GC.disable 50_000.times do begin raise "some exception" rescue => exception exception.backtrace_locations exceptions << exception end end GC.enable GC.compact p ObjectSpace.dump_all(output: :string).lines.grep(/"pinned":true/).count ``` Co-authored-by: Peter Zhu --- vm_backtrace.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index 11b26adbf49781..3fbb8d1e3b6c21 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -134,7 +134,14 @@ static void location_mark(void *ptr) { struct valued_frame_info *vfi = (struct valued_frame_info *)ptr; - rb_gc_mark(vfi->btobj); + rb_gc_mark_movable(vfi->btobj); +} + +static void +location_ref_update(void *ptr) +{ + struct valued_frame_info *vfi = ptr; + vfi->btobj = rb_gc_location(vfi->btobj); } static void @@ -150,6 +157,7 @@ static const rb_data_type_t location_data_type = { location_mark, RUBY_TYPED_DEFAULT_FREE, NULL, // No external memory to report, + location_ref_update, }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; From 3e0eea644f31b0a80da143c18149c750712d833d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Sat, 23 Mar 2024 12:08:48 -0400 Subject: [PATCH 27/58] Don't set RUBY_TYPED_EMBEDDABLE flag on backtrace --- vm_backtrace.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vm_backtrace.c b/vm_backtrace.c index 3fbb8d1e3b6c21..6f4379ad23e85c 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -540,7 +540,10 @@ static const rb_data_type_t backtrace_data_type = { NULL, // No external memory to report, backtrace_update, }, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE + /* Cannot set the RUBY_TYPED_EMBEDDABLE flag because the loc of frame_info + * points elements in the backtrace array. This can cause the loc to become + * incorrect if this backtrace object is moved by compaction. */ + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED }; int From 332f4938cf3adbff8f15b647767dc660583a5bef Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 26 Mar 2024 11:21:31 -0700 Subject: [PATCH 28/58] [DOC] Fix a description about rb_exec_recursive_outer It gives true/TRUE (int) instead of Qtrue (VALUE). --- thread.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thread.c b/thread.c index e672172a4deec3..9400c32adc1953 100644 --- a/thread.c +++ b/thread.c @@ -5297,7 +5297,7 @@ rb_exec_recursive_paired(VALUE (*func) (VALUE, VALUE, int), VALUE obj, VALUE pai /* * If recursion is detected on the current method and obj, the outermost - * func will be called with (obj, arg, Qtrue). All inner func will be + * func will be called with (obj, arg, true). All inner func will be * short-circuited using throw. */ From 16cf9047c63aad5483bebe2068cdaa832447ba77 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 26 Mar 2024 11:24:40 -0700 Subject: [PATCH 29/58] [DOC] Fix a couple other descriptions similarly to 332f4938cf3adbff8f15b647767dc660583a5bef --- thread.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/thread.c b/thread.c index 9400c32adc1953..c798c5a20467a3 100644 --- a/thread.c +++ b/thread.c @@ -5206,7 +5206,7 @@ exec_recursive_i(RB_BLOCK_CALL_FUNC_ARGLIST(tag, data)) * Calls func(obj, arg, recursive), where recursive is non-zero if the * current method is called recursively on obj, or on the pair * If outer is 0, then the innermost func will be called with recursive set - * to Qtrue, otherwise the outermost func will be called. In the latter case, + * to true, otherwise the outermost func will be called. In the latter case, * all inner func are short-circuited by throw. * Implementation details: the value thrown is the recursive list which is * proper to the current method and unlikely to be caught anywhere else. @@ -5315,7 +5315,7 @@ rb_exec_recursive_outer_mid(VALUE (*func) (VALUE, VALUE, int), VALUE obj, VALUE /* * If recursion is detected on the current method, obj and paired_obj, - * the outermost func will be called with (obj, arg, Qtrue). All inner + * the outermost func will be called with (obj, arg, true). All inner * func will be short-circuited using throw. */ From e16086b7f25d334c8049bd0e237191bdb3300d88 Mon Sep 17 00:00:00 2001 From: eileencodes Date: Mon, 25 Mar 2024 13:53:51 -0400 Subject: [PATCH 30/58] Refactor init_copy gc attributes This PR moves `rb_copy_wb_protected_attribute` and `rb_gc_copy_finalizer` into a single function called `rb_gc_copy_attributes` to be called by `init_copy`. This reduces the surface area of the GC API. Co-authored-by: Peter Zhu --- gc.c | 3 ++- internal/gc.h | 2 +- object.c | 4 +--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/gc.c b/gc.c index 4b30bb171d6fd3..8812c0fea5e986 100644 --- a/gc.c +++ b/gc.c @@ -8691,11 +8691,12 @@ rb_gc_writebarrier_remember(VALUE obj) } void -rb_copy_wb_protected_attribute(VALUE dest, VALUE obj) +rb_gc_copy_attributes(VALUE dest, VALUE obj) { if (RVALUE_WB_UNPROTECTED(obj)) { rb_gc_writebarrier_unprotect(dest); } + rb_gc_copy_finalizer(dest, obj); } size_t diff --git a/internal/gc.h b/internal/gc.h index 595dbb9ef6064f..82a1b901f8af59 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -197,7 +197,7 @@ void rb_objspace_set_event_hook(const rb_event_flag_t event); VALUE rb_objspace_gc_enable(struct rb_objspace *); VALUE rb_objspace_gc_disable(struct rb_objspace *); void ruby_gc_set_params(void); -void rb_copy_wb_protected_attribute(VALUE dest, VALUE obj); +void rb_gc_copy_attributes(VALUE dest, VALUE obj); size_t rb_size_mul_or_raise(size_t, size_t, VALUE); /* used in compile.c */ size_t rb_size_mul_add_or_raise(size_t, size_t, size_t, VALUE); /* used in iseq.h */ size_t rb_malloc_grow_capa(size_t current_capacity, size_t type_size); diff --git a/object.c b/object.c index 8eee22fbb4a73c..4673ba9f6965ce 100644 --- a/object.c +++ b/object.c @@ -396,10 +396,8 @@ init_copy(VALUE dest, VALUE obj) RBASIC(dest)->flags &= ~(T_MASK|FL_EXIVAR); // Copies the shape id from obj to dest RBASIC(dest)->flags |= RBASIC(obj)->flags & (T_MASK|FL_EXIVAR); - rb_copy_wb_protected_attribute(dest, obj); + rb_gc_copy_attributes(dest, obj); rb_copy_generic_ivar(dest, obj); - rb_gc_copy_finalizer(dest, obj); - if (RB_TYPE_P(obj, T_OBJECT)) { rb_obj_copy_ivar(dest, obj); } From 696b2716e0ab7c7df983856216d65bb5f06bf956 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Tue, 26 Mar 2024 11:32:09 -0700 Subject: [PATCH 31/58] Return stdbool from recursive_check() The return value is used as a boolean value in C. Since it's not used as a Ruby object, it just seems confusing that it returns a VALUE. --- thread.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/thread.c b/thread.c index c798c5a20467a3..1859f727ea5c2f 100644 --- a/thread.c +++ b/thread.c @@ -5098,12 +5098,12 @@ recursive_list_access(VALUE sym) } /* - * Returns Qtrue if and only if obj (or the pair ) is already + * Returns true if and only if obj (or the pair ) is already * in the recursion list. * Assumes the recursion list is valid. */ -static VALUE +static bool recursive_check(VALUE list, VALUE obj, VALUE paired_obj_id) { #if SIZEOF_LONG == SIZEOF_VOIDP @@ -5115,18 +5115,18 @@ recursive_check(VALUE list, VALUE obj, VALUE paired_obj_id) VALUE pair_list = rb_hash_lookup2(list, obj, Qundef); if (UNDEF_P(pair_list)) - return Qfalse; + return false; if (paired_obj_id) { if (!RB_TYPE_P(pair_list, T_HASH)) { if (!OBJ_ID_EQL(paired_obj_id, pair_list)) - return Qfalse; + return false; } else { if (NIL_P(rb_hash_lookup(pair_list, paired_obj_id))) - return Qfalse; + return false; } } - return Qtrue; + return true; } /* From 4a78d75213b1aee3bf0422fefaef204d06a459ac Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Fri, 22 Mar 2024 02:22:20 +0900 Subject: [PATCH 32/58] [ruby/prism] Fix an incorrect range of `Prism::Location` when `PM_ERR_RETURN_INVALID` This PR fixes the following incorrect range of `Prism::Location` when `PM_ERR_RETURN_INVALID`. It may be hard to tell from the text, but this Ruby error highlights `return`: ```console $ ruby -e 'class Foo return end' -e:1: Invalid return in class/module body class Foo return end -e: compile error (SyntaxError) ``` Previously, the error's `Prism::Location` pointed to `end`: ```console $ bundle exec ruby -Ilib -rprism -ve 'p Prism.parse("class Foo return end").errors' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] [# @level=:fatal>] After this fix, it will indicate `return`. ```console $ bundle exec ruby -Ilib -rprism -ve 'p Prism.parse("class Foo return end").errors' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] [# @level=:fatal>] ``` For reference, here are the before and after of `Prism::Translation::Parser`. Before: ```console $ bundle exec ruby -Ilib -rprism -rprism/translation/parser33 -ve 'p Prism::Translation::Parser33.parse("class Foo return end")' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] (string):1:18: error: invalid `return` in a class or module body (string):1: class Foo return end (string):1: ^~~ /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/diagnostic/engine.rb:72:in `process': invalid `return` in a class or module body (Parser::SyntaxError) from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:220:in `block in unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `each' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:49:in `parse' from /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/base.rb:33:in `parse' from -e:1:in `
' ``` After: ```console $ bundle exec ruby -Ilib -rprism -rprism/translation/parser33 -ve 'p Prism::Translation::Parser33.parse("class Foo return end")' ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [x86_64-darwin22] (string):1:11: error: invalid `return` in a class or module body (string):1: class Foo return end (string):1: ^~~~~~ /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/diagnostic/engine.rb:72:in `process': invalid `return` in a class or module body (Parser::SyntaxError) from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:220:in `block in unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `each' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:218:in `unwrap' from /Users/koic/src/github.com/ruby/prism/lib/prism/translation/parser.rb:49:in `parse' from /Users/koic/.rbenv/versions/3.3.0/lib/ruby/gems/3.3.0/gems/parser-3.3.0.5/lib/parser/base.rb:33:in `parse' from -e:1:in `
' ``` This PR ensures that the originally intended `return` is highlighted as it should be. https://github.com/ruby/prism/commit/1f9af4d2ad --- prism/prism.c | 2 +- test/prism/errors_test.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 4f65e1e0173e33..368a6c12ab768c 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -16444,7 +16444,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b (parser->current_context->context == PM_CONTEXT_CLASS) || (parser->current_context->context == PM_CONTEXT_MODULE) ) { - pm_parser_err_current(parser, PM_ERR_RETURN_INVALID); + pm_parser_err_previous(parser, PM_ERR_RETURN_INVALID); } return (pm_node_t *) pm_return_node_create(parser, &keyword, arguments.arguments); } diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 9221d52ef35a4c..6e6e74ee5d5a1d 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -1091,7 +1091,7 @@ def test_dont_allow_return_inside_class_body ) assert_errors expected, "class A; return; end", [ - ["invalid `return` in a class or module body", 15..16] + ["invalid `return` in a class or module body", 9..15] ] end @@ -1106,7 +1106,7 @@ def test_dont_allow_return_inside_module_body ) assert_errors expected, "module A; return; end", [ - ["invalid `return` in a class or module body", 16..17] + ["invalid `return` in a class or module body", 10..16] ] end From 0c62eb25b5c7a2ce6764e81bf98ac39a804a3721 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:40:34 -0400 Subject: [PATCH 33/58] [PRISM] Use correct encoding for regular expression literals --- .github/workflows/prism.yml | 2 +- prism_compile.c | 423 ++++++++++++---------- test/.excludes-prism/TestRegexp.rb | 7 + test/.excludes-prism/TestUnicodeEscape.rb | 1 - 4 files changed, 233 insertions(+), 200 deletions(-) create mode 100644 test/.excludes-prism/TestRegexp.rb diff --git a/.github/workflows/prism.yml b/.github/workflows/prism.yml index 49058c232f616a..8b295f7f815349 100644 --- a/.github/workflows/prism.yml +++ b/.github/workflows/prism.yml @@ -92,7 +92,7 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="test_regexp.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' + RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-prism-spec diff --git a/prism_compile.c b/prism_compile.c index db032e2a0530a5..d7eada1538ad38 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -289,131 +289,243 @@ pm_optimizable_range_item_p(pm_node_t *node) return (!node || PM_NODE_TYPE_P(node, PM_INTEGER_NODE) || PM_NODE_TYPE_P(node, PM_NIL_NODE)); } +static void pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node); + +static int +pm_interpolated_node_compile(const pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + int number_of_items_pushed = 0; + size_t parts_size = parts->size; + + if (parts_size > 0) { + VALUE current_string = Qnil; + + for (size_t index = 0; index < parts_size; index++) { + const pm_node_t *part = parts->nodes[index]; + + if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { + const pm_string_node_t *string_node = (const pm_string_node_t *)part; + VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); + + if (RTEST(current_string)) { + current_string = rb_str_concat(current_string, string_value); + } + else { + current_string = string_value; + } + } + else if (PM_NODE_TYPE_P(part, PM_EMBEDDED_STATEMENTS_NODE) && + ((const pm_embedded_statements_node_t *) part)->statements != NULL && + ((const pm_embedded_statements_node_t *) part)->statements->body.size == 1 && + PM_NODE_TYPE_P(((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0], PM_STRING_NODE)) { + const pm_string_node_t *string_node = (const pm_string_node_t *) ((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0]; + VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); + + if (RTEST(current_string)) { + current_string = rb_str_concat(current_string, string_value); + } + else { + current_string = string_value; + } + } + else { + if (!RTEST(current_string)) { + current_string = rb_enc_str_new(NULL, 0, scope_node->encoding); + } + + ADD_INSN1(ret, &dummy_line_node, putobject, rb_fstring(current_string)); + + current_string = Qnil; + number_of_items_pushed++; + + PM_COMPILE_NOT_POPPED(part); + PM_DUP; + ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); + ADD_INSN(ret, &dummy_line_node, anytostring); + + number_of_items_pushed++; + } + } + + if (RTEST(current_string)) { + current_string = rb_fstring(current_string); + ADD_INSN1(ret, &dummy_line_node, putobject, current_string); + current_string = Qnil; + number_of_items_pushed++; + } + } + else { + PM_PUTNIL; + } + + return number_of_items_pushed; +} + +static VALUE +pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +{ + VALUE current = Qnil; + + for (size_t index = 0; index < nodes->size; index++) { + const pm_node_t *part = nodes->nodes[index]; + VALUE string; + + switch (PM_NODE_TYPE(part)) { + case PM_STRING_NODE: + string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); + break; + case PM_INTERPOLATED_STRING_NODE: + string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); + break; + default: + RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); + return Qnil; + } + + if (current != Qnil) { + current = rb_str_concat(current, string); + } + else { + current = string; + } + } + + return top ? rb_fstring(current) : current; +} + #define RE_OPTION_ENCODING_SHIFT 8 +#define RE_OPTION_ENCODING(encoding) (((encoding) & 0xFF) << RE_OPTION_ENCODING_SHIFT) +#define ARG_ENCODING_NONE 32 +#define ARG_ENCODING_FIXED 16 +#define ENC_ASCII8BIT 1 +#define ENC_EUC_JP 2 +#define ENC_Windows_31J 3 +#define ENC_UTF8 4 /** * Check the prism flags of a regular expression-like node and return the flags * that are expected by the CRuby VM. */ static int -pm_reg_flags(const pm_node_t *node) { +parse_regexp_flags(const pm_node_t *node) +{ int flags = 0; - int dummy = 0; // Check "no encoding" first so that flags don't get clobbered // We're calling `rb_char_to_option_kcode` in this case so that // we don't need to have access to `ARG_ENCODING_NONE` - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) { - rb_char_to_option_kcode('n', &flags, &dummy); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT)) { + flags |= ARG_ENCODING_NONE; } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) { - rb_char_to_option_kcode('e', &flags, &dummy); - flags |= ('e' << RE_OPTION_ENCODING_SHIFT); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { + flags |= (ARG_ENCODING_FIXED | RE_OPTION_ENCODING(ENC_EUC_JP)); } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) { - rb_char_to_option_kcode('s', &flags, &dummy); - flags |= ('s' << RE_OPTION_ENCODING_SHIFT); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { + flags |= (ARG_ENCODING_FIXED | RE_OPTION_ENCODING(ENC_Windows_31J)); } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) { - rb_char_to_option_kcode('u', &flags, &dummy); - flags |= ('u' << RE_OPTION_ENCODING_SHIFT); + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { + flags |= (ARG_ENCODING_FIXED | RE_OPTION_ENCODING(ENC_UTF8)); } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE)) { flags |= ONIG_OPTION_IGNORECASE; } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE)) { flags |= ONIG_OPTION_MULTILINE; } - if (node->flags & PM_REGULAR_EXPRESSION_FLAGS_EXTENDED) { + if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EXTENDED)) { flags |= ONIG_OPTION_EXTEND; } return flags; } +#undef RE_OPTION_ENCODING_SHIFT +#undef RE_OPTION_ENCODING +#undef ARG_ENCODING_FIXED +#undef ARG_ENCODING_NONE +#undef ENC_ASCII8BIT +#undef ENC_EUC_JP +#undef ENC_Windows_31J +#undef ENC_UTF8 + static rb_encoding * -pm_reg_enc(const pm_scope_node_t *scope_node, const pm_node_t *node) +parse_regexp_encoding(const pm_scope_node_t *scope_node, const pm_node_t *node) { if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT)) { return rb_ascii8bit_encoding(); } - - if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { + else if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { + return rb_utf8_encoding(); + } + else if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EUC_JP)) { return rb_enc_get_from_index(ENCINDEX_EUC_JP); } - - if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { + else if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J)) { return rb_enc_get_from_index(ENCINDEX_Windows_31J); } - - if (PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_UTF_8)) { - return rb_utf8_encoding(); + else { + return scope_node->encoding; } - - return scope_node->encoding; } -/** - * Certain nodes can be compiled literally, which can lead to further - * optimizations. These nodes will all have the PM_NODE_FLAG_STATIC_LITERAL flag - * set. - */ -static inline bool -pm_static_literal_p(const pm_node_t *node) +/** Raise an error corresponding to the invalid regular expression. */ +static VALUE +parse_regexp_error(rb_iseq_t *iseq, int32_t line_number, const char *fmt, ...) { - return node->flags & PM_NODE_FLAG_STATIC_LITERAL; + va_list args; + va_start(args, fmt); + VALUE error = rb_syntax_error_append(Qnil, rb_iseq_path(iseq), line_number, -1, NULL, "%" PRIsVALUE, args); + va_end(args); + rb_exc_raise(error); } static VALUE -pm_new_regex(const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) +parse_regexp(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, VALUE string) { - VALUE regex_str = parse_string(scope_node, unescaped); - rb_encoding *enc = pm_reg_enc(scope_node, node); + VALUE errinfo = rb_errinfo(); - VALUE regex = rb_enc_reg_new(RSTRING_PTR(regex_str), RSTRING_LEN(regex_str), enc, pm_reg_flags(node)); - RB_GC_GUARD(regex_str); + int32_t line_number = pm_node_line_number(scope_node->parser, node); + VALUE regexp = rb_reg_compile(string, parse_regexp_flags(node), (const char *) pm_string_source(&scope_node->parser->filepath), line_number); - rb_obj_freeze(regex); + if (NIL_P(regexp)) { + VALUE message = rb_attr_get(rb_errinfo(), idMesg); + rb_set_errinfo(errinfo); - return regex; + parse_regexp_error(iseq, line_number, "%" PRIsVALUE, message); + return Qnil; + } + + rb_obj_freeze(regexp); + return regexp; } -static VALUE -pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *scope_node, bool top) +static inline VALUE +parse_regexp_literal(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_string_t *unescaped) { - VALUE current = Qnil; - - for (size_t index = 0; index < nodes->size; index++) { - const pm_node_t *part = nodes->nodes[index]; - VALUE string; - - switch (PM_NODE_TYPE(part)) { - case PM_STRING_NODE: - string = parse_string_encoded(scope_node, part, &((const pm_string_node_t *) part)->unescaped); - break; - case PM_INTERPOLATED_STRING_NODE: - string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) part)->parts, scope_node, false); - break; - default: - RUBY_ASSERT(false && "unexpected node type in pm_static_literal_concat"); - return Qnil; - } + VALUE string = rb_enc_str_new((const char *) pm_string_source(unescaped), pm_string_length(unescaped), parse_regexp_encoding(scope_node, node)); + return parse_regexp(iseq, scope_node, node, string); +} - if (current != Qnil) { - current = rb_str_concat(current, string); - } - else { - current = string; - } - } +static inline VALUE +parse_regexp_concat(rb_iseq_t *iseq, const pm_scope_node_t *scope_node, const pm_node_t *node, const pm_node_list_t *parts) +{ + VALUE string = pm_static_literal_concat(parts, scope_node, false); + rb_enc_associate(string, parse_regexp_encoding(scope_node, node)); + return parse_regexp(iseq, scope_node, node, string); +} - return top ? rb_fstring(current) : current; +static void +pm_compile_regexp_dynamic(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_list_t *parts, const pm_line_column_t *node_location, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) +{ + NODE dummy_line_node = generate_dummy_line_node(node_location->line, node_location->column); + int length = pm_interpolated_node_compile(parts, iseq, dummy_line_node, ret, popped, scope_node); + PUSH_INSN2(ret, *node_location, toregexp, INT2FIX(parse_regexp_flags(node) & 0xFF), INT2FIX(length)); } /** @@ -422,11 +534,11 @@ pm_static_literal_concat(const pm_node_list_t *nodes, const pm_scope_node_t *sco * literal values can be compiled into a literal array. */ static VALUE -pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node) +pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_node_t *scope_node) { // Every node that comes into this function should already be marked as // static literal. If it's not, then we have a bug somewhere. - RUBY_ASSERT(pm_static_literal_p(node)); + RUBY_ASSERT(PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)); switch (PM_NODE_TYPE(node)) { case PM_ARRAY_NODE: { @@ -435,7 +547,7 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node VALUE value = rb_ary_hidden_new(elements->size); for (size_t index = 0; index < elements->size; index++) { - rb_ary_push(value, pm_static_literal_value(elements->nodes[index], scope_node)); + rb_ary_push(value, pm_static_literal_value(iseq, elements->nodes[index], scope_node)); } OBJ_FREEZE(value); @@ -453,7 +565,7 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node for (size_t index = 0; index < elements->size; index++) { RUBY_ASSERT(PM_NODE_TYPE_P(elements->nodes[index], PM_ASSOC_NODE)); pm_assoc_node_t *cast = (pm_assoc_node_t *) elements->nodes[index]; - VALUE pair[2] = { pm_static_literal_value(cast->key, scope_node), pm_static_literal_value(cast->value, scope_node) }; + VALUE pair[2] = { pm_static_literal_value(iseq, cast->key, scope_node), pm_static_literal_value(iseq, cast->value, scope_node) }; rb_ary_cat(array, pair, 2); } @@ -470,17 +582,11 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node return parse_integer((const pm_integer_node_t *) node); case PM_INTERPOLATED_MATCH_LAST_LINE_NODE: { const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; - VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); - - rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); - return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; - VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); - - rb_encoding *encoding = pm_reg_enc(scope_node, (const pm_node_t *) cast); - return rb_obj_freeze(rb_enc_reg_new(RSTRING_PTR(string), RSTRING_LEN(string), encoding, pm_reg_flags((const pm_node_t *) cast))); + return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } case PM_INTERPOLATED_STRING_NODE: return pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, true); @@ -492,7 +598,7 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node } case PM_MATCH_LAST_LINE_NODE: { const pm_match_last_line_node_t *cast = (const pm_match_last_line_node_t *) node; - return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + return parse_regexp_literal(iseq, scope_node, (const pm_node_t *) cast, &cast->unescaped); } case PM_NIL_NODE: return Qnil; @@ -500,13 +606,20 @@ pm_static_literal_value(const pm_node_t *node, const pm_scope_node_t *scope_node return parse_rational((const pm_rational_node_t *) node); case PM_REGULAR_EXPRESSION_NODE: { const pm_regular_expression_node_t *cast = (const pm_regular_expression_node_t *) node; - return pm_new_regex(scope_node, (const pm_node_t *) cast, &cast->unescaped); + return parse_regexp_literal(iseq, scope_node, (const pm_node_t *) cast, &cast->unescaped); } case PM_SOURCE_ENCODING_NODE: return rb_enc_from_encoding(scope_node->encoding); case PM_SOURCE_FILE_NODE: { - pm_source_file_node_t *cast = (pm_source_file_node_t *)node; - return cast->filepath.length ? parse_string(scope_node, &cast->filepath) : rb_fstring_lit(""); + const pm_source_file_node_t *cast = (const pm_source_file_node_t *) node; + size_t length = pm_string_length(&cast->filepath); + + if (length > 0) { + return rb_enc_str_new((const char *) pm_string_source(&cast->filepath), length, scope_node->encoding); + } + else { + return rb_fstring_lit(""); + } } case PM_SOURCE_LINE_NODE: return INT2FIX(pm_node_line_number(scope_node->parser, node)); @@ -601,8 +714,6 @@ pm_compile_logical(rb_iseq_t *iseq, LINK_ANCHOR *const ret, pm_node_t *cond, LAB return; } -static void pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node); - static void pm_compile_flip_flop_bound(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) { @@ -842,76 +953,6 @@ pm_compile_loop(rb_iseq_t *iseq, const pm_line_column_t *line_column, pm_node_fl return; } -static int -pm_interpolated_node_compile(const pm_node_list_t *parts, rb_iseq_t *iseq, NODE dummy_line_node, LINK_ANCHOR *const ret, bool popped, pm_scope_node_t *scope_node) -{ - int number_of_items_pushed = 0; - size_t parts_size = parts->size; - - if (parts_size > 0) { - VALUE current_string = Qnil; - - for (size_t index = 0; index < parts_size; index++) { - const pm_node_t *part = parts->nodes[index]; - - if (PM_NODE_TYPE_P(part, PM_STRING_NODE)) { - const pm_string_node_t *string_node = (const pm_string_node_t *)part; - VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - else if (PM_NODE_TYPE_P(part, PM_EMBEDDED_STATEMENTS_NODE) && - ((const pm_embedded_statements_node_t *) part)->statements != NULL && - ((const pm_embedded_statements_node_t *) part)->statements->body.size == 1 && - PM_NODE_TYPE_P(((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0], PM_STRING_NODE)) { - const pm_string_node_t *string_node = (const pm_string_node_t *) ((const pm_embedded_statements_node_t *) part)->statements->body.nodes[0]; - VALUE string_value = parse_string_encoded(scope_node, (pm_node_t *)string_node, &string_node->unescaped); - - if (RTEST(current_string)) { - current_string = rb_str_concat(current_string, string_value); - } - else { - current_string = string_value; - } - } - else { - if (!RTEST(current_string)) { - current_string = rb_enc_str_new(NULL, 0, scope_node->encoding); - } - - ADD_INSN1(ret, &dummy_line_node, putobject, rb_fstring(current_string)); - - current_string = Qnil; - number_of_items_pushed++; - - PM_COMPILE_NOT_POPPED(part); - PM_DUP; - ADD_INSN1(ret, &dummy_line_node, objtostring, new_callinfo(iseq, idTo_s, 0, VM_CALL_FCALL | VM_CALL_ARGS_SIMPLE , NULL, FALSE)); - ADD_INSN(ret, &dummy_line_node, anytostring); - - number_of_items_pushed++; - } - } - - if (RTEST(current_string)) { - current_string = rb_fstring(current_string); - ADD_INSN1(ret, &dummy_line_node, putobject, current_string); - current_string = Qnil; - number_of_items_pushed++; - } - } - else { - PM_PUTNIL; - } - - return number_of_items_pushed; -} - // This recurses through scopes and finds the local index at any scope level // It also takes a pointer to depth, and increments depth appropriately // according to the depth of the local. @@ -1201,7 +1242,7 @@ pm_setup_args_core(const pm_arguments_node_t *arguments_node, const pm_node_t *b // Retrieve the stored index from the hash for this // keyword. - VALUE keyword = pm_static_literal_value(assoc->key, scope_node); + VALUE keyword = pm_static_literal_value(iseq, assoc->key, scope_node); VALUE stored_index = rb_hash_aref(stored_indices, keyword); // If this keyword was already seen in the hash, @@ -1233,7 +1274,7 @@ pm_setup_args_core(const pm_arguments_node_t *arguments_node, const pm_node_t *b bool popped = true; if (rb_ary_entry(keyword_indices, (long) element_index) == Qtrue) { - keywords[keyword_index++] = pm_static_literal_value(assoc->key, scope_node); + keywords[keyword_index++] = pm_static_literal_value(iseq, assoc->key, scope_node); popped = false; } @@ -4232,13 +4273,13 @@ pm_compile_constant_path(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *co * optimization entirely. */ static VALUE -pm_compile_case_node_dispatch(VALUE dispatch, const pm_node_t *node, LABEL *label, const pm_scope_node_t *scope_node) +pm_compile_case_node_dispatch(rb_iseq_t *iseq, VALUE dispatch, const pm_node_t *node, LABEL *label, const pm_scope_node_t *scope_node) { VALUE key = Qundef; switch (PM_NODE_TYPE(node)) { case PM_FLOAT_NODE: { - key = pm_static_literal_value(node, scope_node); + key = pm_static_literal_value(iseq, node, scope_node); double intptr; if (modf(RFLOAT_VALUE(key), &intptr) == 0.0) { @@ -4254,7 +4295,7 @@ pm_compile_case_node_dispatch(VALUE dispatch, const pm_node_t *node, LABEL *labe case PM_SOURCE_LINE_NODE: case PM_SYMBOL_NODE: case PM_TRUE_NODE: - key = pm_static_literal_value(node, scope_node); + key = pm_static_literal_value(iseq, node, scope_node); break; case PM_STRING_NODE: { const pm_string_node_t *cast = (const pm_string_node_t *) node; @@ -4379,13 +4420,13 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // If every node in the array is static, then we can compile the entire // array now instead of later. - if (pm_static_literal_p(node)) { + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { // We're only going to compile this node if it's not popped. If it // is popped, then we know we don't need to do anything since it's // statically known. if (!popped) { if (elements->size) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, duparray, value); } else { @@ -4868,7 +4909,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // we're going to try to compile the condition into the // dispatch hash. if (dispatch != Qundef) { - dispatch = pm_compile_case_node_dispatch(dispatch, condition, label, scope_node); + dispatch = pm_compile_case_node_dispatch(iseq, dispatch, condition, label, scope_node); } if (PM_NODE_TYPE_P(condition, PM_SPLAT_NODE)) { @@ -5886,12 +5927,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_HASH_NODE: { // If every node in the hash is static, then we can compile the entire // hash now instead of later. - if (pm_static_literal_p(node)) { + if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { // We're only going to compile this node if it's not popped. If it // is popped, then we know we don't need to do anything since it's // statically known. if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); ADD_INSN1(ret, &dummy_line_node, duphash, value); RB_OBJ_WRITTEN(iseq, Qundef, value); } @@ -6072,14 +6113,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^^^^^ if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE regexp = pm_static_literal_value(node, scope_node); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, regexp); } } else { - const pm_interpolated_match_last_line_node_t *cast = (const pm_interpolated_match_last_line_node_t *) node; - int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + pm_compile_regexp_dynamic(iseq, node, &((const pm_interpolated_match_last_line_node_t *) node)->parts, &location, ret, popped, scope_node); } PUSH_INSN1(ret, location, getglobal, rb_id2sym(idLASTLINE)); @@ -6106,20 +6145,18 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PUSH_INSN2(ret, location, once, block_iseq, INT2FIX(ise_index)); ISEQ_COMPILE_DATA(iseq)->current_block = prevblock; + if (popped) PUSH_INSN(ret, location, pop); return; } if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE regexp = pm_static_literal_value(node, scope_node); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, regexp); } } else { - const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; - int length = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - - PUSH_INSN2(ret, location, toregexp, INT2FIX(pm_reg_flags((const pm_node_t *) cast)), INT2FIX(length)); + pm_compile_regexp_dynamic(iseq, node, &((const pm_interpolated_regular_expression_node_t *) node)->parts, &location, ret, popped, scope_node); if (popped) PUSH_INSN(ret, location, pop); } @@ -6130,7 +6167,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^^^^^ if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE string = pm_static_literal_value(node, scope_node); + VALUE string = pm_static_literal_value(iseq, node, scope_node); if (PM_NODE_FLAG_P(node, PM_INTERPOLATED_STRING_NODE_FLAGS_FROZEN)) { PUSH_INSN1(ret, location, putobject, string); @@ -6161,7 +6198,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) { if (!popped) { - VALUE symbol = pm_static_literal_value(node, scope_node); + VALUE symbol = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, symbol); } } @@ -6323,7 +6360,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, case PM_MATCH_LAST_LINE_NODE: { // if /foo/ then end // ^^^^^ - VALUE regexp = pm_static_literal_value(node, scope_node); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, regexp); PUSH_INSN2(ret, location, getspecial, INT2FIX(0), INT2FIX(0)); @@ -6895,8 +6932,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // /foo/ // ^^^^^ if (!popped) { - VALUE regex = pm_static_literal_value(node, scope_node); - PUSH_INSN1(ret, location, putobject, regex); + VALUE regexp = pm_static_literal_value(iseq, node, scope_node); + PUSH_INSN1(ret, location, putobject, regexp); } return; } @@ -7500,12 +7537,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_node_t *value = cast->value; name = cast->name; - if (pm_static_literal_p(value) && - !(PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || - PM_NODE_TYPE_P(value, PM_HASH_NODE) || - PM_NODE_TYPE_P(value, PM_RANGE_NODE))) { - - rb_ary_push(default_values, pm_static_literal_value(value, scope_node)); + if (PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) && !(PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE))) { + rb_ary_push(default_values, pm_static_literal_value(iseq, value, scope_node)); } else { rb_ary_push(default_values, complex_mark); @@ -7814,10 +7847,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_node_t *value = cast->value; name = cast->name; - if (!(pm_static_literal_p(value)) || - PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || - PM_NODE_TYPE_P(value, PM_HASH_NODE) || - PM_NODE_TYPE_P(value, PM_RANGE_NODE)) { + if (!PM_NODE_FLAG_P(value, PM_NODE_FLAG_STATIC_LITERAL) || PM_NODE_TYPE_P(value, PM_ARRAY_NODE) || PM_NODE_TYPE_P(value, PM_HASH_NODE) || PM_NODE_TYPE_P(value, PM_RANGE_NODE)) { LABEL *end_label = NEW_LABEL(nd_line(&dummy_line_node)); pm_local_index_t index = pm_lookup_local_index(iseq, scope_node, name, 0); @@ -7910,11 +7940,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } case PM_INTERPOLATED_REGULAR_EXPRESSION_NODE: { - pm_interpolated_regular_expression_node_t *cast = (pm_interpolated_regular_expression_node_t *) scope_node->ast_node; - - int parts_size = pm_interpolated_node_compile(&cast->parts, iseq, dummy_line_node, ret, popped, scope_node); - - ADD_INSN2(ret, &dummy_line_node, toregexp, INT2FIX(pm_reg_flags((pm_node_t *)cast)), INT2FIX(parts_size)); + const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) scope_node->ast_node; + pm_compile_regexp_dynamic(iseq, (const pm_node_t *) cast, &cast->parts, &location, ret, popped, scope_node); break; } default: @@ -8037,7 +8064,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // __ENCODING__ // ^^^^^^^^^^^^ if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, value); } return; @@ -8065,7 +8092,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // __LINE__ // ^^^^^^^^ if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, value); } return; @@ -8156,7 +8183,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // :foo // ^^^^ if (!popped) { - VALUE value = pm_static_literal_value(node, scope_node); + VALUE value = pm_static_literal_value(iseq, node, scope_node); PUSH_INSN1(ret, location, putobject, value); } return; diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb new file mode 100644 index 00000000000000..f2b817d79a7266 --- /dev/null +++ b/test/.excludes-prism/TestRegexp.rb @@ -0,0 +1,7 @@ +exclude(:test_unicode_age_14_0, "unknown") +exclude(:test_invalid_fragment, "unknown") +exclude(:test_assign_named_capture_to_reserved_word, "unknown") +exclude(:test_unicode_age_15_0, "unknown") +exclude(:test_unescape, "unknown") +exclude(:test_invalid_escape_error, "unknown") +exclude(:test_unicode_age, "unknown") diff --git a/test/.excludes-prism/TestUnicodeEscape.rb b/test/.excludes-prism/TestUnicodeEscape.rb index 93ed9fcb4514a5..add4911bc2f0aa 100644 --- a/test/.excludes-prism/TestUnicodeEscape.rb +++ b/test/.excludes-prism/TestUnicodeEscape.rb @@ -1,2 +1 @@ exclude(:test_fail, "unknown") -exclude(:test_regexp, "unknown") From a7ff264477105b5dc0ade6facad4176a1b73df0b Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Wed, 27 Mar 2024 10:10:07 +1300 Subject: [PATCH 34/58] Don't clear pending interrupts in the parent process. (#10365) --- process.c | 1 - test/ruby/test_process.rb | 21 +++++++++++++++++++++ thread.c | 1 + 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/process.c b/process.c index e5415dd170a362..8d08da76505635 100644 --- a/process.c +++ b/process.c @@ -1682,7 +1682,6 @@ before_fork_ruby(void) static void after_fork_ruby(rb_pid_t pid) { - rb_threadptr_pending_interrupt_clear(GET_THREAD()); if (pid == 0) { // child clear_pid_cache(); diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 8982ab8b9a6a2d..59140ba664541c 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -2828,4 +2828,25 @@ def test_concurrent_group_and_pid_wait [t1, t2, t3].each { _1&.join rescue nil } [long_rpipe, long_wpipe, short_rpipe, short_wpipe].each { _1&.close rescue nil } end if defined?(fork) + + def test_handle_interrupt_with_fork + Thread.handle_interrupt(RuntimeError => :never) do + Thread.current.raise(RuntimeError, "Queued error") + + assert_predicate Thread, :pending_interrupt? + + pid = Process.fork do + if Thread.pending_interrupt? + exit 1 + end + end + + _, status = Process.waitpid2(pid) + assert_predicate status, :success? + + assert_predicate Thread, :pending_interrupt? + end + rescue RuntimeError + # Ignore. + end if defined?(fork) end diff --git a/thread.c b/thread.c index 1859f727ea5c2f..196f9092b49ebb 100644 --- a/thread.c +++ b/thread.c @@ -4725,6 +4725,7 @@ void rb_thread_atfork(void) { rb_thread_t *th = GET_THREAD(); + rb_threadptr_pending_interrupt_clear(th); rb_thread_atfork_internal(th, terminate_atfork_i); th->join_list = NULL; rb_fiber_atfork(th); From b2b665eba59e3fc9ad9656d9c74509a975db6fe8 Mon Sep 17 00:00:00 2001 From: crazeteam Date: Wed, 27 Mar 2024 00:15:40 +0800 Subject: [PATCH 35/58] [DOC] remove repetitive words in comments Signed-off-by: crazeteam --- doc/syntax/calling_methods.rdoc | 2 +- include/ruby/internal/intern/string.h | 2 +- spec/ruby/optional/capi/util_spec.rb | 2 +- test/ruby/test_string.rb | 2 +- warning.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/syntax/calling_methods.rdoc b/doc/syntax/calling_methods.rdoc index 6cc8678450582f..c2c6c61a1039f4 100644 --- a/doc/syntax/calling_methods.rdoc +++ b/doc/syntax/calling_methods.rdoc @@ -210,7 +210,7 @@ definition. If a keyword argument is given that the method did not list, and the method definition does not accept arbitrary keyword arguments, an ArgumentError will be raised. -Keyword argument value can be omitted, meaning the value will be be fetched +Keyword argument value can be omitted, meaning the value will be fetched from the context by the name of the key keyword1 = 'some value' diff --git a/include/ruby/internal/intern/string.h b/include/ruby/internal/intern/string.h index cfe0454ee88562..6827563e8dc3e0 100644 --- a/include/ruby/internal/intern/string.h +++ b/include/ruby/internal/intern/string.h @@ -454,7 +454,7 @@ VALUE rb_interned_str(const char *ptr, long len); RBIMPL_ATTR_NONNULL(()) /** * Identical to rb_interned_str(), except it assumes the passed pointer is a - * pointer to a C's string. It can also be seen as a routine identical to to + * pointer to a C's string. It can also be seen as a routine identical to * rb_str_to_interned_str(), except it takes a C's string instead of Ruby's. * Or it can also be seen as a routine identical to rb_str_new_cstr(), except * it returns an infamous "f"string. diff --git a/spec/ruby/optional/capi/util_spec.rb b/spec/ruby/optional/capi/util_spec.rb index 2c16999cdc3589..9ff8b4760a83f7 100644 --- a/spec/ruby/optional/capi/util_spec.rb +++ b/spec/ruby/optional/capi/util_spec.rb @@ -48,7 +48,7 @@ ScratchPad.recorded.should == [1, 2, [3, 4]] end - it "assigns the required and optional arguments and and empty Array when there are no arguments to splat" do + it "assigns the required and optional arguments and empty Array when there are no arguments to splat" do @o.rb_scan_args([1, 2], "11*", 3, @acc).should == 2 ScratchPad.recorded.should == [1, 2, []] end diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 8ec80d06fc7f2f..0dae1c5a8e49ea 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3623,7 +3623,7 @@ def test_chilled_string assert_not_predicate +chilled_string, :frozen? assert_not_same chilled_string, +chilled_string - # @- the the original string as mutable + # @- the original string as mutable assert_predicate -chilled_string, :frozen? assert_not_same chilled_string, -chilled_string end diff --git a/warning.rb b/warning.rb index 4e34c6383393e5..aab5e7c2c6965d 100644 --- a/warning.rb +++ b/warning.rb @@ -40,7 +40,7 @@ module Kernel # baz.rb:6: warning: invalid call to foo # # If category keyword argument is given, passes the category - # to Warning.warn. The category given must be be one of the + # to Warning.warn. The category given must be one of the # following categories: # # :deprecated :: Used for warning for deprecated functionality that may From 6e34386794db69949b13f055fa338431527910eb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 21 Mar 2024 12:17:15 -0400 Subject: [PATCH 36/58] [flori/json] Fix memory leak when exception is raised during JSON generation If an exception is raised the FBuffer is leaked. For example, the following script leaks memory: o = Object.new def o.to_json(a) = raise 10.times do 100_000.times do begin JSON(o) rescue end end puts `ps -o rss= -p #{$$}` end Before: 31824 35696 40240 44304 47424 50944 54000 58384 62416 65296 After: 24416 24640 24640 24736 24736 24736 24736 24736 24736 24736 https://github.com/flori/json/commit/44df509dc2 --- ext/json/generator/generator.c | 40 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/ext/json/generator/generator.c b/ext/json/generator/generator.c index a33df848df638c..6d78284bc444cc 100644 --- a/ext/json/generator/generator.c +++ b/ext/json/generator/generator.c @@ -892,7 +892,6 @@ static void generate_json_object(FBuffer *buffer, VALUE Vstate, JSON_Generator_S struct hash_foreach_arg arg; if (max_nesting != 0 && depth > max_nesting) { - fbuffer_free(buffer); rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth); } fbuffer_append_char(buffer, '{'); @@ -927,7 +926,6 @@ static void generate_json_array(FBuffer *buffer, VALUE Vstate, JSON_Generator_St long depth = ++state->depth; int i, j; if (max_nesting != 0 && depth > max_nesting) { - fbuffer_free(buffer); rb_raise(eNestingError, "nesting of %ld is too deep", --state->depth); } fbuffer_append_char(buffer, '['); @@ -1020,10 +1018,8 @@ static void generate_json_float(FBuffer *buffer, VALUE Vstate, JSON_Generator_St VALUE tmp = rb_funcall(obj, i_to_s, 0); if (!allow_nan) { if (isinf(value)) { - fbuffer_free(buffer); rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(tmp)); } else if (isnan(value)) { - fbuffer_free(buffer); rb_raise(eGeneratorError, "%"PRIsVALUE" not allowed in JSON", RB_OBJ_STRING(tmp)); } } @@ -1096,11 +1092,45 @@ static FBuffer *cState_prepare_buffer(VALUE self) return buffer; } +struct generate_json_data { + FBuffer *buffer; + VALUE vstate; + JSON_Generator_State *state; + VALUE obj; +}; + +static VALUE generate_json_try(VALUE d) +{ + struct generate_json_data *data = (struct generate_json_data *)d; + + generate_json(data->buffer, data->vstate, data->state, data->obj); + + return Qnil; +} + +static VALUE generate_json_rescue(VALUE d, VALUE exc) +{ + struct generate_json_data *data = (struct generate_json_data *)d; + fbuffer_free(data->buffer); + + rb_exc_raise(exc); + + return Qundef; +} + static VALUE cState_partial_generate(VALUE self, VALUE obj) { FBuffer *buffer = cState_prepare_buffer(self); GET_STATE(self); - generate_json(buffer, self, state, obj); + + struct generate_json_data data = { + .buffer = buffer, + .vstate = self, + .state = state, + .obj = obj + }; + rb_rescue(generate_json_try, (VALUE)&data, generate_json_rescue, (VALUE)&data); + return fbuffer_to_s(buffer); } From 8896ac0289dcd7a6c9c4a7fd6ccd4cc2dae30507 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Tue, 12 Mar 2024 17:12:08 +0100 Subject: [PATCH 37/58] [ruby/openssl] Fix test_pkey_dsa.rb in FIPS. Note that I created the `dsa2048.pem` and signature text (`signature_encoded.txt`), that is used as a text to create the `signature0` in the `test_sign_verify` by the following steps with the `openssl` CLI on FIPS module. ``` $ OPENSSL_DIR="${HOME}/.local/openssl-3.3.0-dev-fips-debug-1f03d33ef5" $ export OPENSSL_CONF="${OPENSSL_DIR}/ssl/openssl_fips.cnf" $ "${OPENSSL_DIR}/bin/openssl" dsaparam -out dsaparam2048.pem 2048 $ "${OPENSSL_DIR}/bin/openssl" gendsa -out dsa2048.pem dsaparam2048.pem $ echo -n "Sign me!" > data.txt $ "${OPENSSL_DIR}/bin/openssl" dgst -sha256 -sign dsa2048.pem data.txt > signature.txt $ cat signature.txt | base64 > signature_encoded.txt ``` Skip the `test_DSAPrivateKey_encrypted` on FIPS because AES-128-CBC, the password based encryption used in the PEM format uses MD5 for deriving the encryption key from the password, and MD5 is not FIPS-approved. See also the comment on the `test/openssl/utils.rb#omit_on_fips`. https://github.com/ruby/openssl/commit/4bdcb419a9 --- test/openssl/fixtures/pkey/dsa2048.pem | 15 ++++++++++ test/openssl/test_pkey_dsa.rb | 40 ++++++++++++++++---------- 2 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 test/openssl/fixtures/pkey/dsa2048.pem diff --git a/test/openssl/fixtures/pkey/dsa2048.pem b/test/openssl/fixtures/pkey/dsa2048.pem new file mode 100644 index 00000000000000..3f22b22b58bd0a --- /dev/null +++ b/test/openssl/fixtures/pkey/dsa2048.pem @@ -0,0 +1,15 @@ +-----BEGIN PRIVATE KEY----- +MIICXgIBADCCAjYGByqGSM44BAEwggIpAoIBAQDXZhJ/dQoWkQELzjzlx8FtIp96 +voCYe5NY0H8j0jz7GyHpXt41+MteqkZK3/Ah+cNR9uG8iEYArAZ71LcWotfee2Gz +xdxozr9bRt0POYhO2YIsfMpBrEskPsDH2g/2nFV8l4OJgxU2qZUrF4PN5ha+Mu6u +sVtN8hjvAvnbf4Pxn0b8NN9f4PJncroUa8acv5WsV85E1RW7NYCefggU4LytYIHg +euRF9eY9gVCX5MkUgW2xODHIYJhwk/+5lJxG7qUsSahD/nPHO/yoWgdVHq2DkdTq +KYXkAxx2PJcTBOHTglhE6mgCbEKp8vcfElnBWyCT6QykclZiPXXD2JV829J/Ah0A +vYa+/G/gUZiomyejVje6UsGoCc+vInxmovOL8QKCAQEAhnKEigYPw6u8JY7v5iGo +Ylz8qiMFYmaJCwevf3KCjWeEXuNO4OrKdfzkQl1tPuGLioYFfP1A2yGosjdUdLEB +0JqnzlKxUp+G6RfBj+WYzbgc5hr7t0M+reAJh09/hDzqfxjcgiHstq7mpRXBP8Y7 +iu27s7TRYJNSAYRvWcXNSBEUym3mHBBbZn7VszYooSrn60/iZ8I+VY1UF/fgqhbj +JfaaZNQCDO9K3Vb3rsXoYd8+bOZIen9uHB+pNjMqhpl4waysqrlpGFeeqdxivH6S +vkrHLs6/eWVMnS08RdcryoCrI3Bm8mMBKQglDwKLnWLfzG565qEhslzyCd/l9k9a +cwQfAh0Ao8/g72fSFmo04FizM7DZJSIPqDLjfZu9hLvUFA== +-----END PRIVATE KEY----- diff --git a/test/openssl/test_pkey_dsa.rb b/test/openssl/test_pkey_dsa.rb index 3f64a80e324814..4c93f2869d8040 100644 --- a/test/openssl/test_pkey_dsa.rb +++ b/test/openssl/test_pkey_dsa.rb @@ -31,11 +31,6 @@ def test_new_break def test_generate # DSA.generate used to call DSA_generate_parameters_ex(), which adjusts the # size of q according to the size of p - key1024 = OpenSSL::PKey::DSA.generate(1024) - assert_predicate key1024, :private? - assert_equal 1024, key1024.p.num_bits - assert_equal 160, key1024.q.num_bits - key2048 = OpenSSL::PKey::DSA.generate(2048) assert_equal 2048, key2048.p.num_bits assert_equal 256, key2048.q.num_bits @@ -47,28 +42,41 @@ def test_generate end end + def test_generate_on_non_fips + # DSA with 1024 bits is invalid on FIPS 186-4. + # https://github.com/openssl/openssl/commit/49ed5ba8f62875074f04417189147fd3dda072ab + omit_on_fips + + key1024 = OpenSSL::PKey::DSA.generate(1024) + assert_predicate key1024, :private? + assert_equal 1024, key1024.p.num_bits + assert_equal 160, key1024.q.num_bits + end + def test_sign_verify - dsa512 = Fixtures.pkey("dsa512") + # The DSA valid size is 2048 or 3072 on FIPS. + # https://github.com/openssl/openssl/blob/7649b5548e5c0352b91d9d3ed695e42a2ac1e99c/providers/common/securitycheck.c#L185-L188 + dsa = Fixtures.pkey("dsa2048") data = "Sign me!" if defined?(OpenSSL::Digest::DSS1) - signature = dsa512.sign(OpenSSL::Digest.new('DSS1'), data) - assert_equal true, dsa512.verify(OpenSSL::Digest.new('DSS1'), signature, data) + signature = dsa.sign(OpenSSL::Digest.new('DSS1'), data) + assert_equal true, dsa.verify(OpenSSL::Digest.new('DSS1'), signature, data) end - signature = dsa512.sign("SHA256", data) - assert_equal true, dsa512.verify("SHA256", signature, data) + signature = dsa.sign("SHA256", data) + assert_equal true, dsa.verify("SHA256", signature, data) signature0 = (<<~'end;').unpack1("m") - MCwCFH5h40plgU5Fh0Z4wvEEpz0eE9SnAhRPbkRB8ggsN/vsSEYMXvJwjGg/ - 6g== + MD4CHQC0zmRkVOAHJTm28fS5PVUv+4LtBeNaKqr/yfmVAh0AsTcLqofWHoW8X5oWu8AOvngOcFVZ + cLTvhY3XNw== end; - assert_equal true, dsa512.verify("SHA256", signature0, data) + assert_equal true, dsa.verify("SHA256", signature0, data) signature1 = signature0.succ - assert_equal false, dsa512.verify("SHA256", signature1, data) + assert_equal false, dsa.verify("SHA256", signature1, data) end def test_sign_verify_raw - key = Fixtures.pkey("dsa512") + key = Fixtures.pkey("dsa2048") data = 'Sign me!' digest = OpenSSL::Digest.digest('SHA1', data) @@ -127,6 +135,8 @@ def test_DSAPrivateKey end def test_DSAPrivateKey_encrypted + omit_on_fips + # key = abcdef dsa512 = Fixtures.pkey("dsa512") pem = <<~EOF From 8fa6c364925bff4e704d4c0fd73555fb33aa7029 Mon Sep 17 00:00:00 2001 From: Andrii Konchyn Date: Wed, 27 Mar 2024 02:39:16 +0200 Subject: [PATCH 38/58] [ruby/strscan] Omit tests for `#scan_byte` and `#peek_byte` on TruffleRuby temporary (https://github.com/ruby/strscan/pull/91) The methods were added in #89 but they aren't implemented in TruffleRuby yet. So let's omit them for now to have CI green. https://github.com/ruby/strscan/commit/844d963b56 --- test/strscan/test_stringscanner.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/strscan/test_stringscanner.rb b/test/strscan/test_stringscanner.rb index 2884b8ef054ff7..143cf7197df673 100644 --- a/test/strscan/test_stringscanner.rb +++ b/test/strscan/test_stringscanner.rb @@ -9,6 +9,7 @@ module StringScannerTests def test_peek_byte + omit("not implemented on TruffleRuby") if RUBY_ENGINE == "truffleruby" s = create_string_scanner('ab') assert_equal 97, s.peek_byte assert_equal 97, s.scan_byte @@ -19,6 +20,7 @@ def test_peek_byte end def test_scan_byte + omit("not implemented on TruffleRuby") if RUBY_ENGINE == "truffleruby" s = create_string_scanner('ab') assert_equal 97, s.scan_byte assert_equal 98, s.scan_byte From e51435177e88fc845528dff7cf2bc2b75dd36144 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 19 Mar 2024 11:37:11 +0900 Subject: [PATCH 39/58] Update vendored resolv to 0.4.0 --- lib/rubygems/vendor/resolv/lib/resolv.rb | 79 ++++++++++++++++++++---- tool/bundler/vendor_gems.rb | 2 +- 2 files changed, 68 insertions(+), 13 deletions(-) diff --git a/lib/rubygems/vendor/resolv/lib/resolv.rb b/lib/rubygems/vendor/resolv/lib/resolv.rb index 8e31eb1bee97df..ac0ba0b3136a79 100644 --- a/lib/rubygems/vendor/resolv/lib/resolv.rb +++ b/lib/rubygems/vendor/resolv/lib/resolv.rb @@ -37,7 +37,7 @@ class Gem::Resolv - VERSION = "0.3.0" + VERSION = "0.4.0" ## # Looks up the first IP address for +name+. @@ -194,17 +194,10 @@ def lazy_initialize # :nodoc: File.open(@filename, 'rb') {|f| f.each {|line| line.sub!(/#.*/, '') - addr, hostname, *aliases = line.split(/\s+/) + addr, *hostnames = line.split(/\s+/) next unless addr - @addr2name[addr] = [] unless @addr2name.include? addr - @addr2name[addr] << hostname - @addr2name[addr].concat(aliases) - @name2addr[hostname] = [] unless @name2addr.include? hostname - @name2addr[hostname] << addr - aliases.each {|n| - @name2addr[n] = [] unless @name2addr.include? n - @name2addr[n] << addr - } + (@addr2name[addr] ||= []).concat(hostnames) + hostnames.each {|hostname| (@name2addr[hostname] ||= []) << addr} } } @name2addr.each {|name, arr| arr.reverse!} @@ -2544,8 +2537,70 @@ class ANY < Query TypeValue = 255 # :nodoc: end + ## + # CAA resource record defined in RFC 8659 + # + # These records identify certificate authority allowed to issue + # certificates for the given domain. + + class CAA < Resource + TypeValue = 257 + + ## + # Creates a new CAA for +flags+, +tag+ and +value+. + + def initialize(flags, tag, value) + unless (0..255) === flags + raise ArgumentError.new('flags must be an Integer between 0 and 255') + end + unless (1..15) === tag.bytesize + raise ArgumentError.new('length of tag must be between 1 and 15') + end + + @flags = flags + @tag = tag + @value = value + end + + ## + # Flags for this proprty: + # - Bit 0 : 0 = not critical, 1 = critical + + attr_reader :flags + + ## + # Property tag ("issue", "issuewild", "iodef"...). + + attr_reader :tag + + ## + # Property value. + + attr_reader :value + + ## + # Whether the critical flag is set on this property. + + def critical? + flags & 0x80 != 0 + end + + def encode_rdata(msg) # :nodoc: + msg.put_pack('C', @flags) + msg.put_string(@tag) + msg.put_bytes(@value) + end + + def self.decode_rdata(msg) # :nodoc: + flags, = msg.get_unpack('C') + tag = msg.get_string + value = msg.get_bytes + self.new flags, tag, value + end + end + ClassInsensitiveTypes = [ # :nodoc: - NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY + NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY, CAA ] ## diff --git a/tool/bundler/vendor_gems.rb b/tool/bundler/vendor_gems.rb index 2500e6c80028d3..f02d02656d2e45 100644 --- a/tool/bundler/vendor_gems.rb +++ b/tool/bundler/vendor_gems.rb @@ -9,7 +9,7 @@ gem "net-protocol", "0.2.2" gem "optparse", "0.4.0" gem "pub_grub", github: "jhawthorn/pub_grub" -gem "resolv", "0.3.0" +gem "resolv", "0.4.0" gem "timeout", "0.4.1" gem "thor", "1.3.0" gem "tsort", "0.2.0" From 0c114dfcc79cb4690705ec88ebf9147e5c03702d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 27 Mar 2024 11:45:01 +0900 Subject: [PATCH 40/58] Check existing ISeq wrapper --- iseq.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iseq.c b/iseq.c index d00cc9e9cc495b..3e45c7b6218902 100644 --- a/iseq.c +++ b/iseq.c @@ -1417,6 +1417,10 @@ static VALUE iseqw_new(const rb_iseq_t *iseq) { if (iseq->wrapper) { + if (rb_check_typeddata(iseq->wrapper, &iseqw_data_type) != iseq) { + rb_raise(rb_eTypeError, "wrong iseq wrapper: %" PRIsVALUE " for %p", + iseq->wrapper, (void *)iseq); + } return iseq->wrapper; } else { From 16c18eafb579cf2263c7e0057c4c81358fe62075 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 27 Mar 2024 12:53:43 +0900 Subject: [PATCH 41/58] Revert "Mark iseq structs with rb_gc_mark_movable" This reverts commit a31ca3500d995b6706f94ff72166d699c5faeb27 which broke debug inspector API. --- iseq.c | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/iseq.c b/iseq.c index 3e45c7b6218902..c2657f29d7fab9 100644 --- a/iseq.c +++ b/iseq.c @@ -1386,30 +1386,18 @@ rb_iseq_remove_coverage_all(void) static void iseqw_mark(void *ptr) { - rb_gc_mark_movable(*(VALUE *)ptr); + rb_gc_mark((VALUE)ptr); } static size_t iseqw_memsize(const void *ptr) { - return rb_iseq_memsize(*(const rb_iseq_t **)ptr); -} - -static void -iseqw_ref_update(void *ptr) -{ - VALUE *vptr = ptr; - *vptr = rb_gc_location(*vptr); + return rb_iseq_memsize((const rb_iseq_t *)ptr); } static const rb_data_type_t iseqw_data_type = { "T_IMEMO/iseq", - { - iseqw_mark, - RUBY_TYPED_DEFAULT_FREE, - iseqw_memsize, - iseqw_ref_update, - }, + {iseqw_mark, NULL, iseqw_memsize,}, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY|RUBY_TYPED_WB_PROTECTED }; @@ -1424,9 +1412,11 @@ iseqw_new(const rb_iseq_t *iseq) return iseq->wrapper; } else { - rb_iseq_t **ptr; - VALUE obj = TypedData_Make_Struct(rb_cISeq, rb_iseq_t *, &iseqw_data_type, ptr); - RB_OBJ_WRITE(obj, ptr, iseq); + union { const rb_iseq_t *in; void *out; } deconst; + VALUE obj; + deconst.in = iseq; + obj = TypedData_Wrap_Struct(rb_cISeq, &iseqw_data_type, deconst.out); + RB_OBJ_WRITTEN(obj, Qundef, iseq); /* cache a wrapper object */ RB_OBJ_WRITE((VALUE)iseq, &iseq->wrapper, obj); @@ -1750,9 +1740,7 @@ iseqw_s_compile_option_get(VALUE self) static const rb_iseq_t * iseqw_check(VALUE iseqw) { - rb_iseq_t **iseq_ptr; - TypedData_Get_Struct(iseqw, rb_iseq_t *, &iseqw_data_type, iseq_ptr); - rb_iseq_t *iseq = *iseq_ptr; + rb_iseq_t *iseq = DATA_PTR(iseqw); if (!ISEQ_BODY(iseq)) { rb_ibf_load_iseq_complete(iseq); From 66a0e8b008acb07c7eb2f31d65100bac1f361ff9 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Wed, 27 Mar 2024 11:07:25 +0900 Subject: [PATCH 42/58] d9234ba87b7e48381c8c44ef4a302ef368ee0ee7 is done to fix at related gems --- .github/workflows/macos.yml | 2 +- .github/workflows/ubuntu.yml | 2 +- .github/workflows/yjit-ubuntu.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index c5d88e30d70a6c..4bb41de22e799f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -105,7 +105,7 @@ jobs: timeout-minutes: 60 env: RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'minitest,test-unit,debug,bigdecimal,drb,typeprof' + TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' PRECHECK_BUNDLED_GEMS: 'no' - name: make skipped tests diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 62b55845b9e8d8..62e1b564b8828b 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -118,7 +118,7 @@ jobs: timeout-minutes: 40 env: RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'minitest,test-unit,debug,bigdecimal,drb,typeprof' + TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' PRECHECK_BUNDLED_GEMS: 'no' - name: make skipped tests diff --git a/.github/workflows/yjit-ubuntu.yml b/.github/workflows/yjit-ubuntu.yml index d93fb24e419bac..b74629340f0a7e 100644 --- a/.github/workflows/yjit-ubuntu.yml +++ b/.github/workflows/yjit-ubuntu.yml @@ -185,7 +185,7 @@ jobs: timeout-minutes: 60 env: RUBY_TESTOPTS: '-q --tty=no' - TEST_BUNDLED_GEMS_ALLOW_FAILURES: 'minitest,test-unit,debug,bigdecimal,drb,typeprof' + TEST_BUNDLED_GEMS_ALLOW_FAILURES: '' PRECHECK_BUNDLED_GEMS: 'no' SYNTAX_SUGGEST_TIMEOUT: '5' YJIT_BINDGEN_DIFF_OPTS: '--exit-code' From 6498c4399511ddf4b953be4c0745d5395406ed01 Mon Sep 17 00:00:00 2001 From: git Date: Wed, 27 Mar 2024 06:59:42 +0000 Subject: [PATCH 43/58] Update bundled gems list as of 2024-03-26 --- NEWS.md | 2 +- gems/bundled_gems | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index adc0ac5daaa0ef..ea274937758cba 100644 --- a/NEWS.md +++ b/NEWS.md @@ -57,7 +57,7 @@ The following bundled gems are updated. * test-unit 3.6.2 * net-ftp 0.3.4 * net-imap 0.4.10 -* net-smtp 0.4.0.1 +* net-smtp 0.5.0 * rbs 3.4.4 * typeprof 0.21.11 * debug 1.9.1 diff --git a/gems/bundled_gems b/gems/bundled_gems index 00a49537de3f86..5dfd755706a379 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -15,7 +15,7 @@ rss 0.3.0 https://github.com/ruby/rss net-ftp 0.3.4 https://github.com/ruby/net-ftp net-imap 0.4.10 https://github.com/ruby/net-imap net-pop 0.1.2 https://github.com/ruby/net-pop -net-smtp 0.4.0.1 https://github.com/ruby/net-smtp +net-smtp 0.5.0 https://github.com/ruby/net-smtp matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime rbs 3.4.4 https://github.com/ruby/rbs ba7872795d5de04adb8ff500c0e6afdc81a041dd From cbc11bcb63ccebd8fc3f12362b8d6dbcf06d7fdd Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 27 Mar 2024 15:48:41 +0900 Subject: [PATCH 44/58] Ignore errors on prerelease gems --- tool/downloader.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tool/downloader.rb b/tool/downloader.rb index 59bfd55f17c192..3a91ea0b938aed 100644 --- a/tool/downloader.rb +++ b/tool/downloader.rb @@ -79,6 +79,9 @@ def self.download(name, dir = nil, since = true, options = {}) require 'rubygems' options = options.dup options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__))) + if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease? + options[:ignore_http_client_errors] = true + end super("https://rubygems.org/downloads/#{name}", name, dir, since, options) end end @@ -237,6 +240,7 @@ def self.download(url, name, dir = nil, since = true, options = {}) $stdout.flush end mtime = nil + ignore_http_client_errors = options.delete(:ignore_http_client_errors) options = options.merge(http_options(file, since.nil? ? true : since)) begin data = with_retry(10) do @@ -247,12 +251,18 @@ def self.download(url, name, dir = nil, since = true, options = {}) data end rescue OpenURI::HTTPError => http_error - if http_error.message =~ /^304 / # 304 Not Modified + case http_error.message + when /^304 / # 304 Not Modified if $VERBOSE $stdout.puts "#{name} not modified" $stdout.flush end return file.to_path + when /^40/ # Net::HTTPClientError: 403 Forbidden, 404 Not Found + if ignore_http_client_errors + puts "Ignore #{url}: #{http_error.message}" + return file.to_path + end end raise rescue Timeout::Error From 44b5c912daae6d4e6dba6e40f13a2840c144cfa8 Mon Sep 17 00:00:00 2001 From: Cody Cutrer Date: Thu, 14 Sep 2023 11:35:50 -0600 Subject: [PATCH 45/58] [rubygems/rubygems] Allow installing plugins from path via CLI Also bring the man page up to date. https://github.com/rubygems/rubygems/commit/a849bd6947 --- lib/bundler/cli/plugin.rb | 3 ++- lib/bundler/man/bundle-add.1 | 2 +- lib/bundler/man/bundle-binstubs.1 | 2 +- lib/bundler/man/bundle-cache.1 | 2 +- lib/bundler/man/bundle-check.1 | 2 +- lib/bundler/man/bundle-clean.1 | 2 +- lib/bundler/man/bundle-config.1 | 2 +- lib/bundler/man/bundle-console.1 | 2 +- lib/bundler/man/bundle-doctor.1 | 2 +- lib/bundler/man/bundle-exec.1 | 2 +- lib/bundler/man/bundle-gem.1 | 2 +- lib/bundler/man/bundle-help.1 | 2 +- lib/bundler/man/bundle-info.1 | 2 +- lib/bundler/man/bundle-init.1 | 2 +- lib/bundler/man/bundle-inject.1 | 2 +- lib/bundler/man/bundle-install.1 | 2 +- lib/bundler/man/bundle-list.1 | 2 +- lib/bundler/man/bundle-lock.1 | 2 +- lib/bundler/man/bundle-open.1 | 2 +- lib/bundler/man/bundle-outdated.1 | 2 +- lib/bundler/man/bundle-platform.1 | 2 +- lib/bundler/man/bundle-plugin.1 | 9 ++++--- lib/bundler/man/bundle-plugin.1.ronn | 8 +++++-- lib/bundler/man/bundle-pristine.1 | 2 +- lib/bundler/man/bundle-remove.1 | 2 +- lib/bundler/man/bundle-show.1 | 2 +- lib/bundler/man/bundle-update.1 | 2 +- lib/bundler/man/bundle-version.1 | 2 +- lib/bundler/man/bundle-viz.1 | 2 +- lib/bundler/man/bundle.1 | 2 +- lib/bundler/man/gemfile.5 | 2 +- lib/bundler/plugin/installer.rb | 35 ++++++++++++++++++---------- lib/bundler/plugin/installer/path.rb | 18 ++++++++++++++ lib/bundler/plugin/source_list.rb | 8 +++---- spec/bundler/plugins/install_spec.rb | 10 ++++++++ 35 files changed, 97 insertions(+), 50 deletions(-) create mode 100644 lib/bundler/plugin/installer/path.rb diff --git a/lib/bundler/cli/plugin.rb b/lib/bundler/cli/plugin.rb index d946e495e93532..fd61ef0d954dea 100644 --- a/lib/bundler/cli/plugin.rb +++ b/lib/bundler/cli/plugin.rb @@ -5,7 +5,7 @@ module Bundler class CLI::Plugin < Thor desc "install PLUGINS", "Install the plugin from the source" long_desc <<-D - Install plugins either from the rubygems source provided (with --source option) or from a git source provided with --git. If no sources are provided, it uses Gem.sources + Install plugins either from the rubygems source provided (with --source option), from a git source provided with --git, or a local path provided with --path. If no sources are provided, it uses Gem.sources D method_option "source", type: :string, default: nil, banner: "URL of the RubyGems source to fetch the plugin from" method_option "version", type: :string, default: nil, banner: "The version of the plugin to fetch" @@ -13,6 +13,7 @@ class CLI::Plugin < Thor method_option "local_git", type: :string, default: nil, banner: "Path of the local git repo to fetch from (deprecated)" method_option "branch", type: :string, default: nil, banner: "The git branch to checkout" method_option "ref", type: :string, default: nil, banner: "The git revision to check out" + method_option "path", type: :string, default: nil, banner: "Path of a local gem to directly use" def install(*plugins) Bundler::Plugin.install(plugins, options) end diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index f85d21f9599b83..a6cbc88f344e27 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-ADD" "1" "February 2024" "" +.TH "BUNDLE\-ADD" "1" "March 2024" "" .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index dd0b3cd4e899bc..2b35bc956a7837 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-BINSTUBS" "1" "February 2024" "" +.TH "BUNDLE\-BINSTUBS" "1" "March 2024" "" .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index 8e39bb92c3c464..3b86b995a62689 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CACHE" "1" "February 2024" "" +.TH "BUNDLE\-CACHE" "1" "March 2024" "" .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index 82920d71887425..7f18e265375288 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CHECK" "1" "February 2024" "" +.TH "BUNDLE\-CHECK" "1" "March 2024" "" .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index 04cf55275caea2..0180eb38a213ac 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CLEAN" "1" "February 2024" "" +.TH "BUNDLE\-CLEAN" "1" "March 2024" "" .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 4b0728c213c2e1..b768f1e3d29ec6 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONFIG" "1" "February 2024" "" +.TH "BUNDLE\-CONFIG" "1" "March 2024" "" .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index 467a375f89de92..1368a50eb163fa 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONSOLE" "1" "February 2024" "" +.TH "BUNDLE\-CONSOLE" "1" "March 2024" "" .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index e77bafec29b46c..80eaf2a8882524 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-DOCTOR" "1" "February 2024" "" +.TH "BUNDLE\-DOCTOR" "1" "March 2024" "" .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 926675aa5fdb06..191863c045ba5f 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-EXEC" "1" "February 2024" "" +.TH "BUNDLE\-EXEC" "1" "March 2024" "" .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index 744c4917c7e152..464d8d11264553 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-GEM" "1" "February 2024" "" +.TH "BUNDLE\-GEM" "1" "March 2024" "" .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index 613ff261dde66a..3604ad6127d6bd 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-HELP" "1" "February 2024" "" +.TH "BUNDLE\-HELP" "1" "March 2024" "" .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index 7df617d859cea3..647f5987befbbc 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INFO" "1" "February 2024" "" +.TH "BUNDLE\-INFO" "1" "March 2024" "" .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index 4a9c6b01a1b58f..2c41a3c7deff9a 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INIT" "1" "February 2024" "" +.TH "BUNDLE\-INIT" "1" "March 2024" "" .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index f88f6dbe605253..c7269db34d0a21 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INJECT" "1" "February 2024" "" +.TH "BUNDLE\-INJECT" "1" "March 2024" "" .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index f41def305fe71b..3fa1a467e2b7ce 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INSTALL" "1" "February 2024" "" +.TH "BUNDLE\-INSTALL" "1" "March 2024" "" .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index e5b56767518278..f91fd95739351f 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LIST" "1" "February 2024" "" +.TH "BUNDLE\-LIST" "1" "March 2024" "" .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index a63c6fd5a5283a..f992f5ee5f0a78 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LOCK" "1" "February 2024" "" +.TH "BUNDLE\-LOCK" "1" "March 2024" "" .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index 8cf0ec0e364805..53d3541555153b 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OPEN" "1" "February 2024" "" +.TH "BUNDLE\-OPEN" "1" "March 2024" "" .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index 30e795748c0d24..f79eff5ae99808 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OUTDATED" "1" "February 2024" "" +.TH "BUNDLE\-OUTDATED" "1" "March 2024" "" .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 4f5240c242c430..d2133ec4d3fecb 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLATFORM" "1" "February 2024" "" +.TH "BUNDLE\-PLATFORM" "1" "March 2024" "" .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index 29db99376010ba..cbdfac11b6bfb5 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,10 +1,10 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLUGIN" "1" "February 2024" "" +.TH "BUNDLE\-PLUGIN" "1" "March 2024" "" .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins .SH "SYNOPSIS" -\fBbundle plugin\fR install PLUGINS [\-\-source=\fISOURCE\fR] [\-\-version=\fIversion\fR] [\-\-git=\fIgit\-url\fR] [\-\-branch=\fIbranch\fR|\-\-ref=\fIrev\fR] +\fBbundle plugin\fR install PLUGINS [\-\-source=\fISOURCE\fR] [\-\-version=\fIversion\fR] [\-\-git=\fIgit\-url\fR] [\-\-branch=\fIbranch\fR|\-\-ref=\fIrev\fR] [\-\-path=\fIpath\fR] .br \fBbundle plugin\fR uninstall PLUGINS .br @@ -37,7 +37,10 @@ Install bundler\-graph gem from Git repository\. You can use standard Git URLs l .br \fBfile:///path/to/repo\fR .IP -When you specify \fB\-\-git\fR, you can use \fB\-\-branch\fR or \fB\-\-ref\fR to specify any branch, tag, or commit hash (revision) to use\. When you specify both, only the latter is used\. +When you specify \fB\-\-git\fR, you can use \fB\-\-branch\fR or \fB\-\-ref\fR to specify any branch, tag, or commit hash (revision) to use\. +.TP +\fBbundle plugin install bundler\-graph \-\-path \.\./bundler\-graph\fR +Install bundler\-graph gem from a local path\. .SS "uninstall" Uninstall the plugin(s) specified in PLUGINS\. .SS "list" diff --git a/lib/bundler/man/bundle-plugin.1.ronn b/lib/bundler/man/bundle-plugin.1.ronn index e38776a47cd7a5..b0a34660ea84da 100644 --- a/lib/bundler/man/bundle-plugin.1.ronn +++ b/lib/bundler/man/bundle-plugin.1.ronn @@ -4,7 +4,8 @@ bundle-plugin(1) -- Manage Bundler plugins ## SYNOPSIS `bundle plugin` install PLUGINS [--source=] [--version=] - [--git=] [--branch=|--ref=]
+ [--git=] [--branch=|--ref=] + [--path=]
`bundle plugin` uninstall PLUGINS
`bundle plugin` list
`bundle plugin` help [COMMAND] @@ -36,7 +37,10 @@ Install the given plugin(s). `/path/to/repo`
`file:///path/to/repo` - When you specify `--git`, you can use `--branch` or `--ref` to specify any branch, tag, or commit hash (revision) to use. When you specify both, only the latter is used. + When you specify `--git`, you can use `--branch` or `--ref` to specify any branch, tag, or commit hash (revision) to use. + +* `bundle plugin install bundler-graph --path ../bundler-graph`: + Install bundler-graph gem from a local path. ### uninstall diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index 81a4758374fa86..faa04d76762a30 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PRISTINE" "1" "February 2024" "" +.TH "BUNDLE\-PRISTINE" "1" "March 2024" "" .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index 1172577bca9d68..3f8cbbd9b684bf 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-REMOVE" "1" "February 2024" "" +.TH "BUNDLE\-REMOVE" "1" "March 2024" "" .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index 338502ae358d18..bc72c6e3b6baf3 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-SHOW" "1" "February 2024" "" +.TH "BUNDLE\-SHOW" "1" "March 2024" "" .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 8174d9b84133e9..d1284c2e72b54f 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-UPDATE" "1" "February 2024" "" +.TH "BUNDLE\-UPDATE" "1" "March 2024" "" .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index 5361b0493e2fee..05905e1347fc35 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VERSION" "1" "February 2024" "" +.TH "BUNDLE\-VERSION" "1" "March 2024" "" .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index ea726ceb499537..681563cd4c3d7f 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VIZ" "1" "February 2024" "" +.TH "BUNDLE\-VIZ" "1" "March 2024" "" .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index 053b56d00a3b9c..1d2c780060f7c1 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE" "1" "February 2024" "" +.TH "BUNDLE" "1" "March 2024" "" .SH "NAME" \fBbundle\fR \- Ruby Dependency Management .SH "SYNOPSIS" diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index 6db532c34ae96d..39503f22a66233 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "GEMFILE" "5" "February 2024" "" +.TH "GEMFILE" "5" "March 2024" "" .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs .SH "SYNOPSIS" diff --git a/lib/bundler/plugin/installer.rb b/lib/bundler/plugin/installer.rb index 7267f58f5d5ea9..6771f3f1537698 100644 --- a/lib/bundler/plugin/installer.rb +++ b/lib/bundler/plugin/installer.rb @@ -10,6 +10,7 @@ module Plugin class Installer autoload :Rubygems, File.expand_path("installer/rubygems", __dir__) autoload :Git, File.expand_path("installer/git", __dir__) + autoload :Path, File.expand_path("installer/path", __dir__) def install(names, options) check_sources_consistency!(options) @@ -18,6 +19,8 @@ def install(names, options) if options[:git] install_git(names, version, options) + elsif options[:path] + install_path(names, version, options[:path]) else sources = options[:source] || Gem.sources install_rubygems(names, version, sources) @@ -50,8 +53,8 @@ def check_sources_consistency!(options) options[:git] = options.delete(:local_git) end - if (options.keys & [:source, :git]).length > 1 - raise InvalidOption, "Only one of --source, or --git may be specified" + if (options.keys & [:source, :git, :path]).length > 1 + raise InvalidOption, "Only one of --source, --git, or --path may be specified" end if (options.key?(:branch) || options.key?(:ref)) && !options.key?(:git) @@ -64,10 +67,19 @@ def check_sources_consistency!(options) end def install_git(names, version, options) - uri = options.delete(:git) - options["uri"] = uri + source_list = SourceList.new + source = source_list.add_git_source({ "uri" => options[:git], + "branch" => options[:branch], + "ref" => options[:ref] }) + + install_all_sources(names, version, source_list, source) + end + + def install_path(names, version, path) + source_list = SourceList.new + source = source_list.add_path_source({ "path" => path }) - install_all_sources(names, version, options, options[:source]) + install_all_sources(names, version, source_list, source) end # Installs the plugin from rubygems source and returns the path where the @@ -79,16 +91,15 @@ def install_git(names, version, options) # # @return [Hash] map of names to the specs of plugins installed def install_rubygems(names, version, sources) - install_all_sources(names, version, nil, sources) - end - - def install_all_sources(names, version, git_source_options, rubygems_source) source_list = SourceList.new - source_list.add_git_source(git_source_options) if git_source_options - Array(rubygems_source).each {|remote| source_list.add_global_rubygems_remote(remote) } if rubygems_source + Array(sources).each {|remote| source_list.add_global_rubygems_remote(remote) } + + install_all_sources(names, version, source_list) + end - deps = names.map {|name| Dependency.new name, version } + def install_all_sources(names, version, source_list, source = nil) + deps = names.map {|name| Dependency.new(name, version, { "source" => source }) } Bundler.configure_gem_home_and_path(Plugin.root) diff --git a/lib/bundler/plugin/installer/path.rb b/lib/bundler/plugin/installer/path.rb new file mode 100644 index 00000000000000..1b60724b5e4ea3 --- /dev/null +++ b/lib/bundler/plugin/installer/path.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Bundler + module Plugin + class Installer + class Path < Bundler::Source::Path + def root + Plugin.root + end + + def generate_bin(spec, disable_extensions = false) + # Need to find a way without code duplication + # For now, we can ignore this + end + end + end + end +end diff --git a/lib/bundler/plugin/source_list.rb b/lib/bundler/plugin/source_list.rb index 547661cf2f73da..746996de5548ac 100644 --- a/lib/bundler/plugin/source_list.rb +++ b/lib/bundler/plugin/source_list.rb @@ -9,6 +9,10 @@ def add_git_source(options = {}) add_source_to_list Plugin::Installer::Git.new(options), git_sources end + def add_path_source(options = {}) + add_source_to_list Plugin::Installer::Path.new(options), path_sources + end + def add_rubygems_source(options = {}) add_source_to_list Plugin::Installer::Rubygems.new(options), @rubygems_sources end @@ -17,10 +21,6 @@ def all_sources path_sources + git_sources + rubygems_sources + [metadata_source] end - def default_source - git_sources.first || global_rubygems_source - end - private def rubygems_aggregate_class diff --git a/spec/bundler/plugins/install_spec.rb b/spec/bundler/plugins/install_spec.rb index 86eb4e584c6a8b..61c513ed723d76 100644 --- a/spec/bundler/plugins/install_spec.rb +++ b/spec/bundler/plugins/install_spec.rb @@ -212,6 +212,16 @@ def exec(command, args) end end + it "installs from a path source" do + build_lib "path_plugin" do |s| + s.write "plugins.rb" + end + bundle "plugin install path_plugin --path #{lib_path("path_plugin-1.0")}" + + expect(out).to include("Installed plugin path_plugin") + plugin_should_be_installed("path_plugin") + end + context "Gemfile eval" do before do allow(Bundler::SharedHelpers).to receive(:find_gemfile).and_return(bundled_app_gemfile) From e4b210906564cb5e922e8331a6b3724624377fdb Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:54:50 -0400 Subject: [PATCH 46/58] [PRISM] Fix ASCII-compatible check for eval encoding --- prism_compile.c | 7 +++++-- test/.excludes-prism/TestEval.rb | 1 - 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index d7eada1538ad38..a5b1219d6c1743 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8611,9 +8611,12 @@ pm_load_parse_file(pm_parse_result_t *result, VALUE filepath) VALUE pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath) { - pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); - rb_encoding *encoding = rb_enc_get(source); + if (!rb_enc_asciicompat(encoding)) { + return rb_exc_new_cstr(rb_eArgError, "invalid source encoding"); + } + + pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); pm_options_encoding_set(&result->options, rb_enc_name(encoding)); pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); diff --git a/test/.excludes-prism/TestEval.rb b/test/.excludes-prism/TestEval.rb index 83f3e38fc9721f..6cc6bdfb1d65e6 100644 --- a/test/.excludes-prism/TestEval.rb +++ b/test/.excludes-prism/TestEval.rb @@ -1,2 +1 @@ -exclude(:test_eval_ascii_incompatible, "incorrect encoding") exclude(:test_file_encoding, "incorrect encoding") From 6f8a252e96715a72f5aef20e081a0efe7d66a850 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:55:29 -0400 Subject: [PATCH 47/58] [PRISM] Enable passing heredoc test --- test/.excludes-prism/TestParse.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index b5c4c6202d8cff..87694ac15a9851 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -1,5 +1,4 @@ exclude(:test_assign_in_conditional, "missing warning") -exclude(:test_here_document, "incorrect heredoc") exclude(:test_magic_comment, "incorrect encoding") exclude(:test_shareable_constant_value_nested, "ractor support") exclude(:test_shareable_constant_value_nonliteral, "ractor support") From 8b2fc859703fe3be0408f05b8734cc7624017f96 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:56:17 -0400 Subject: [PATCH 48/58] [PRISM] Enable passing frozen string in array test --- test/.excludes-prism/TestRubyLiteral.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index 1f78ea55a4172b..4cfe25215e1190 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,4 +1,3 @@ exclude(:test_dregexp, "unknown") -exclude(:test_frozen_string_in_array_literal, "incorrect frozen value") exclude(:test_hash_duplicated_key, "parser implementation dependent") exclude(:test_string, "https://github.com/ruby/prism/issues/2331") From 843c760a0ff7857d9ded3352877ca7d22f69e065 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 14:57:28 -0400 Subject: [PATCH 49/58] [PRISM] Enable passing syntax tests --- test/.excludes-prism/TestSyntax.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb index 23c755738e196c..680974906bcb6f 100644 --- a/test/.excludes-prism/TestSyntax.rb +++ b/test/.excludes-prism/TestSyntax.rb @@ -1,8 +1,4 @@ exclude(:test_dedented_heredoc_concatenation, "unknown") exclude(:test_dedented_heredoc_continued_line, "unknown") exclude(:test_duplicated_when, "unknown") -exclude(:test_integer_suffix, "unknown") exclude(:test_it, "https://github.com/ruby/prism/issues/2323") -exclude(:test_keyword_duplicated, "unknown") -exclude(:test_numbered_parameter, "unknown") -exclude(:test_too_big_nth_ref, "unknown") From a1ae29e87dd6d68da99df68420cd451b63b2211d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 15:14:51 -0400 Subject: [PATCH 50/58] [PRISM] Enable other passing specs --- spec/prism.mspec | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/spec/prism.mspec b/spec/prism.mspec index 99af52c0f1601c..490de9e348339b 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -2,28 +2,33 @@ ## Command line MSpec.register(:exclude, "The -S command line option runs launcher found in PATH, but only code after the first /#!.*ruby.*/-ish line in target file") -MSpec.register(:exclude, /^The -x command line option/) -MSpec.register(:exclude, "The --enable and --disable flags can be used with frozen-string-literal") -MSpec.register(:exclude, /^The --enable-frozen-string-literal flag/) +MSpec.register(:exclude, "The -x command line option runs code after the first /#!.*ruby.*/-ish line in target file") +MSpec.register(:exclude, "The -x command line option fails when /#!.*ruby.*/-ish line in target file is not found") +MSpec.register(:exclude, "The -x command line option behaves as -x was set when non-ruby shebang is encountered on first line") +MSpec.register(:exclude, "The --enable-frozen-string-literal flag causes string literals to produce the same object for literals with the same content in different files") MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") ## Language MSpec.register(:exclude, "The BEGIN keyword runs multiple begins in FIFO order") MSpec.register(:exclude, "Executing break from within a block works when passing through a super call") MSpec.register(:exclude, "The defined? keyword when called with a method name in a void context warns about the void context when parsing it") -MSpec.register(:exclude, /^Hash literal expands an '\*\*\{\}'/) +MSpec.register(:exclude, "Hash literal expands an '**{}' or '**obj' element with the last key/value pair taking precedence") +MSpec.register(:exclude, "Hash literal expands an '**{}' and warns when finding an additional duplicate key afterwards") MSpec.register(:exclude, "Hash literal merges multiple nested '**obj' in Hash literals") -MSpec.register(:exclude, /^Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes/) +MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes") +MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes and 'key: value' syntax used") MSpec.register(:exclude, "The next statement in a method is invalid and raises a SyntaxError") MSpec.register(:exclude, "Pattern matching variable pattern does not support using variable name (except _) several times") MSpec.register(:exclude, "Pattern matching Hash pattern raise SyntaxError when keys duplicate in pattern") -MSpec.register(:exclude, "Regexp with character classes supports [[:alpha:][:digit:][:etc:]] (predefined character classes)") -MSpec.register(:exclude, /^Regexps with encoding modifiers/) -MSpec.register(:exclude, "Regexps with grouping raises a SyntaxError when parentheses aren't balanced") -MSpec.register(:exclude, "Regexps with modifiers supports (?imx-imx) (inline modifiers)") -MSpec.register(:exclude, "Regexps with modifiers supports (?imx-imx:expr) (scoped inline modifiers)") -MSpec.register(:exclude, "Literal Regexps throws SyntaxError for malformed literals") -MSpec.register(:exclude, "The rescue keyword raises SyntaxError when else is used without rescue and ensure") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation /o") +MSpec.register(:exclude, "Regexps with encoding modifiers preserves EUC-JP as /e encoding through interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /s (Windows_31J encoding) with interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /s (Windows_31J encoding) with interpolation and /o") +MSpec.register(:exclude, "Regexps with encoding modifiers preserves Windows-31J as /s encoding through interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /u (UTF8 encoding) with interpolation") +MSpec.register(:exclude, "Regexps with encoding modifiers supports /u (UTF8 encoding) with interpolation and /o") +MSpec.register(:exclude, "Regexps with encoding modifiers preserves UTF-8 as /u encoding through interpolation") MSpec.register(:exclude, "A Symbol literal raises an SyntaxError at parse time when Symbol with invalid bytes") ## Core @@ -33,8 +38,6 @@ MSpec.register(:exclude, "Kernel#eval includes file and line information in synt MSpec.register(:exclude, "Kernel#eval evaluates string with given filename and negative linenumber") MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows spaces before the magic encoding comment") MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows a shebang line and some spaces before the magic encoding comment") -MSpec.register(:exclude, "Kernel#eval with a magic encoding comment ignores the frozen_string_literal magic comment if it appears after a token and warns if $VERBOSE is true") -MSpec.register(:exclude, "Regexp#source has US-ASCII encoding when created from an ASCII-only \\u{} literal") MSpec.register(:exclude, "TracePoint#eval_script is the evald source code") MSpec.register(:exclude, "TracePoint#event returns the type of event") MSpec.register(:exclude, "TracePoint#inspect returns a String showing the event, method, path and line for a :return event") @@ -58,9 +61,6 @@ MSpec.register(:exclude, "Coverage.result does not clear counters when stop: fal MSpec.register(:exclude, "Coverage.result clears counters (sets 0 values) when stop: false and clear: true specified") MSpec.register(:exclude, "Coverage.result does not clear counters when stop: false and clear: false specified") MSpec.register(:exclude, "Coverage.start measures coverage within eval") -MSpec.register(:exclude, "Digest::SHA256.file when passed a path to a file that exists can be used with frozen-string-literal") MSpec.register(:exclude, "ERB#filename raises an exception if there are errors processing content") MSpec.register(:exclude, "ERB#filename uses '(erb)' as filename when filename is not set") -MSpec.register(:exclude, "mkmf can be required with --enable-frozen-string-literal") -MSpec.register(:exclude, "RbConfig::CONFIG contains no frozen strings even with --enable-frozen-string-literal") MSpec.register(:exclude, "Socket.gethostbyaddr using an IPv6 address with an explicit address family raises SocketError when the address is not supported by the family") From 42d1cd8f7fa62b585283e566d4be9a390631c43d Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 15:32:01 -0400 Subject: [PATCH 51/58] [PRISM] Pass --enable-frozen-string-literal through to evals --- iseq.c | 2 -- prism_compile.c | 46 ++++++++++++++++++++++++++++------------------ prism_compile.h | 1 - ruby.c | 2 -- spec/prism.mspec | 1 - vm_eval.c | 2 -- 6 files changed, 28 insertions(+), 26 deletions(-) diff --git a/iseq.c b/iseq.c index c2657f29d7fab9..2d412dfcc9265d 100644 --- a/iseq.c +++ b/iseq.c @@ -1253,8 +1253,6 @@ pm_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, NUM2INT(line)); - pm_options_frozen_string_literal_init(&result, option.frozen_string_literal); - VALUE error; if (RB_TYPE_P(src, T_FILE)) { VALUE filepath = rb_io_path(src); diff --git a/prism_compile.c b/prism_compile.c index a5b1219d6c1743..aec2f80f4102dc 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -8479,6 +8479,30 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) return Qnil; } +/** + * Set the frozen_string_literal option based on the default value used by the + * CRuby compiler. + */ +static void +pm_options_frozen_string_literal_init(pm_options_t *options) +{ + int frozen_string_literal = rb_iseq_opt_frozen_string_literal(); + + switch (frozen_string_literal) { + case ISEQ_FROZEN_STRING_LITERAL_UNSET: + break; + case ISEQ_FROZEN_STRING_LITERAL_DISABLED: + pm_options_frozen_string_literal_set(options, false); + break; + case ISEQ_FROZEN_STRING_LITERAL_ENABLED: + pm_options_frozen_string_literal_set(options, true); + break; + default: + rb_bug("pm_options_frozen_string_literal_init: invalid frozen_string_literal=%d", frozen_string_literal); + break; + } +} + /** * Returns an array of ruby String objects that represent the lines of the * source file that the given parser parsed. @@ -8514,24 +8538,6 @@ pm_parse_file_script_lines(const pm_scope_node_t *scope_node, const pm_parser_t return lines; } -void -pm_options_frozen_string_literal_init(pm_parse_result_t *result, int frozen_string_literal) -{ - switch (frozen_string_literal) { - case ISEQ_FROZEN_STRING_LITERAL_UNSET: - break; - case ISEQ_FROZEN_STRING_LITERAL_DISABLED: - pm_options_frozen_string_literal_set(&result->options, false); - break; - case ISEQ_FROZEN_STRING_LITERAL_ENABLED: - pm_options_frozen_string_literal_set(&result->options, true); - break; - default: - rb_bug("pm_options_frozen_string_literal_init: invalid frozen_string_literal=%d", frozen_string_literal); - break; - } -} - /** * Attempt to load the file into memory. Return a Ruby error if the file cannot * be read. @@ -8551,6 +8557,7 @@ pm_load_file(pm_parse_result_t *result, VALUE filepath) return err; } + pm_options_frozen_string_literal_init(&result->options); return Qnil; } @@ -8616,6 +8623,7 @@ pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath) return rb_exc_new_cstr(rb_eArgError, "invalid source encoding"); } + pm_options_frozen_string_literal_init(&result->options); pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); pm_options_encoding_set(&result->options, rb_enc_name(encoding)); @@ -8658,6 +8666,8 @@ pm_parse_stdin_fgets(char *string, int size, void *stream) VALUE pm_parse_stdin(pm_parse_result_t *result) { + pm_options_frozen_string_literal_init(&result->options); + pm_buffer_t buffer; pm_node_t *node = pm_parse_stream(&result->parser, &buffer, (void *) rb_stdin, pm_parse_stdin_fgets, &result->options); diff --git a/prism_compile.h b/prism_compile.h index 0c1510d67f9814..427fa54b516013 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -47,7 +47,6 @@ typedef struct { bool parsed; } pm_parse_result_t; -void pm_options_frozen_string_literal_init(pm_parse_result_t *result, int frozen_string_literal); VALUE pm_load_file(pm_parse_result_t *result, VALUE filepath); VALUE pm_parse_file(pm_parse_result_t *result, VALUE filepath); VALUE pm_load_parse_file(pm_parse_result_t *result, VALUE filepath); diff --git a/ruby.c b/ruby.c index 5c1d44ec6df56c..59cbbd0360b690 100644 --- a/ruby.c +++ b/ruby.c @@ -2116,8 +2116,6 @@ prism_script(ruby_cmdline_options_t *opt, pm_parse_result_t *result) pm_options_t *options = &result->options; pm_options_line_set(options, 1); - pm_options_frozen_string_literal_init(result, rb_iseq_opt_frozen_string_literal()); - if (opt->ext.enc.name != 0) { pm_options_encoding_set(options, StringValueCStr(opt->ext.enc.name)); } diff --git a/spec/prism.mspec b/spec/prism.mspec index 490de9e348339b..bd427392e2f4e3 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -5,7 +5,6 @@ MSpec.register(:exclude, "The -S command line option runs launcher found in PATH MSpec.register(:exclude, "The -x command line option runs code after the first /#!.*ruby.*/-ish line in target file") MSpec.register(:exclude, "The -x command line option fails when /#!.*ruby.*/-ish line in target file is not found") MSpec.register(:exclude, "The -x command line option behaves as -x was set when non-ruby shebang is encountered on first line") -MSpec.register(:exclude, "The --enable-frozen-string-literal flag causes string literals to produce the same object for literals with the same content in different files") MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") ## Language diff --git a/vm_eval.c b/vm_eval.c index 50c81f79d25353..a859a76f245bc8 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1663,8 +1663,6 @@ pm_eval_make_iseq(VALUE src, VALUE fname, int line, pm_parse_result_t result = { 0 }; pm_options_line_set(&result.options, line); - pm_options_frozen_string_literal_init(&result, rb_iseq_opt_frozen_string_literal()); - // Cout scopes, one for each parent iseq, plus one for our local scope int scopes_count = 0; do { From 2505c27fdfa67c3191f9b5902a30ab3139a8ab6f Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Tue, 26 Mar 2024 15:45:43 -0400 Subject: [PATCH 52/58] [PRISM] Fix up some error formatting edge cases --- prism/prism.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 368a6c12ab768c..c6353f451b586c 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -19588,7 +19588,7 @@ typedef struct { #define PM_COLOR_GRAY "\033[38;5;102m" #define PM_COLOR_RED "\033[1;31m" -#define PM_COLOR_RESET "\033[0m" +#define PM_COLOR_RESET "\033[m" static inline pm_error_t * pm_parser_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_list, const pm_newline_list_t *newline_list) { @@ -19644,7 +19644,11 @@ pm_parser_errors_format_sort(const pm_parser_t *parser, const pm_list_t *error_l static inline void pm_parser_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t *newline_list, const char *number_prefix, int32_t line, pm_buffer_t *buffer) { - size_t index = (size_t) (line - parser->start_line); + int32_t line_delta = line - parser->start_line; + assert(line_delta >= 0); + + size_t index = (size_t) line_delta; + assert(index < newline_list->size); const uint8_t *start = &parser->start[newline_list->offsets[index]]; const uint8_t *end; @@ -19765,7 +19769,7 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // the source before the error to give some context. We'll be careful not to // display the same line twice in case the errors are close enough in the // source. - int32_t last_line = 0; + int32_t last_line = parser->start_line - 1; const pm_encoding_t *encoding = parser->encoding; for (size_t index = 0; index < error_list->size; index++) { @@ -19791,13 +19795,16 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // the line that has the error in it. if ((index == 0) || (error->line != last_line)) { if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 13); + pm_buffer_append_string(buffer, PM_COLOR_RED "> " PM_COLOR_RESET, 12); } else { pm_buffer_append_string(buffer, "> ", 2); } pm_parser_errors_format_line(parser, newline_list, error_format.number_prefix, error->line, buffer); } + const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; + if (start == parser->end) pm_buffer_append_byte(buffer, '\n'); + // Now we'll display the actual error message. We'll do this by first // putting the prefix to the line, then a bunch of blank spaces // depending on the column, then as many carets as we need to display @@ -19811,13 +19818,11 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col pm_buffer_append_string(buffer, error_format.blank_prefix, error_format.blank_prefix_length); size_t column = 0; - const uint8_t *start = &parser->start[newline_list->offsets[error->line - start_line]]; - while (column < error->column_end) { if (column < error->column_start) { pm_buffer_append_byte(buffer, ' '); } else if (colorize) { - pm_buffer_append_string(buffer, PM_COLOR_RED "^" PM_COLOR_RESET, 12); + pm_buffer_append_string(buffer, PM_COLOR_RED "^" PM_COLOR_RESET, 11); } else { pm_buffer_append_byte(buffer, '^'); } From 9ad175c1ee7137321310c4e87d8ce952b95abc8f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:34:28 -0400 Subject: [PATCH 53/58] Register rb_fix_to_s_static as global right after creating If a GC runs right during creating a rb_fix_to_s_static, it may cause the previous ones to become swept by the GC because they have not been registered by rb_vm_register_global_object. --- numeric.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/numeric.c b/numeric.c index 41beff0919f7f3..f0a0e3c279e9d1 100644 --- a/numeric.c +++ b/numeric.c @@ -6251,19 +6251,25 @@ Init_Numeric(void) rb_define_method(rb_cInteger, "digits", rb_int_digits, -1); - rb_fix_to_s_static[0] = rb_fstring_literal("0"); - rb_fix_to_s_static[1] = rb_fstring_literal("1"); - rb_fix_to_s_static[2] = rb_fstring_literal("2"); - rb_fix_to_s_static[3] = rb_fstring_literal("3"); - rb_fix_to_s_static[4] = rb_fstring_literal("4"); - rb_fix_to_s_static[5] = rb_fstring_literal("5"); - rb_fix_to_s_static[6] = rb_fstring_literal("6"); - rb_fix_to_s_static[7] = rb_fstring_literal("7"); - rb_fix_to_s_static[8] = rb_fstring_literal("8"); - rb_fix_to_s_static[9] = rb_fstring_literal("9"); - for(int i = 0; i < 10; i++) { - rb_vm_register_global_object(rb_fix_to_s_static[i]); - } +#define fix_to_s_static(n) do { \ + VALUE lit = rb_fstring_literal(#n); \ + rb_fix_to_s_static[n] = lit; \ + rb_vm_register_global_object(lit); \ + RB_GC_GUARD(lit); \ + } while (0) + + fix_to_s_static(0); + fix_to_s_static(1); + fix_to_s_static(2); + fix_to_s_static(3); + fix_to_s_static(4); + fix_to_s_static(5); + fix_to_s_static(6); + fix_to_s_static(7); + fix_to_s_static(8); + fix_to_s_static(9); + +#undef fix_to_s_static rb_cFloat = rb_define_class("Float", rb_cNumeric); From 1d99fe430aff34a90544d23edbd707f6f240dafe Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:35:44 -0400 Subject: [PATCH 54/58] Register classpath of FrozenCore before converting to ICLASS Since ICLASS do not mark the classpath, we need to register it as a global object before we convert RubyVM::FrozenCore as a ICLASS. --- vm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm.c b/vm.c index 882514db41e57a..c9d99a34ea7aa1 100644 --- a/vm.c +++ b/vm.c @@ -3908,6 +3908,7 @@ Init_VM(void) /* FrozenCore (hidden) */ fcore = rb_class_new(rb_cBasicObject); rb_set_class_path(fcore, rb_cRubyVM, "FrozenCore"); + rb_vm_register_global_object(rb_class_path_cached(fcore)); RBASIC(fcore)->flags = T_ICLASS; klass = rb_singleton_class(fcore); rb_define_method_id(klass, id_core_set_method_alias, m_core_set_method_alias, 3); @@ -3927,7 +3928,6 @@ Init_VM(void) RBASIC_CLEAR_CLASS(klass); rb_obj_freeze(klass); rb_vm_register_global_object(fcore); - rb_vm_register_global_object(rb_class_path_cached(fcore)); rb_mRubyVMFrozenCore = fcore; /* From f14e52c8c45f1288537ff38c153096d095b5ad20 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:37:21 -0400 Subject: [PATCH 55/58] Fix setting GC stress at boot when objspace not available --- debug.c | 2 +- gc.c | 26 +++++++++++++++----------- internal/gc.h | 2 +- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/debug.c b/debug.c index 546ffc440f155b..4717a0bc9c5e90 100644 --- a/debug.c +++ b/debug.c @@ -226,7 +226,7 @@ ruby_env_debug_option(const char *str, int len, void *arg) } if (NAME_MATCH("gc_stress")) { - rb_gc_stress_set(Qtrue); + rb_gc_initial_stress_set(Qtrue); return 1; } SET_WHEN("core", ruby_enable_coredump, 1); diff --git a/gc.c b/gc.c index 8812c0fea5e986..04df9dc71bf998 100644 --- a/gc.c +++ b/gc.c @@ -1876,10 +1876,22 @@ calloc1(size_t n) return calloc(1, n); } +static VALUE initial_stress = Qfalse; + +void +rb_gc_initial_stress_set(VALUE flag) +{ + initial_stress = flag; +} + rb_objspace_t * rb_objspace_alloc(void) { rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t)); + + objspace->flags.gc_stressful = RTEST(initial_stress); + objspace->gc_stress_mode = initial_stress; + objspace->flags.measure_gc = 1; malloc_limit = gc_params.malloc_limit_min; objspace->finalize_deferred_pjob = rb_postponed_job_preregister(0, gc_finalize_deferred, objspace); @@ -1898,8 +1910,6 @@ rb_objspace_alloc(void) rb_darray_make_without_gc(&objspace->weak_references, 0); - dont_gc_on(); - return objspace; } @@ -8951,7 +8961,7 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) /* reason may be clobbered, later, so keep set immediate_sweep here */ objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); - if (!heap_allocated_pages) return FALSE; /* heap is not ready */ + if (!heap_allocated_pages) return TRUE; /* heap is not ready */ if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */ GC_ASSERT(gc_mode(objspace) == gc_mode_none); @@ -11147,20 +11157,14 @@ gc_stress_get(rb_execution_context_t *ec, VALUE self) return ruby_gc_stress_mode; } -void -rb_gc_stress_set(VALUE flag) +static VALUE +gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) { rb_objspace_t *objspace = &rb_objspace; objspace->flags.gc_stressful = RTEST(flag); objspace->gc_stress_mode = flag; -} - -static VALUE -gc_stress_set_m(rb_execution_context_t *ec, VALUE self, VALUE flag) -{ - rb_gc_stress_set(flag); return flag; } diff --git a/internal/gc.h b/internal/gc.h index 82a1b901f8af59..5b1180fd91ccdc 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -223,7 +223,7 @@ void rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr); void rb_gc_ref_update_table_values_only(st_table *tbl); -void rb_gc_stress_set(VALUE flag); +void rb_gc_initial_stress_set(VALUE flag); #define rb_gc_mark_and_move_ptr(ptr) do { \ VALUE _obj = (VALUE)*(ptr); \ From 19916bac968fa79437649821932a6b8869414002 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 11:37:54 -0400 Subject: [PATCH 56/58] Revert "skip `test_gc_stress_at_startup`" This reverts commit 3680981c7b71df8c3a426164787ccefe5296bb25. --- test/ruby/test_gc.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 470e53fc1fb3f0..39b001c3d0fee6 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -768,7 +768,6 @@ def initialize end def test_gc_stress_at_startup - omit 'TODO: fixme later' assert_in_out_err([{"RUBY_DEBUG"=>"gc_stress"}], '', [], [], '[Bug #15784]', success: true, timeout: 60) end From aa794cc5a237bf4b7aa8f1917fa4ed86949d71c6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 26 Mar 2024 14:25:41 -0400 Subject: [PATCH 57/58] Turn GC off at boot on Windows This is to stop crashes like: .\miniruby.exe: [BUG] Segmentation fault ruby 3.4.0dev (2024-03-26T15:38:26Z pull/10370/merge 040ea2ae2f) [x64-mswin64_140] -- Control frame information ----------------------------------------------- c:0001 p:0000 s:0003 E:000d00 DUMMY [FINISH] -- Threading information --------------------------------------------------- Total ractor count: 1 Ruby thread count for this ractor: 1 -- C level backtrace information ------------------------------------------- C:\Windows\SYSTEM32\ntdll.dll(NtWaitForSingleObject+0x14) [0x00007FFA091AFC74] C:\Windows\System32\KERNELBASE.dll(WaitForSingleObjectEx+0x93) [0x00007FFA05BB4513] D:\a\ruby\ruby\build\miniruby.exe(rb_print_backtrace+0x3e) [0x00007FF64E536EFE] d:\a\ruby\ruby\src\vm_dump.c:844 D:\a\ruby\ruby\build\miniruby.exe(rb_vm_bugreport+0x1ae) [0x00007FF64E5370B2] d:\a\ruby\ruby\src\vm_dump.c:1154 D:\a\ruby\ruby\build\miniruby.exe(rb_bug_for_fatal_signal+0x77) [0x00007FF64E3FF357] d:\a\ruby\ruby\src\error.c:1087 D:\a\ruby\ruby\build\miniruby.exe(sigsegv+0x71) [0x00007FF64E4C79E5] d:\a\ruby\ruby\src\signal.c:926 C:\Windows\System32\ucrtbase.dll(seh_filter_exe+0x233) [0x00007FFA0521CE03] D:\a\ruby\ruby\build\miniruby.exe(`__scrt_common_main_seh'::`1'::filt$0+0x16) [0x00007FF64E594DA0] f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl:269 C:\Windows\SYSTEM32\VCRUNTIME140.dll(_C_specific_handler+0x9f) [0x00007FF9E54AF73F] C:\Windows\SYSTEM32\ntdll.dll(_chkstk+0x11f) [0x00007FFA091B4C2F] C:\Windows\SYSTEM32\ntdll.dll(RtlWalkFrameChain+0x14bf) [0x00007FFA09114CEF] C:\Windows\SYSTEM32\ntdll.dll(KiUserExceptionDispatcher+0x2e) [0x00007FFA091B399E] D:\a\ruby\ruby\build\miniruby.exe(newobj_of+0x6d) [0x00007FF64E418615] d:\a\ruby\ruby\src\gc.c:2949 D:\a\ruby\ruby\build\miniruby.exe(rb_wb_protected_newobj_of+0x32) [0x00007FF64E41C7DA] d:\a\ruby\ruby\src\gc.c:2974 D:\a\ruby\ruby\build\miniruby.exe(str_new0+0x64) [0x00007FF64E4E7F48] d:\a\ruby\ruby\src\string.c:887 D:\a\ruby\ruby\build\miniruby.exe(rb_enc_str_new+0x40) [0x00007FF64E4D89B8] d:\a\ruby\ruby\src\string.c:945 D:\a\ruby\ruby\build\miniruby.exe(iseq_compile_each0+0xdd7) [0x00007FF64E3B4A23] d:\a\ruby\ruby\src\compile.c:10368 D:\a\ruby\ruby\build\miniruby.exe(iseq_compile_each+0x74) [0x00007FF64E3B3C40] d:\a\ruby\ruby\src\compile.c:9971 --- gc.c | 5 +++++ vm.c | 2 ++ 2 files changed, 7 insertions(+) diff --git a/gc.c b/gc.c index 04df9dc71bf998..2b363ac7d08abc 100644 --- a/gc.c +++ b/gc.c @@ -1910,6 +1910,11 @@ rb_objspace_alloc(void) rb_darray_make_without_gc(&objspace->weak_references, 0); + // TODO: debug why on Windows Ruby crashes on boot when GC is on. +#ifdef _WIN32 + dont_gc_on(); +#endif + return objspace; } diff --git a/vm.c b/vm.c index c9d99a34ea7aa1..36f6700ad7d459 100644 --- a/vm.c +++ b/vm.c @@ -4198,7 +4198,9 @@ Init_VM(void) */ rb_define_global_const("TOPLEVEL_BINDING", rb_binding_new()); +#ifdef _WIN32 rb_objspace_gc_enable(vm->objspace); +#endif } vm_init_redefined_flag(); From e4d6479730fa87fc420cfe9ae6b83206737b0db5 Mon Sep 17 00:00:00 2001 From: Jeremy Evans Date: Fri, 22 Mar 2024 13:53:35 -0700 Subject: [PATCH 58/58] Add array/hash implicit allocation tests These are designed to prevent allocation regressions (commits that increase the number of implicitly allocated arrays and hashes). We have already had three commits in the last couple weeks to fix allocation regressions: * 15dc3aaa311b32203d8ffb414bcf9b8e55ce5691 * aceee71c35e0b387691836e756b4e008efd84cf1 * c38878494377c94f2425a81e598260ea944ef7f3 This test suite should hopefully allow us to find such regressions in CI before commit, to avoid committing future allocation regressions. This uses assert_separately around each set of tests. Doing it for each individual check was too slow. Failures are gathered and reported at the end of the the suite as a single assertion, with the message describing all failures. --- test/ruby/test_allocation.rb | 656 +++++++++++++++++++++++++++++++++++ 1 file changed, 656 insertions(+) create mode 100644 test/ruby/test_allocation.rb diff --git a/test/ruby/test_allocation.rb b/test/ruby/test_allocation.rb new file mode 100644 index 00000000000000..48348c0fbd514d --- /dev/null +++ b/test/ruby/test_allocation.rb @@ -0,0 +1,656 @@ +# frozen_string_literal: false +require 'test/unit' + +class TestAllocation < Test::Unit::TestCase + def check_allocations(checks) + assert_separately([], <<~RUBY) + $allocations = [0, 0] + $counts = {} + failures = [] + + def self.num_allocations + ObjectSpace.count_objects($counts) + arrays = $counts[:T_ARRAY] + hashes = $counts[:T_HASH] + yield + ObjectSpace.count_objects($counts) + arrays -= $counts[:T_ARRAY] + hashes -= $counts[:T_HASH] + $allocations[0] = -arrays + $allocations[1] = -hashes + end + + define_singleton_method(:check_allocations) do |num_arrays, num_hashes, check_code| + instance_eval <<~RB + empty_array = empty_array = [] + empty_hash = empty_hash = {} + array1 = array1 = [1] + hash1 = hash1 = {a: 2} + nill = nill = nil + block = block = lambda{} + + num_allocations do + \#{check_code} + end + RB + + if num_arrays != $allocations[0] + failures << "Expected \#{num_arrays} array allocations for \#{check_code.inspect}, but \#{$allocations[0]} arrays allocated" + end + if num_hashes != $allocations[1] + failures << "Expected \#{num_hashes} hash allocations for \#{check_code.inspect}, but \#{$allocations[1]} hashes allocated" + end + end + + GC.start + GC.disable + + #{checks} + + unless failures.empty? + assert_equal(true, false, failures.join("\n")) + end + RUBY + end + + class Literal < self + def test_array_literal + check_allocations(<<~RUBY) + check_allocations(1, 0, "[]") + check_allocations(1, 0, "[1]") + check_allocations(1, 0, "[*empty_array]") + check_allocations(1, 0, "[*empty_array, 1, *empty_array]") + check_allocations(1, 0, "[*empty_array, *empty_array]") + check_allocations(1, 0, "[#{'1,'*100000}]") + RUBY + end + + def test_hash_literal + check_allocations(<<~RUBY) + check_allocations(0, 1, "{}") + check_allocations(0, 1, "{a: 1}") + check_allocations(0, 1, "{**empty_hash}") + check_allocations(0, 1, "{**empty_hash, a: 1, **empty_hash}") + check_allocations(0, 1, "{**empty_hash, **empty_hash}") + check_allocations(0, 1, "{#{100000.times.map{|i| "a#{i}: 1"}.join(',')}}") + RUBY + end + end + + class MethodCall < self + def block + '' + end + + def test_no_parameters + only_block = block.empty? ? block : block[2..] + check_allocations(<<~RUBY) + def self.none(#{only_block}); end + + check_allocations(0, 0, "none(#{only_block})") + check_allocations(0, 0, "none(*empty_array#{block})") + check_allocations(0, 0, "none(**empty_hash#{block})") + check_allocations(0, 0, "none(*empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "none(*empty_array, *empty_array#{block})") + check_allocations(0, 1, "none(**empty_hash, **empty_hash#{block})") + check_allocations(1, 1, "none(*empty_array, *empty_array, **empty_hash, **empty_hash#{block})") + RUBY + end + + def test_required_parameter + check_allocations(<<~RUBY) + def self.required(x#{block}); end + + check_allocations(0, 0, "required(1#{block})") + check_allocations(0, 0, "required(1, *empty_array#{block})") + check_allocations(0, 0, "required(1, **empty_hash#{block})") + check_allocations(0, 0, "required(1, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "required(*array1#{block})") + check_allocations(0, 1, "required(**hash1#{block})") + + check_allocations(1, 0, "required(*array1, *empty_array#{block})") + check_allocations(0, 1, "required(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "required(*array1, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "required(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_optional_parameter + check_allocations(<<~RUBY) + def self.optional(x=nil#{block}); end + + check_allocations(0, 0, "optional(1#{block})") + check_allocations(0, 0, "optional(1, *empty_array#{block})") + check_allocations(0, 0, "optional(1, **empty_hash#{block})") + check_allocations(0, 0, "optional(1, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "optional(*array1#{block})") + check_allocations(0, 1, "optional(**hash1#{block})") + + check_allocations(1, 0, "optional(*array1, *empty_array#{block})") + check_allocations(0, 1, "optional(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "optional(*array1, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "optional(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_positional_splat_parameter + check_allocations(<<~RUBY) + def self.splat(*x#{block}); end + + check_allocations(1, 0, "splat(1#{block})") + check_allocations(1, 0, "splat(1, *empty_array#{block})") + check_allocations(1, 0, "splat(1, **empty_hash#{block})") + check_allocations(1, 0, "splat(1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat(*array1#{block})") + check_allocations(1, 0, "splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "splat(*array1, **empty_hash#{block})") + check_allocations(1, 0, "splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat(1, *array1#{block})") + check_allocations(1, 0, "splat(1, *array1, *empty_array#{block})") + check_allocations(1, 0, "splat(1, *array1, **empty_hash#{block})") + check_allocations(1, 0, "splat(1, *array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat(*array1#{block})") + check_allocations(1, 1, "splat(**hash1#{block})") + + check_allocations(1, 0, "splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "splat(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat(*array1, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_required_and_positional_splat_parameters + check_allocations(<<~RUBY) + def self.req_splat(x, *y#{block}); end + + check_allocations(1, 0, "req_splat(1#{block})") + check_allocations(1, 0, "req_splat(1, *empty_array#{block})") + check_allocations(1, 0, "req_splat(1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(*array1#{block})") + check_allocations(1, 0, "req_splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "req_splat(*array1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(1, *array1#{block})") + check_allocations(1, 0, "req_splat(1, *array1, *empty_array#{block})") + check_allocations(1, 0, "req_splat(1, *array1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(1, *array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "req_splat(*array1#{block})") + check_allocations(1, 1, "req_splat(**hash1#{block})") + + check_allocations(1, 0, "req_splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "req_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "req_splat(*array1, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "req_splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_positional_splat_and_post_parameters + check_allocations(<<~RUBY) + def self.splat_post(*x, y#{block}); end + + check_allocations(1, 0, "splat_post(1#{block})") + check_allocations(1, 0, "splat_post(1, *empty_array#{block})") + check_allocations(1, 0, "splat_post(1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(*array1#{block})") + check_allocations(1, 0, "splat_post(*array1, *empty_array#{block})") + check_allocations(1, 0, "splat_post(*array1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(1, *array1#{block})") + check_allocations(1, 0, "splat_post(1, *array1, *empty_array#{block})") + check_allocations(1, 0, "splat_post(1, *array1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(1, *array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_post(*array1#{block})") + check_allocations(1, 1, "splat_post(**hash1#{block})") + + check_allocations(1, 0, "splat_post(*array1, *empty_array#{block})") + check_allocations(1, 1, "splat_post(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat_post(*array1, *empty_array, **empty_hash#{block})") + check_allocations(1, 1, "splat_post(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_keyword_parameter + check_allocations(<<~RUBY) + def self.keyword(a: nil#{block}); end + + check_allocations(0, 0, "keyword(a: 2#{block})") + check_allocations(0, 0, "keyword(*empty_array, a: 2#{block})") + check_allocations(0, 1, "keyword(a:2, **empty_hash#{block})") + check_allocations(0, 1, "keyword(**empty_hash, a: 2#{block})") + + check_allocations(0, 0, "keyword(**nil#{block})") + check_allocations(0, 0, "keyword(**empty_hash#{block})") + check_allocations(0, 0, "keyword(**hash1#{block})") + check_allocations(0, 0, "keyword(*empty_array, **hash1#{block})") + check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") + check_allocations(0, 1, "keyword(**empty_hash, **hash1#{block})") + + check_allocations(0, 0, "keyword(*empty_array#{block})") + check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "keyword(*empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "keyword(*empty_array#{block})") + check_allocations(0, 1, "keyword(**hash1, **empty_hash#{block})") + check_allocations(1, 0, "keyword(*empty_array, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "keyword(*empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "keyword(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.keyword_splat(**kw#{block}); end + + check_allocations(0, 1, "keyword_splat(a: 2#{block})") + check_allocations(0, 1, "keyword_splat(*empty_array, a: 2#{block})") + check_allocations(0, 1, "keyword_splat(a:2, **empty_hash#{block})") + check_allocations(0, 1, "keyword_splat(**empty_hash, a: 2#{block})") + + check_allocations(0, 1, "keyword_splat(**nil#{block})") + check_allocations(0, 1, "keyword_splat(**empty_hash#{block})") + check_allocations(0, 1, "keyword_splat(**hash1#{block})") + check_allocations(0, 1, "keyword_splat(*empty_array, **hash1#{block})") + check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(0, 1, "keyword_splat(**empty_hash, **hash1#{block})") + + check_allocations(0, 1, "keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 1, "keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "keyword_splat(*empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "keyword_splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_keyword_and_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.keyword_and_keyword_splat(a: 1, **kw#{block}); end + + check_allocations(0, 1, "keyword_and_keyword_splat(a: 2#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array, a: 2#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(a:2, **empty_hash#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**empty_hash, a: 2#{block})") + + check_allocations(0, 1, "keyword_and_keyword_splat(**nil#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**empty_hash#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array, **hash1#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**empty_hash, **hash1#{block})") + + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 1, "keyword_and_keyword_splat(*empty_array#{block})") + check_allocations(0, 1, "keyword_and_keyword_splat(**hash1, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, *empty_array, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "keyword_and_keyword_splat(*empty_array, **hash1, **empty_hash#{block})") + RUBY + end + + def test_required_positional_and_keyword_parameter + check_allocations(<<~RUBY) + def self.required_and_keyword(b, a: nil#{block}); end + + check_allocations(0, 0, "required_and_keyword(1, a: 2#{block})") + check_allocations(0, 0, "required_and_keyword(1, *empty_array, a: 2#{block})") + check_allocations(0, 1, "required_and_keyword(1, a:2, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword(1, **empty_hash, a: 2#{block})") + + check_allocations(0, 0, "required_and_keyword(1, **nil#{block})") + check_allocations(0, 0, "required_and_keyword(1, **empty_hash#{block})") + check_allocations(0, 0, "required_and_keyword(1, **hash1#{block})") + check_allocations(0, 0, "required_and_keyword(1, *empty_array, **hash1#{block})") + check_allocations(0, 1, "required_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword(1, **empty_hash, **hash1#{block})") + + check_allocations(0, 0, "required_and_keyword(1, *empty_array#{block})") + check_allocations(0, 1, "required_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "required_and_keyword(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "required_and_keyword(*array1, a: 2#{block})") + + check_allocations(0, 0, "required_and_keyword(*array1, **nill#{block})") + check_allocations(0, 0, "required_and_keyword(*array1, **empty_hash#{block})") + check_allocations(0, 0, "required_and_keyword(*array1, **hash1#{block})") + check_allocations(1, 0, "required_and_keyword(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "required_and_keyword(*array1, *empty_array#{block})") + check_allocations(1, 0, "required_and_keyword(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword(*array1, *empty_array, **hash1, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "required_and_keyword(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "required_and_keyword(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "required_and_keyword(*array1, **nil#{block})") + RUBY + end + + def test_positional_splat_and_keyword_parameter + check_allocations(<<~RUBY) + def self.splat_and_keyword(*b, a: nil#{block}); end + + check_allocations(1, 0, "splat_and_keyword(1, a: 2#{block})") + check_allocations(1, 0, "splat_and_keyword(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "splat_and_keyword(1, **nil#{block})") + check_allocations(1, 0, "splat_and_keyword(1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(1, **hash1#{block})") + check_allocations(1, 0, "splat_and_keyword(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "splat_and_keyword(1, *empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(1, 0, "splat_and_keyword(*array1, a: 2#{block})") + + check_allocations(1, 0, "splat_and_keyword(*array1, **nill#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, **hash1#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "splat_and_keyword(*array1, *empty_array#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "splat_and_keyword(*array1, **nil#{block})") + RUBY + end + + def test_required_and_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.required_and_keyword_splat(b, **kw#{block}); end + + check_allocations(0, 1, "required_and_keyword_splat(1, a: 2#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(1, **nil#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **hash1#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(1, *empty_array#{block})") + check_allocations(0, 1, "required_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(*array1, a: 2#{block})") + + check_allocations(0, 1, "required_and_keyword_splat(*array1, **nill#{block})") + check_allocations(0, 1, "required_and_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(0, 1, "required_and_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + # Currently allocates 1 array unnecessarily due to splatarray true + check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "required_and_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_positional_splat_and_keyword_splat_parameter + check_allocations(<<~RUBY) + def self.splat_and_keyword_splat(*b, **kw#{block}); end + + check_allocations(1, 1, "splat_and_keyword_splat(1, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(1, **nil#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, a: 2#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **nill#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "splat_and_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_anonymous_splat_and_anonymous_keyword_splat_parameters + check_allocations(<<~RUBY) + def self.anon_splat_and_anon_keyword_splat(*, **#{block}); end + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a: 2#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **nil#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, a: 2#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **nill#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_nested_anonymous_splat_and_anonymous_keyword_splat_parameters + check_allocations(<<~RUBY) + def self.anon_splat_and_anon_keyword_splat(*, **#{block}); t(*, **) end; def self.t(*, **#{block}); end + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a: 2#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **nil#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, a: 2#{block})") + + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **nill#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash#{block})") + check_allocations(0, 0, "anon_splat_and_anon_keyword_splat(*array1, **hash1#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "anon_splat_and_anon_keyword_splat(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "anon_splat_and_anon_keyword_splat(*array1, **nil#{block})") + RUBY + end + + def test_argument_forwarding + check_allocations(<<~RUBY) + def self.argument_forwarding(...); end + + check_allocations(1, 1, "argument_forwarding(1, a: 2#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "argument_forwarding(1, **nil#{block})") + check_allocations(1, 0, "argument_forwarding(1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(1, *empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, a: 2#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, **nill#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **empty_hash#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + RUBY + end + + def test_nested_argument_forwarding + check_allocations(<<~RUBY) + def self.argument_forwarding(...); t(...) end; def self.t(...) end + + check_allocations(1, 1, "argument_forwarding(1, a: 2#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(1, a:2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, a: 2#{block})") + + check_allocations(1, 0, "argument_forwarding(1, **nil#{block})") + check_allocations(1, 0, "argument_forwarding(1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, **hash1#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, **empty_hash, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(1, *empty_array#{block})") + check_allocations(1, 1, "argument_forwarding(1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(1, *empty_array, *empty_array, **empty_hash#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, a: 2#{block})") + + check_allocations(0, 0, "argument_forwarding(*array1, **nill#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **empty_hash#{block})") + check_allocations(0, 0, "argument_forwarding(*array1, **hash1#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **hash1#{block})") + + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, *empty_array, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, *empty_array, **hash1, **empty_hash#{block})") + + check_allocations(1, 1, "argument_forwarding(1, *empty_array, a: 2, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(1, *empty_array, **hash1, **empty_hash#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **empty_hash, a: 2#{block})") + check_allocations(1, 1, "argument_forwarding(*array1, **hash1, **empty_hash#{block})") + check_allocations(1, 0, "argument_forwarding(*array1, **nil#{block})") + RUBY + end + + class WithBlock < self + def block + ', &block' + end + end + end +end