From 02cc9d48f1958e8f22757116358cf5863cb109f4 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Fri, 24 Nov 2023 15:06:20 +1100 Subject: [PATCH 01/78] [ruby/resolv] Catch EPROTONOSUPPORT as a sign of no IPv6 as well (https://github.com/ruby/resolv/pull/41) If IPv6 is disabled inside a freebsd jail, it seems this returns EPROTONOSUPPORT and not EAFNOSUPPORT. In both cases, we should simply try some other listed DNS servers. Fixes [Bug #19928] https://bugs.ruby-lang.org/issues/19928 https://github.com/ruby/resolv/commit/5e2d48708b --- lib/resolv.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/resolv.rb b/lib/resolv.rb index 9e8335389a933b..0c96ee80b90467 100644 --- a/lib/resolv.rb +++ b/lib/resolv.rb @@ -750,7 +750,7 @@ def lazy_initialize next if @socks_hash[bind_host] begin sock = UDPSocket.new(af) - rescue Errno::EAFNOSUPPORT + rescue Errno::EAFNOSUPPORT, Errno::EPROTONOSUPPORT next # The kernel doesn't support the address family. end @socks << sock From 12f4e9655e6278b14f3b75ccccf63a535d72c120 Mon Sep 17 00:00:00 2001 From: Burdette Lamar Date: Thu, 23 Nov 2023 22:39:24 -0600 Subject: [PATCH 02/78] Add recommendations on link formatting in documentation --- doc/contributing/documentation_guide.md | 30 +++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/contributing/documentation_guide.md b/doc/contributing/documentation_guide.md index af918170030324..2192c12d469699 100644 --- a/doc/contributing/documentation_guide.md +++ b/doc/contributing/documentation_guide.md @@ -199,6 +199,36 @@ will be autolinked: If not, or if you suppress autolinking, consider forcing [monofont](rdoc-ref:RDoc::MarkupReference@Monofont). +### Explicit Links + +When writing an explicit link, follow these guidelines. + +#### +rdoc-ref+ Scheme + +Use the +rdoc-ref+ scheme for: + +- A link in core documentation to other core documentation. +- A link in core documentation to documentation in a standard library package. +- A link in a standard library package to other documentation in that same + standard library package. + +See section "+rdoc-ref+ Scheme" in {Links}[rdoc-ref:RDoc::MarkupReference@Links]. + +#### URL-Based Link + +Use a full URL-based link for: + +- A link in standard library documentation to documentation in the core. +- A link in standard library documentation to documentation in a different + standard library package. + +Doing so ensures that the link will valid even when the package documentation +is built independently (separately from the core documentation). + +The link should lead to a target in https://docs.ruby-lang.org/en/master/. + +Also use a full URL-based link for a link to an off-site document. + ### Variable Names The name of a variable (as specified in its call-seq) should be marked up as From f792b55b2138c75bfc4efe1d15af5e924f4349bc Mon Sep 17 00:00:00 2001 From: Mau Magnaguagno Date: Tue, 21 Nov 2023 00:52:54 -0300 Subject: [PATCH 03/78] [rubygems/rubygems] Prefer String#each_line in Gem::Command Replace ``String#split("\n").each`` with ``String#each_line``. https://github.com/rubygems/rubygems/commit/958744807d --- lib/rubygems/command.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/rubygems/command.rb b/lib/rubygems/command.rb index 01deac6890b25d..14a95c78061946 100644 --- a/lib/rubygems/command.rb +++ b/lib/rubygems/command.rb @@ -489,7 +489,7 @@ def add_parser_description # :nodoc: @parser.separator nil @parser.separator " Description:" - formatted.split("\n").each do |line| + formatted.each_line |line| @parser.separator " #{line.rstrip}" end end @@ -516,8 +516,8 @@ def add_parser_run_info(title, content) @parser.separator nil @parser.separator " #{title}:" - content.split(/\n/).each do |line| - @parser.separator " #{line}" + content.each_line do |line| + @parser.separator " #{line.rstrip}" end end @@ -526,7 +526,7 @@ def add_parser_summary # :nodoc: @parser.separator nil @parser.separator " Summary:" - wrap(@summary, 80 - 4).split("\n").each do |line| + wrap(@summary, 80 - 4).each_line do |line| @parser.separator " #{line.strip}" end end From 87ddfb33a0bb7f771d4a1aee0c27c598267fb926 Mon Sep 17 00:00:00 2001 From: Mau Magnaguagno Date: Tue, 21 Nov 2023 00:59:30 -0300 Subject: [PATCH 04/78] [rubygems/rubygems] Fix typo missing do https://github.com/rubygems/rubygems/commit/4eade32ad6 --- lib/rubygems/command.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/rubygems/command.rb b/lib/rubygems/command.rb index 14a95c78061946..fd2cf61a050921 100644 --- a/lib/rubygems/command.rb +++ b/lib/rubygems/command.rb @@ -489,7 +489,7 @@ def add_parser_description # :nodoc: @parser.separator nil @parser.separator " Description:" - formatted.each_line |line| + formatted.each_line do |line| @parser.separator " #{line.rstrip}" end end From 24e0b185ab7ea67eea298fc2ad7985f7ce4deba1 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Fri, 24 Nov 2023 08:06:14 +0900 Subject: [PATCH 05/78] .travis.yml: Add s390x again. As Travis CI IBM z pipeine is operational again, I will add s390x again to check the case. Though Travis s390x infra team is still investigating the root cause. https://www.traviscistatus.com/ Please revert this commit or comment on the , when you see the s390x builds are not starting again. Sorry for inconvenience. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b12149408eac2..6d942d142584f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -104,8 +104,7 @@ matrix: include: - <<: *arm64-linux - <<: *ppc64le-linux - # The s390x builds are not starting. - # - <<: *s390x-linux + - <<: *s390x-linux # FIXME: lib/rubygems/util.rb:104 glob_files_in_dir - # :411:in glob: File name too long - (Errno::ENAMETOOLONG) # https://github.com/rubygems/rubygems/issues/7132 @@ -114,7 +113,7 @@ matrix: # Allow failures for the unstable jobs. # - name: arm64-linux # - name: ppc64le-linux - - name: s390x-linux + # - name: s390x-linux # The 2nd arm64 pipeline may be unstable. # - name: arm32-linux fast_finish: true From 9d7ac1ba1ca279df14400814516fab14240cdc35 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 13 Nov 2023 17:45:12 +0900 Subject: [PATCH 06/78] [DOC] Add links about timezones --- doc/timezones.rdoc | 4 +++- timev.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/timezones.rdoc b/doc/timezones.rdoc index 5b3a224d14ef86..949e0f65c1b79c 100644 --- a/doc/timezones.rdoc +++ b/doc/timezones.rdoc @@ -57,7 +57,9 @@ in the range -86399..86399: === Timezone Objects -The zone value may be an object responding to certain timezone methods. +The zone value may be an object responding to certain timezone methods, an +instance of {Timezone}[https://github.com/panthomakos/timezone] and +{TZInfo}[https://tzinfo.github.io] for example. The timezone methods are: diff --git a/timev.rb b/timev.rb index e710dc699003e2..80de049a0722f0 100644 --- a/timev.rb +++ b/timev.rb @@ -66,7 +66,7 @@ # # Time.new(2002, 10, 31, 2, 2, 2, "+02:00") #=> 2002-10-31 02:02:02 +0200 # -# Or a timezone object: +# Or {a timezone object}[rdoc-ref:timezones.rdoc@Timezone+Objects]: # # zone = timezone("Europe/Athens") # Eastern European Time, UTC+2 # Time.new(2002, 10, 31, 2, 2, 2, zone) #=> 2002-10-31 02:02:02 +0200 From 045e74d8e484bca963eba6210d85222fa9362615 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 24 Nov 2023 17:28:52 +0900 Subject: [PATCH 07/78] [DOC] Simplify signature of `Time.new` --- timev.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/timev.rb b/timev.rb index 80de049a0722f0..28e0a444b17800 100644 --- a/timev.rb +++ b/timev.rb @@ -287,6 +287,9 @@ def self.at(time, subsec = false, unit = :microsecond, in: nil) end end + # call-seq: + # Time.new(year = nil, mon = nil, mday = nil, hour = nil, min = nil, sec = nil, zone = nil, in: nil, precision: 9) + # # Returns a new \Time object based on the given arguments, # by default in the local timezone. # From 2ecc372a5d623ff4e470c354c771f8797e527c2d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 24 Nov 2023 19:16:18 +0900 Subject: [PATCH 08/78] [DOC] State timezone info in the string wins `in:` keyword --- test/ruby/test_time.rb | 1 + timev.rb | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 45439be995a6d4..2a541bbe8c8f47 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -77,6 +77,7 @@ def test_new_from_string assert_equal(Time.new(2021), Time.new("2021")) assert_equal(Time.new(2021, 12, 25, in: "+09:00"), Time.new("2021-12-25+09:00")) + assert_equal(Time.new(2021, 12, 25, in: "+09:00"), Time.new("2021-12-25+09:00", in: "-01:00")) assert_equal(0.123456r, Time.new("2021-12-25 00:00:00.123456 +09:00").subsec) assert_equal(0.123456789r, Time.new("2021-12-25 00:00:00.123456789876 +09:00").subsec) diff --git a/timev.rb b/timev.rb index 28e0a444b17800..db3a1925cec639 100644 --- a/timev.rb +++ b/timev.rb @@ -380,6 +380,12 @@ def self.at(time, subsec = false, unit = :microsecond, in: nil) # Time.new(in: '-12:00') # # => 2022-08-23 08:49:26.1941467 -1200 # + # Since +in:+ keyword argument just provides the default, so if the + # first argument in single string form contains time zone information, + # this keyword argument will be silently ignored. + # + # Time.new('2000-01-01 00:00:00 +0100', in: '-0500').utc_offset # => 3600 + # # - +precision+: maximum effective digits in sub-second part, default is 9. # More digits will be truncated, as other operations of \Time. # Ignored unless the first argument is a string. From 0c0875fe6050a79e5525a71f6819168bd4947ff8 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 24 Nov 2023 19:37:54 +0900 Subject: [PATCH 09/78] [DOC] Mention `Time.find_timezone` method --- doc/timezones.rdoc | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/doc/timezones.rdoc b/doc/timezones.rdoc index 949e0f65c1b79c..d59efba37d402c 100644 --- a/doc/timezones.rdoc +++ b/doc/timezones.rdoc @@ -17,6 +17,7 @@ The value given with any of these must be one of the following - {Single-letter offset}[rdoc-ref:timezones.rdoc@Single-Letter+Offsets]. - {Integer offset}[rdoc-ref:timezones.rdoc@Integer+Offsets]. - {Timezone object}[rdoc-ref:timezones.rdoc@Timezone+Objects]. +- {Timezone name}[rdoc-ref:timezones.rdoc@Timezone+Names]. === Hours/Minutes Offsets @@ -102,3 +103,29 @@ which will be called if defined: - Called when Marshal.dump(t) is invoked - Argument: none. - Returns: the string name of the timezone. + +=== Timezone Names + +If the class (the receiver of class methods, or the class of the receiver +of instance methods) has `find_timezone` singleton method, this method is +called to achieve the corresponding timezone object from a timezone name. + +For example, using {Timezone}[https://github.com/panthomakos/timezone]: + class TimeWithTimezone < Time + require 'timezone' + def self.find_timezone(z) = Timezone[z] + end + + TimeWithTimezone.now(in: "America/New_York") #=> 2023-12-25 00:00:00 -0500 + TimeWithTimezone.new("2023-12-25 America/New_York") #=> 2023-12-25 00:00:00 -0500 + +Or, using {TZInfo}[https://tzinfo.github.io]: + class TimeWithTZInfo < Time + require 'tzinfo' + def self.find_timezone(z) = TZInfo::Timezone.get(z) + end + + TimeWithTZInfo.now(in: "America/New_York") #=> 2023-12-25 00:00:00 -0500 + TimeWithTZInfo.new("2023-12-25 America/New_York") #=> 2023-12-25 00:00:00 -0500 + +You can define this method per subclasses, or on the toplevel `Time` class. From 1cba1b3992dcc7476a24e00e4207cfd38e37d8d4 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Fri, 24 Nov 2023 20:00:13 +0900 Subject: [PATCH 10/78] Check `windres` message using the found `windres` only if needed --- configure.ac | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index ec1d6adc493c94..0eb623e7f6adb8 100644 --- a/configure.ac +++ b/configure.ac @@ -506,13 +506,10 @@ AS_CASE(["$target_os"], ]) rb_cv_binary_elf=no : ${enable_shared=yes} + AS_IF([$WINDRES --version | grep LLVM > /dev/null], [USE_LLVM_WINDRES=yes], [USE_LLVM_WINDRES=no]) ], [hiuxmpp*], [AC_DEFINE(__HIUX_MPP__)]) # by TOYODA Eizi -USE_LLVM_WINDRES=no -windres_version=`windres --version | grep LLVM` -test -z "$windres_version" || USE_LLVM_WINDRES=yes - AC_PROG_LN_S AC_PROG_MAKE_SET AC_PROG_INSTALL From 406dafbb347cb81f6d256d18ac9fd2908a9cb8ef Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Fri, 24 Nov 2023 11:36:55 +0000 Subject: [PATCH 11/78] [PRISM] Insert Tracepoint line events on line change --- prism_compile.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/prism_compile.c b/prism_compile.c index 840f9802530b24..6850c84c04fd9c 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -1490,6 +1490,17 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, int lineno = (int)pm_newline_list_line_column(&newline_list, node->location.start).line; NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + if (node->flags & PM_NODE_FLAG_NEWLINE && + ISEQ_COMPILE_DATA(iseq)->last_line != lineno) { + int event = RUBY_EVENT_LINE; + + ISEQ_COMPILE_DATA(iseq)->last_line = lineno; + if (ISEQ_COVERAGE(iseq) && ISEQ_LINE_COVERAGE(iseq)) { + event |= RUBY_EVENT_COVERAGE_LINE; + } + ADD_TRACE(ret, event); + } + switch (PM_NODE_TYPE(node)) { case PM_ALIAS_GLOBAL_VARIABLE_NODE: { pm_alias_global_variable_node_t *alias_node = (pm_alias_global_variable_node_t *) node; From 99e1f7b60717bc5ca3b160f10f4a8fe1521cba7c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 23 Nov 2023 14:53:53 -0500 Subject: [PATCH 12/78] Abort GC on shutdown On large Ruby applications, shutdown may be slow if a major GC has just started because rb_objspace_call_finalizer completes the GC. This commit adds gc_abort which discards the mark stack if during incremental marking and stops sweeping if during lazy sweeping. --- gc.c | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/gc.c b/gc.c index 3f992604b2a2a2..d1c5efb7804e75 100644 --- a/gc.c +++ b/gc.c @@ -1302,7 +1302,7 @@ total_freed_objects(rb_objspace_t *objspace) } #define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode) -#define gc_mode_set(objspace, mode) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(mode)) +#define gc_mode_set(objspace, m) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(m)) #define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking) #define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping) @@ -4511,6 +4511,36 @@ gc_finalize_deferred_register(rb_objspace_t *objspace) } } +static int pop_mark_stack(mark_stack_t *stack, VALUE *data); + +static void +gc_abort(rb_objspace_t *objspace) +{ + if (is_incremental_marking(objspace)) { + /* Remove all objects from the mark stack. */ + VALUE obj; + while (pop_mark_stack(&objspace->mark_stack, &obj)); + + objspace->flags.during_incremental_marking = FALSE; + } + + if (is_lazy_sweeping(objspace)) { + for (int i = 0; i < SIZE_POOL_COUNT; i++) { + rb_size_pool_t *size_pool = &size_pools[i]; + rb_heap_t *heap = SIZE_POOL_EDEN_HEAP(size_pool); + + heap->sweeping_page = NULL; + struct heap_page *page = NULL; + + ccan_list_for_each(&heap->pages, page, page_node) { + page->flags.before_sweep = false; + } + } + } + + gc_mode_set(objspace, gc_mode_none); +} + struct force_finalize_list { VALUE obj; VALUE table; @@ -4539,15 +4569,12 @@ rb_objspace_call_finalizer(rb_objspace_t *objspace) #if RGENGC_CHECK_MODE >= 2 gc_verify_internal_consistency(objspace); #endif - gc_rest(objspace); - if (ATOMIC_EXCHANGE(finalizing, 1)) return; /* run finalizers */ finalize_deferred(objspace); GC_ASSERT(heap_pages_deferred_final == 0); - gc_rest(objspace); /* prohibit incremental GC */ objspace->flags.dont_incremental = 1; @@ -4565,6 +4592,9 @@ rb_objspace_call_finalizer(rb_objspace_t *objspace) } } + /* Abort incremental marking and lazy sweeping to speed up shutdown. */ + gc_abort(objspace); + /* prohibit GC because force T_DATA finalizers can break an object graph consistency */ dont_gc_on(); From e201b81f79828c30500947fe8c8ea3c515e3d112 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Sat, 11 Nov 2023 16:12:12 +1100 Subject: [PATCH 13/78] Mark cc->cme_ for refinement callcaches as well This is required for the same reason that super CC needs it. See 36023d5cb751d62fca0c27901c07527b20170f4d. Reproducer: def cached_foo_callsite(obj) = obj.foo class Foo def foo = :v1 module R refine Foo do def foo = :unused end end end obj = Foo.new cached_foo_callsite(obj) # set up cc with cme for foo=:v1 class Foo def foo = :v2 end GC.start # cme for foo=:v1 collected, if not reachable by cached_foo_callsite cached_foo_callsite(obj) [Bug #19994] --- gc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gc.c b/gc.c index d1c5efb7804e75..260b891b1e36ac 100644 --- a/gc.c +++ b/gc.c @@ -7216,12 +7216,13 @@ gc_mark_imemo(rb_objspace_t *objspace, VALUE obj) * - On the multi-Ractors, cme will be collected with global GC * so that it is safe if GC is not interleaving while accessing * cc and cme. - * - However, cc_type_super is not chained from cc so the cc->cme - * should be marked. + * - However, cc_type_super and cc_type_refinement are not chained + * from ccs so cc->cme should be marked; the cme might be + * reachable only through cc in these cases. */ { const struct rb_callcache *cc = (const struct rb_callcache *)obj; - if (vm_cc_super_p(cc)) { + if (vm_cc_super_p(cc) || vm_cc_refinement_p(cc)) { gc_mark(objspace, (VALUE)cc->cme_); } } From 269c705f93c4db631f4cad89991bc5d69a7dcd03 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 24 Nov 2023 10:25:06 -0500 Subject: [PATCH 14/78] Fix compaction for generic ivars When generic instance variable has a shape, it is marked movable. If it it transitions to too complex, it needs to update references otherwise it may have incorrect references. --- gc.c | 10 ++++++++-- internal/gc.h | 2 ++ internal/variable.h | 3 ++- variable.c | 23 ++++++++++++++++++++--- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/gc.c b/gc.c index 260b891b1e36ac..2d7eb4de93af65 100644 --- a/gc.c +++ b/gc.c @@ -7255,7 +7255,7 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj) gc_mark_set_parent(objspace, obj); if (FL_TEST(obj, FL_EXIVAR)) { - rb_mark_and_update_generic_ivar(obj); + rb_mark_generic_ivar(obj); } switch (BUILTIN_TYPE(obj)) { @@ -10249,6 +10249,12 @@ gc_ref_update_table_values_only(rb_objspace_t *objspace, st_table *tbl) } } +void +rb_gc_ref_update_table_values_only(st_table *tbl) +{ + gc_ref_update_table_values_only(&rb_objspace, tbl); +} + static void gc_update_table_refs(rb_objspace_t * objspace, st_table *tbl) { @@ -10623,7 +10629,7 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj) gc_report(4, objspace, "update-refs: %p ->\n", (void *)obj); if (FL_TEST(obj, FL_EXIVAR)) { - rb_mark_and_update_generic_ivar(obj); + rb_ref_update_generic_ivar(obj); } switch (BUILTIN_TYPE(obj)) { diff --git a/internal/gc.h b/internal/gc.h index 188497b0078fce..34a6043e8a1f6c 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -250,6 +250,8 @@ void rb_gc_mark_and_move(VALUE *ptr); void rb_gc_mark_weak(VALUE *ptr); void rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr); +void rb_gc_ref_update_table_values_only(st_table *tbl); + #define rb_gc_mark_and_move_ptr(ptr) do { \ VALUE _obj = (VALUE)*(ptr); \ rb_gc_mark_and_move(&_obj); \ diff --git a/internal/variable.h b/internal/variable.h index 63b074a30884d6..b2a30c7c583700 100644 --- a/internal/variable.h +++ b/internal/variable.h @@ -53,7 +53,8 @@ void rb_evict_ivars_to_hash(VALUE obj); RUBY_SYMBOL_EXPORT_BEGIN /* variable.c (export) */ -void rb_mark_and_update_generic_ivar(VALUE); +void rb_mark_generic_ivar(VALUE obj); +void rb_ref_update_generic_ivar(VALUE); void rb_mv_generic_ivar(VALUE src, VALUE dst); VALUE rb_const_missing(VALUE klass, VALUE name); int rb_class_ivar_set(VALUE klass, ID vid, VALUE value); diff --git a/variable.c b/variable.c index 5eeee636bb0fdc..056572cc3680ad 100644 --- a/variable.c +++ b/variable.c @@ -1066,17 +1066,34 @@ gen_ivtbl_resize(struct gen_ivtbl *old, uint32_t n) } void -rb_mark_and_update_generic_ivar(VALUE obj) +rb_mark_generic_ivar(VALUE obj) { struct gen_ivtbl *ivtbl; if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) { if (rb_shape_obj_too_complex(obj)) { - rb_mark_tbl(ivtbl->as.complex.table); + rb_mark_tbl_no_pin(ivtbl->as.complex.table); } else { for (uint32_t i = 0; i < ivtbl->as.shape.numiv; i++) { - rb_gc_mark_and_move(&ivtbl->as.shape.ivptr[i]); + rb_gc_mark_movable(ivtbl->as.shape.ivptr[i]); + } + } + } +} + +void +rb_ref_update_generic_ivar(VALUE obj) +{ + struct gen_ivtbl *ivtbl; + + if (rb_gen_ivtbl_get(obj, 0, &ivtbl)) { + if (rb_shape_obj_too_complex(obj)) { + rb_gc_ref_update_table_values_only(ivtbl->as.complex.table); + } + else { + for (uint32_t i = 0; i < ivtbl->as.shape.numiv; i++) { + ivtbl->as.shape.ivptr[i] = rb_gc_location(ivtbl->as.shape.ivptr[i]); } } } From 974d18fd0c13bd19120cad70187f5b646c901dff Mon Sep 17 00:00:00 2001 From: Jeremy Evans Date: Mon, 8 Mar 2021 15:28:04 -0800 Subject: [PATCH 15/78] [ruby/resolv] Fix the fallback from UDP to TCP due to message truncation If truncation is detected, return immediately from decode so that the UDP connection can be retried with TCP, instead of failing to decode due to trying to decode a truncated response. Fixes [Bug #13513] https://github.com/ruby/resolv/commit/0de996dbca --- lib/resolv.rb | 6 +- test/resolv/test_dns.rb | 172 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 2 deletions(-) diff --git a/lib/resolv.rb b/lib/resolv.rb index 0c96ee80b90467..8203e5110a8b8f 100644 --- a/lib/resolv.rb +++ b/lib/resolv.rb @@ -1520,13 +1520,15 @@ def Message.decode(m) id, flag, qdcount, ancount, nscount, arcount = msg.get_unpack('nnnnnn') o.id = id + o.tc = (flag >> 9) & 1 + o.rcode = flag & 15 + return o unless o.tc.zero? + o.qr = (flag >> 15) & 1 o.opcode = (flag >> 11) & 15 o.aa = (flag >> 10) & 1 - o.tc = (flag >> 9) & 1 o.rd = (flag >> 8) & 1 o.ra = (flag >> 7) & 1 - o.rcode = flag & 15 (1..qdcount).each { name, typeclass = msg.get_question o.add_question(name, typeclass) diff --git a/test/resolv/test_dns.rb b/test/resolv/test_dns.rb index d9db8408fd1ff7..c0400b4bb888fe 100644 --- a/test/resolv/test_dns.rb +++ b/test/resolv/test_dns.rb @@ -44,6 +44,16 @@ def teardown BasicSocket.do_not_reverse_lookup = @save_do_not_reverse_lookup end + def with_tcp(host, port) + t = TCPServer.new(host, port) + begin + t.listen(1) + yield t + ensure + t.close + end + end + def with_udp(host, port) u = UDPSocket.new begin @@ -157,6 +167,168 @@ def test_query_ipv4_address } end + def test_query_ipv4_address_truncated_tcp_fallback + begin + OpenSSL + rescue LoadError + skip 'autoload problem. see [ruby-dev:45021][Bug #5786]' + end if defined?(OpenSSL) + + num_records = 50 + + with_udp('127.0.0.1', 0) {|u| + _, server_port, _, server_address = u.addr + with_tcp('127.0.0.1', server_port) {|t| + client_thread = Thread.new { + Resolv::DNS.open(:nameserver_port => [[server_address, server_port]]) {|dns| + dns.getresources("foo.example.org", Resolv::DNS::Resource::IN::A) + } + } + udp_server_thread = Thread.new { + msg, (_, client_port, _, client_address) = Timeout.timeout(5) {u.recvfrom(4096)} + id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") + qr = (word2 & 0x8000) >> 15 + opcode = (word2 & 0x7800) >> 11 + aa = (word2 & 0x0400) >> 10 + tc = (word2 & 0x0200) >> 9 + rd = (word2 & 0x0100) >> 8 + ra = (word2 & 0x0080) >> 7 + z = (word2 & 0x0070) >> 4 + rcode = word2 & 0x000f + rest = msg[12..-1] + assert_equal(0, qr) # 0:query 1:response + assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS + assert_equal(0, aa) # Authoritative Answer + assert_equal(0, tc) # TrunCation + assert_equal(1, rd) # Recursion Desired + assert_equal(0, ra) # Recursion Available + assert_equal(0, z) # Reserved for future use + assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused + assert_equal(1, qdcount) # number of entries in the question section. + assert_equal(0, ancount) # number of entries in the answer section. + assert_equal(0, nscount) # number of entries in the authority records section. + assert_equal(0, arcount) # number of entries in the additional records section. + name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") + assert_operator(rest, :start_with?, name) + rest = rest[name.length..-1] + assert_equal(4, rest.length) + qtype, _ = rest.unpack("nn") + assert_equal(1, qtype) # A + assert_equal(1, qtype) # IN + id = id + qr = 1 + opcode = opcode + aa = 0 + tc = 1 + rd = rd + ra = 1 + z = 0 + rcode = 0 + qdcount = 0 + ancount = num_records + nscount = 0 + arcount = 0 + word2 = (qr << 15) | + (opcode << 11) | + (aa << 10) | + (tc << 9) | + (rd << 8) | + (ra << 7) | + (z << 4) | + rcode + msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") + type = 1 + klass = 1 + ttl = 3600 + rdlength = 4 + num_records.times do |i| + rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 + rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") + msg << rr + end + u.send(msg[0...512], 0, client_address, client_port) + } + tcp_server_thread = Thread.new { + ct = t.accept + msg = ct.recv(512) + msg.slice!(0..1) # Size (only for TCP) + id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") + qr = (word2 & 0x8000) >> 15 + opcode = (word2 & 0x7800) >> 11 + aa = (word2 & 0x0400) >> 10 + tc = (word2 & 0x0200) >> 9 + rd = (word2 & 0x0100) >> 8 + ra = (word2 & 0x0080) >> 7 + z = (word2 & 0x0070) >> 4 + rcode = word2 & 0x000f + rest = msg[12..-1] + assert_equal(0, qr) # 0:query 1:response + assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS + assert_equal(0, aa) # Authoritative Answer + assert_equal(0, tc) # TrunCation + assert_equal(1, rd) # Recursion Desired + assert_equal(0, ra) # Recursion Available + assert_equal(0, z) # Reserved for future use + assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused + assert_equal(1, qdcount) # number of entries in the question section. + assert_equal(0, ancount) # number of entries in the answer section. + assert_equal(0, nscount) # number of entries in the authority records section. + assert_equal(0, arcount) # number of entries in the additional records section. + name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") + assert_operator(rest, :start_with?, name) + rest = rest[name.length..-1] + assert_equal(4, rest.length) + qtype, _ = rest.unpack("nn") + assert_equal(1, qtype) # A + assert_equal(1, qtype) # IN + id = id + qr = 1 + opcode = opcode + aa = 0 + tc = 0 + rd = rd + ra = 1 + z = 0 + rcode = 0 + qdcount = 0 + ancount = num_records + nscount = 0 + arcount = 0 + word2 = (qr << 15) | + (opcode << 11) | + (aa << 10) | + (tc << 9) | + (rd << 8) | + (ra << 7) | + (z << 4) | + rcode + msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") + type = 1 + klass = 1 + ttl = 3600 + rdlength = 4 + num_records.times do |i| + rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 + rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") + msg << rr + end + msg = "#{[msg.bytesize].pack("n")}#{msg}" # Prefix with size + ct.send(msg, 0) + ct.close + } + result, _ = assert_join_threads([client_thread, udp_server_thread, tcp_server_thread]) + assert_instance_of(Array, result) + assert_equal(50, result.length) + result.each_with_index do |rr, i| + assert_instance_of(Resolv::DNS::Resource::IN::A, rr) + assert_instance_of(Resolv::IPv4, rr.address) + assert_equal("192.0.2.#{i}", rr.address.to_s) + assert_equal(3600, rr.ttl) + end + } + } + end + def test_query_ipv4_duplicate_responses begin OpenSSL From fb7add495454322ea00efa7549feb957cb1ca538 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 24 Nov 2023 14:48:02 -0500 Subject: [PATCH 16/78] Switch shape test to use exhaust_shapes --- test/ruby/test_shapes.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/test/ruby/test_shapes.rb b/test/ruby/test_shapes.rb index 7919a210a78b65..c4b0d251f1eace 100644 --- a/test/ruby/test_shapes.rb +++ b/test/ruby/test_shapes.rb @@ -132,17 +132,12 @@ def test_too_many_ivs_on_obj begin; class Hi; end - obj = Hi.new - i = 0 - while RubyVM::Shape.shapes_available > 2 - obj.instance_variable_set(:"@a#{i}", 1) - i += 1 - end + RubyVM::Shape.exhaust_shapes(2) obj = Hi.new - obj.instance_variable_set(:"@b", 1) - obj.instance_variable_set(:"@c", 1) - obj.instance_variable_set(:"@d", 1) + obj.instance_variable_set(:@b, 1) + obj.instance_variable_set(:@c, 1) + obj.instance_variable_set(:@d, 1) assert_predicate RubyVM::Shape.of(obj), :too_complex? end; From 7276d4b4e87bfdc9b609f481a734e39c499de253 Mon Sep 17 00:00:00 2001 From: Jeremy Evans Date: Fri, 17 Sep 2021 17:17:20 -0700 Subject: [PATCH 17/78] [ruby/resolv] Support a :raise_timeout_errors option to raise timeouts as Resolv::ResolvError This allows to differentiate a timeout from an NXDOMAIN response. Fixes [Bug #18151] https://github.com/ruby/resolv/commit/c0e5abab76 --- lib/resolv.rb | 6 ++++++ test/resolv/test_dns.rb | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/lib/resolv.rb b/lib/resolv.rb index 8203e5110a8b8f..0943edc40086b4 100644 --- a/lib/resolv.rb +++ b/lib/resolv.rb @@ -312,6 +312,8 @@ def self.open(*args) # String:: Path to a file using /etc/resolv.conf's format. # Hash:: Must contain :nameserver, :search and :ndots keys. # :nameserver_port can be used to specify port number of nameserver address. + # :raise_timeout_errors can be used to raise timeout errors + # as exceptions instead of treating the same as an NXDOMAIN response. # # The value of :nameserver should be an address string or # an array of address strings. @@ -1032,6 +1034,7 @@ def lazy_initialize end @search = config_hash[:search] if config_hash.include? :search @ndots = config_hash[:ndots] if config_hash.include? :ndots + @raise_timeout_errors = config_hash[:raise_timeout_errors] if @nameserver_port.empty? @nameserver_port << ['0.0.0.0', Port] @@ -1118,6 +1121,7 @@ def generate_timeouts def resolv(name) candidates = generate_candidates(name) timeouts = @timeouts || generate_timeouts + timeout_error = false begin candidates.each {|candidate| begin @@ -1129,11 +1133,13 @@ def resolv(name) end } } + timeout_error = true raise ResolvError.new("DNS resolv timeout: #{name}") rescue NXDomain end } rescue ResolvError + raise if @raise_timeout_errors && timeout_error end end diff --git a/test/resolv/test_dns.rb b/test/resolv/test_dns.rb index c0400b4bb888fe..63ab66657c9c8f 100644 --- a/test/resolv/test_dns.rb +++ b/test/resolv/test_dns.rb @@ -630,4 +630,28 @@ def dns.each_resource(name, typeclass) end assert_raise(Resolv::ResolvError) { dns.each_name('example.com') } end + + def test_unreachable_server + unreachable_ip = '127.0.0.1' + sock = UDPSocket.new + sock.connect(unreachable_ip, 53) + begin + sock.send('1', 0) + rescue Errno::ENETUNREACH, Errno::EHOSTUNREACH + else + omit('cannot test unreachable server, as IP used is reachable') + end + + config = { + :nameserver => [unreachable_ip], + :search => ['lan'], + :ndots => 1 + } + r = Resolv.new([Resolv::DNS.new(config)]) + assert_equal([], r.getaddresses('www.google.com')) + + config[:raise_timeout_errors] = true + r = Resolv.new([Resolv::DNS.new(config)]) + assert_raise(Resolv::ResolvError) { r.getaddresses('www.google.com') } + end end From de37b780508d4ced5837f39446d3c8c93ef9ec7c Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 24 Nov 2023 21:43:29 -0800 Subject: [PATCH 18/78] Omit a broken https test on MinGW This started to reliably fail on MinGW at an irrelevant commit: https://github.com/ruby/ruby/actions/runs/6981002841/job/18997302124 https://github.com/ruby/ruby/actions/runs/6981946473/job/19000104223 https://github.com/ruby/ruby/actions/runs/6983823136/job/19005613809 https://github.com/ruby/ruby/actions/runs/6983912116/job/19005844596 https://github.com/ruby/ruby/actions/runs/6984215921/job/19006649495 https://github.com/ruby/ruby/actions/runs/6984383103/job/19007100446 https://github.com/ruby/ruby/actions/runs/6986489509/job/19012000642 So this failure is not detecting a new bug. Let's skip this until we fix this test for MinGW. --- test/net/http/test_https.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/net/http/test_https.rb b/test/net/http/test_https.rb index 2fb895a8ae4931..5f9e066b6bedbf 100644 --- a/test/net/http/test_https.rb +++ b/test/net/http/test_https.rb @@ -167,6 +167,7 @@ def test_session_reuse def test_session_reuse_but_expire # FIXME: The new_session_cb is known broken for clients in OpenSSL 1.1.0h. omit if OpenSSL::OPENSSL_LIBRARY_VERSION.include?('OpenSSL 1.1.0h') + omit 'failing on MinGW' if /mingw/ =~ RUBY_PLATFORM http = Net::HTTP.new(HOST, config("port")) http.use_ssl = true From 84f45c6ed5357de206153b1524620335814d05d9 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 24 Nov 2023 22:28:39 -0800 Subject: [PATCH 19/78] Place continue-on-error consistently Once it fails on "Perform CodeQL Analysis", it proceeds to subsequent steps and fails because required files are not created by previous steps. When we have a continue-on-error, all subsequent steps that rely on the step should have a continue-on-error as well. --- .github/workflows/codeql-analysis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4c6e9fd63902f5..56cfd8abce4b39 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -104,8 +104,10 @@ jobs: input: sarif-results/${{ matrix.language }}.sarif output: sarif-results/${{ matrix.language }}.sarif if: ${{ matrix.language == 'ruby' }} + continue-on-error: true - name: Upload SARIF uses: github/codeql-action/upload-sarif@cdcdbb579706841c47f7063dda365e292e5cad7a # v2.13.4 with: sarif_file: sarif-results/${{ matrix.language }}.sarif + continue-on-error: true From 3140886b759b3ca5602b0fc2ce85d6022133146b Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Mon, 20 Nov 2023 13:18:55 +0100 Subject: [PATCH 20/78] [ruby/openssl] History.md: Escape Markdown syntax Italic "*". [ci skip] https://github.com/ruby/openssl/commit/dc26433ae5 --- ext/openssl/History.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/openssl/History.md b/ext/openssl/History.md index bd7b1ec1b1f9cb..3249f6617adadf 100644 --- a/ext/openssl/History.md +++ b/ext/openssl/History.md @@ -457,7 +457,7 @@ Security fixes Bug fixes --------- -* Fixed OpenSSL::PKey::*.{new,generate} immediately aborting if the thread is +* Fixed OpenSSL::PKey::\*.{new,generate} immediately aborting if the thread is interrupted. [[Bug #14882]](https://bugs.ruby-lang.org/issues/14882) [[GitHub #205]](https://github.com/ruby/openssl/pull/205) From 543dd74049f18db2f8dd9ac05b25f8dbff2edc14 Mon Sep 17 00:00:00 2001 From: Jun Aruga Date: Tue, 7 Nov 2023 14:36:17 +0100 Subject: [PATCH 21/78] Fix test_pkey_dh.rb in FIPS. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We use dh2048_ffdhe2048.pem file (DH 2048 bits) instead of dh1024.pem file in both non-FIPS and FIPS cases. Because the following command fails to generate the pem file with 1024 bits. And the OpenSSL FIPS 140-2 security policy document explains the DH public keys are allowed from 2048 bits.[1] ``` $ OPENSSL_CONF=/home/jaruga/.local/openssl-3.3.0-dev-fips-debug-1aa08644ec/ssl/openssl_fips.cnf \ /home/jaruga/.local/openssl-3.3.0-dev-fips-debug-1aa08644ec/bin/openssl \ dhparam -out dh1024.pem 1024 Generating DH parameters, 1024 bit long safe prime dhparam: Generating DH key parameters failed ``` The dh2048_ffdhe2048.pem file was created by the following command with the OpenSSL FIPS configuration file. The logic to generate the DH pem file is different between non-FIPS and FIPS cases. In FIPS, it seems that the command always returns the text defined as ffdhe2048 in the FFDHE groups in RFC 7919 unlike non-FIPS.[2] As the generated pem file is a normal and valid PKCS#3-style group parameter, we use the file for the non-FIPS case too. ``` $ OPENSSL_CONF=/home/jaruga/.local/openssl-3.3.0-dev-fips-debug-1aa08644ec/ssl/openssl_fips.cnf \ /home/jaruga/.local/openssl-3.3.0-dev-fips-debug-1aa08644ec/bin/openssl \ dhparam -out dh2048_ffdhe2048.pem 2048 ``` Note that the hard-coded PEM-encoded string in the `test_DHparams` is intentional to avoid modifying the content unintentionally. * [1] https://www.openssl.org/source/ - OpenSSL 3.0.8 FIPS 140-2 security policy document page 25, Table 10 – Public Keys - DH Public - DH (2048/3072/4096/6144/8192) public key agreement key * [2] RFC7919 - Appendix A.1: ffdhe2048 https://www.rfc-editor.org/rfc/rfc7919#appendix-A.1 --- test/openssl/fixtures/pkey/dh1024.pem | 5 -- .../fixtures/pkey/dh2048_ffdhe2048.pem | 8 +++ test/openssl/test_pkey_dh.rb | 64 +++++++++++++------ test/openssl/utils.rb | 6 +- 4 files changed, 57 insertions(+), 26 deletions(-) delete mode 100644 test/openssl/fixtures/pkey/dh1024.pem create mode 100644 test/openssl/fixtures/pkey/dh2048_ffdhe2048.pem diff --git a/test/openssl/fixtures/pkey/dh1024.pem b/test/openssl/fixtures/pkey/dh1024.pem deleted file mode 100644 index f99c757f21a419..00000000000000 --- a/test/openssl/fixtures/pkey/dh1024.pem +++ /dev/null @@ -1,5 +0,0 @@ ------BEGIN DH PARAMETERS----- -MIGHAoGBAKnKQ8MNK6nYZzLrrcuTsLxuiJGXoOO5gT+tljOTbHBuiktdMTITzIY0 -pFxIvjG05D7HoBZQfrR0c92NGWPkAiCkhQKB8JCbPVzwNLDy6DZ0pmofDKrEsYHG -AQjjxMXhwULlmuR/K+WwlaZPiLIBYalLAZQ7ZbOPeVkJ8ePao0eLAgEC ------END DH PARAMETERS----- diff --git a/test/openssl/fixtures/pkey/dh2048_ffdhe2048.pem b/test/openssl/fixtures/pkey/dh2048_ffdhe2048.pem new file mode 100644 index 00000000000000..9b182b7201fd94 --- /dev/null +++ b/test/openssl/fixtures/pkey/dh2048_ffdhe2048.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== +-----END DH PARAMETERS----- diff --git a/test/openssl/test_pkey_dh.rb b/test/openssl/test_pkey_dh.rb index 161af1897bd1c4..d32ffaf6b111ae 100644 --- a/test/openssl/test_pkey_dh.rb +++ b/test/openssl/test_pkey_dh.rb @@ -18,15 +18,26 @@ def test_new_generate assert_key(dh) end if ENV["OSSL_TEST_ALL"] - def test_new_break + def test_new_break_on_non_fips + omit_on_fips + assert_nil(OpenSSL::PKey::DH.new(NEW_KEYLEN) { break }) assert_raise(RuntimeError) do OpenSSL::PKey::DH.new(NEW_KEYLEN) { raise } end end + def test_new_break_on_fips + omit_on_non_fips + + # The block argument is not executed in FIPS case. + # See https://github.com/ruby/openssl/issues/692 for details. + assert(OpenSSL::PKey::DH.new(NEW_KEYLEN) { break }) + assert(OpenSSL::PKey::DH.new(NEW_KEYLEN) { raise }) + end + def test_derive_key - params = Fixtures.pkey("dh1024") + params = Fixtures.pkey("dh2048_ffdhe2048") dh1 = OpenSSL::PKey.generate_key(params) dh2 = OpenSSL::PKey.generate_key(params) dh1_pub = OpenSSL::PKey.read(dh1.public_to_der) @@ -44,34 +55,38 @@ def test_derive_key end def test_DHparams - dh1024 = Fixtures.pkey("dh1024") - dh1024params = dh1024.public_key + dh = Fixtures.pkey("dh2048_ffdhe2048") + dh_params = dh.public_key asn1 = OpenSSL::ASN1::Sequence([ - OpenSSL::ASN1::Integer(dh1024.p), - OpenSSL::ASN1::Integer(dh1024.g) + OpenSSL::ASN1::Integer(dh.p), + OpenSSL::ASN1::Integer(dh.g) ]) key = OpenSSL::PKey::DH.new(asn1.to_der) - assert_same_dh dh1024params, key + assert_same_dh dh_params, key pem = <<~EOF -----BEGIN DH PARAMETERS----- - MIGHAoGBAKnKQ8MNK6nYZzLrrcuTsLxuiJGXoOO5gT+tljOTbHBuiktdMTITzIY0 - pFxIvjG05D7HoBZQfrR0c92NGWPkAiCkhQKB8JCbPVzwNLDy6DZ0pmofDKrEsYHG - AQjjxMXhwULlmuR/K+WwlaZPiLIBYalLAZQ7ZbOPeVkJ8ePao0eLAgEC + MIIBCAKCAQEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz + +8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a + 87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 + YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi + 7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD + ssbzSibBsu/6iGtCOGEoXJf//////////wIBAg== -----END DH PARAMETERS----- EOF + key = OpenSSL::PKey::DH.new(pem) - assert_same_dh dh1024params, key + assert_same_dh dh_params, key key = OpenSSL::PKey.read(pem) - assert_same_dh dh1024params, key + assert_same_dh dh_params, key - assert_equal asn1.to_der, dh1024.to_der - assert_equal pem, dh1024.export + assert_equal asn1.to_der, dh.to_der + assert_equal pem, dh.export end def test_public_key - dh = Fixtures.pkey("dh1024") + dh = Fixtures.pkey("dh2048_ffdhe2048") public_key = dh.public_key assert_no_key(public_key) #implies public_key.public? is false! assert_equal(dh.to_der, public_key.to_der) @@ -80,7 +95,8 @@ def test_public_key def test_generate_key # Deprecated in v3.0.0; incompatible with OpenSSL 3.0 - dh = Fixtures.pkey("dh1024").public_key # creates a copy with params only + # Creates a copy with params only + dh = Fixtures.pkey("dh2048_ffdhe2048").public_key assert_no_key(dh) dh.generate_key! assert_key(dh) @@ -91,7 +107,15 @@ def test_generate_key end if !openssl?(3, 0, 0) def test_params_ok? - dh0 = Fixtures.pkey("dh1024") + # Skip the tests in old OpenSSL version 1.1.1c or early versions before + # applying the following commits in OpenSSL 1.1.1d to make `DH_check` + # function pass the RFC 7919 FFDHE group texts. + # https://github.com/openssl/openssl/pull/9435 + unless openssl?(1, 1, 1, 4) + pend 'DH check for RFC 7919 FFDHE group texts is not implemented' + end + + dh0 = Fixtures.pkey("dh2048_ffdhe2048") dh1 = OpenSSL::PKey::DH.new(OpenSSL::ASN1::Sequence([ OpenSSL::ASN1::Integer(dh0.p), @@ -108,7 +132,7 @@ def test_params_ok? def test_dup # Parameters only - dh1 = Fixtures.pkey("dh1024") + dh1 = Fixtures.pkey("dh2048_ffdhe2048") dh2 = dh1.dup assert_equal dh1.to_der, dh2.to_der assert_not_equal nil, dh1.p @@ -125,7 +149,7 @@ def test_dup end # With a key pair - dh3 = OpenSSL::PKey.generate_key(Fixtures.pkey("dh1024")) + dh3 = OpenSSL::PKey.generate_key(Fixtures.pkey("dh2048_ffdhe2048")) dh4 = dh3.dup assert_equal dh3.to_der, dh4.to_der assert_equal dh1.to_der, dh4.to_der # encodes parameters only @@ -136,7 +160,7 @@ def test_dup end def test_marshal - dh = Fixtures.pkey("dh1024") + dh = Fixtures.pkey("dh2048_ffdhe2048") deserialized = Marshal.load(Marshal.dump(dh)) assert_equal dh.to_der, deserialized.to_der diff --git a/test/openssl/utils.rb b/test/openssl/utils.rb index cd70d4886f614e..f6c84eef67c291 100644 --- a/test/openssl/utils.rb +++ b/test/openssl/utils.rb @@ -151,7 +151,11 @@ def teardown def omit_on_fips return unless OpenSSL.fips_mode - omit 'An encryption used in the test is not FIPS-approved' + omit <<~MESSAGE + Only for OpenSSL non-FIPS with the following possible reasons: + * A testing logic is non-FIPS specific. + * An encryption used in the test is not FIPS-approved. + MESSAGE end def omit_on_non_fips From 68a03613d8f7ec173addbfbb3989045d8f639ec5 Mon Sep 17 00:00:00 2001 From: hogelog Date: Sat, 25 Nov 2023 19:14:03 +0900 Subject: [PATCH 22/78] [ruby/irb] Fix flaky test case test_autocomplete_with_multiple_doc_namespaces (https://github.com/ruby/irb/pull/786) https://github.com/ruby/irb/commit/85c6ddeb7d --- test/irb/yamatanooroti/test_rendering.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index 5d8719ac3ff81e..f9b4befbf5c8dc 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -207,15 +207,13 @@ def test_autocomplete_with_multiple_doc_namespaces write_irbrc <<~'LINES' puts 'start IRB' LINES - start_terminal(4, 40, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb}, startup_message: 'start IRB') + start_terminal(3, 50, %W{ruby -I#{@pwd}/lib #{@pwd}/exe/irb}, startup_message: 'start IRB') write("{}.__id_") write("\C-i") close - assert_screen(<<~EOC) - start IRB - irb(main):001> {}.__id__ - }.__id__ - EOC + screen = result.join("\n").sub(/\n*\z/, "\n") + # This assertion passes whether showdoc dialog completed or not. + assert_match(/start\ IRB\nirb\(main\):001> {}\.__id__\n }\.__id__(?:Press )?/, screen) end def test_autocomplete_with_showdoc_in_gaps_on_narrow_screen_right From f6b292b5ca8b397b00cc4e82d8ae7ede8b09f25f Mon Sep 17 00:00:00 2001 From: tomoya ishida Date: Sat, 25 Nov 2023 19:15:58 +0900 Subject: [PATCH 23/78] [ruby/irb] Fix exception(backtrace=nil) prints nothing (https://github.com/ruby/irb/pull/782) https://github.com/ruby/irb/commit/fa9ecf9a5b --- lib/irb.rb | 67 +++++++++++++++----------------- test/irb/test_raise_exception.rb | 7 +--- 2 files changed, 34 insertions(+), 40 deletions(-) diff --git a/lib/irb.rb b/lib/irb.rb index 75476542571deb..8c3039482e41d3 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -697,7 +697,7 @@ def encode_with_invalid_byte_sequence(str, enc) end def handle_exception(exc) - if exc.backtrace && exc.backtrace[0] =~ /\/irb(2)?(\/.*|-.*|\.rb)?:/ && exc.class.to_s !~ /^IRB/ && + if exc.backtrace[0] =~ /\/irb(2)?(\/.*|-.*|\.rb)?:/ && exc.class.to_s !~ /^IRB/ && !(SyntaxError === exc) && !(EncodingError === exc) # The backtrace of invalid encoding hash (ex. {"\xAE": 1}) raises EncodingError without lineno. irb_bug = true @@ -705,44 +705,41 @@ def handle_exception(exc) irb_bug = false end - if exc.backtrace - order = nil - if RUBY_VERSION < '3.0.0' - if STDOUT.tty? - message = exc.full_message(order: :bottom) - order = :bottom - else - message = exc.full_message(order: :top) - order = :top - end - else # '3.0.0' <= RUBY_VERSION + if RUBY_VERSION < '3.0.0' + if STDOUT.tty? + message = exc.full_message(order: :bottom) + order = :bottom + else message = exc.full_message(order: :top) order = :top end - message = convert_invalid_byte_sequence(message, exc.message.encoding) - message = encode_with_invalid_byte_sequence(message, IRB.conf[:LC_MESSAGES].encoding) unless message.encoding.to_s.casecmp?(IRB.conf[:LC_MESSAGES].encoding.to_s) - message = message.gsub(/((?:^\t.+$\n)+)/) { |m| - case order - when :top - lines = m.split("\n") - when :bottom - lines = m.split("\n").reverse - end - unless irb_bug - lines = lines.map { |l| @context.workspace.filter_backtrace(l) }.compact - if lines.size > @context.back_trace_limit - omit = lines.size - @context.back_trace_limit - lines = lines[0..(@context.back_trace_limit - 1)] - lines << "\t... %d levels..." % omit - end - end - lines = lines.reverse if order == :bottom - lines.map{ |l| l + "\n" }.join - } - # The "" in "(irb)" may be the top level of IRB so imitate the main object. - message = message.gsub(/\(irb\):(?\d+):in `<(?top \(required\))>'/) { "(irb):#{$~[:num]}:in `
'" } - puts message + else # '3.0.0' <= RUBY_VERSION + message = exc.full_message(order: :top) + order = :top end + message = convert_invalid_byte_sequence(message, exc.message.encoding) + message = encode_with_invalid_byte_sequence(message, IRB.conf[:LC_MESSAGES].encoding) unless message.encoding.to_s.casecmp?(IRB.conf[:LC_MESSAGES].encoding.to_s) + message = message.gsub(/((?:^\t.+$\n)+)/) { |m| + case order + when :top + lines = m.split("\n") + when :bottom + lines = m.split("\n").reverse + end + unless irb_bug + lines = lines.map { |l| @context.workspace.filter_backtrace(l) }.compact + if lines.size > @context.back_trace_limit + omit = lines.size - @context.back_trace_limit + lines = lines[0..(@context.back_trace_limit - 1)] + lines << "\t... %d levels..." % omit + end + end + lines = lines.reverse if order == :bottom + lines.map{ |l| l + "\n" }.join + } + # The "" in "(irb)" may be the top level of IRB so imitate the main object. + message = message.gsub(/\(irb\):(?\d+):in `<(?top \(required\))>'/) { "(irb):#{$~[:num]}:in `
'" } + puts message puts 'Maybe IRB bug!' if irb_bug rescue Exception => handler_exc begin diff --git a/test/irb/test_raise_exception.rb b/test/irb/test_raise_exception.rb index 9ca534dba179e7..c373dd7335ace1 100644 --- a/test/irb/test_raise_exception.rb +++ b/test/irb/test_raise_exception.rb @@ -7,11 +7,8 @@ module TestIRB class RaiseExceptionTest < TestCase def test_raise_exception_with_nil_backtrace bundle_exec = ENV.key?('BUNDLE_GEMFILE') ? ['-rbundler/setup'] : [] - assert_in_out_err(bundle_exec + %w[-rirb -W0 -e IRB.start(__FILE__) -- -f --], <<-IRB, /Exception: foo/, []) - e = Exception.new("foo") - puts e.inspect - def e.backtrace; nil; end - raise e + assert_in_out_err(bundle_exec + %w[-rirb -W0 -e IRB.start(__FILE__) -- -f --], <<-IRB, /#/, []) + raise Exception.new("foo").tap {|e| def e.backtrace; nil; end } IRB end From 564ef66e26454079188f024eb28e48a4ef1b2085 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 24 Nov 2023 14:31:36 -0500 Subject: [PATCH 24/78] Verify that duplicate shape is not created This adds an assertion that the instance variable does not already exist in the shape tree when creating a new shape. --- shape.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/shape.c b/shape.c index 6d6235dd3d546c..96cd2750af8e95 100644 --- a/shape.c +++ b/shape.c @@ -654,13 +654,20 @@ rb_shape_get_next_iv_shape(rb_shape_t* shape, ID id) } rb_shape_t * -rb_shape_get_next(rb_shape_t* shape, VALUE obj, ID id) +rb_shape_get_next(rb_shape_t *shape, VALUE obj, ID id) { RUBY_ASSERT(!is_instance_id(id) || RTEST(rb_sym2str(ID2SYM(id)))); if (UNLIKELY(shape->type == SHAPE_OBJ_TOO_COMPLEX)) { return shape; } +#if RUBY_DEBUG + attr_index_t index; + if (rb_shape_get_iv_index(shape, id, &index)) { + rb_bug("rb_shape_get_next: trying to create ivar that already exists at index %u", index); + } +#endif + bool allow_new_shape = true; if (BUILTIN_TYPE(obj) == T_OBJECT) { From b93a1bb40bf868ba8d664539ba4a44740e641772 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Fri, 24 Nov 2023 14:31:50 -0500 Subject: [PATCH 25/78] Verify correctness of shape cache This commit adds assertions to verify that the shape cache is correct compared to the shape tree. --- shape.c | 88 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/shape.c b/shape.c index 96cd2750af8e95..7406e74dbf5a48 100644 --- a/shape.c +++ b/shape.c @@ -759,50 +759,74 @@ rb_shape_get_iv_index_with_hint(shape_id_t shape_id, ID id, attr_index_t *value, return rb_shape_get_iv_index(shape, id, value); } -bool -rb_shape_get_iv_index(rb_shape_t * shape, ID id, attr_index_t *value) +static bool +shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) { - // It doesn't make sense to ask for the index of an IV that's stored - // on an object that is "too complex" as it uses a hash for storing IVs - RUBY_ASSERT(rb_shape_id(shape) != OBJ_TOO_COMPLEX_SHAPE_ID); - while (shape->parent_id != INVALID_SHAPE_ID) { - // Try the ancestor cache if it's available - if (shape->ancestor_index && shape->next_iv_index >= ANCESTOR_CACHE_THRESHOLD) { - redblack_node_t * node = redblack_find(shape->ancestor_index, id); - if (node) { - rb_shape_t * shape = redblack_value(node); + if (shape->edge_name == id) { + enum shape_type shape_type; + shape_type = (enum shape_type)shape->type; + + switch (shape_type) { + case SHAPE_IVAR: + RUBY_ASSERT(shape->next_iv_index > 0); *value = shape->next_iv_index - 1; return true; - } - else { + case SHAPE_ROOT: + case SHAPE_T_OBJECT: return false; + case SHAPE_OBJ_TOO_COMPLEX: + case SHAPE_FROZEN: + rb_bug("Ivar should not exist on transition"); } } - else { - if (shape->edge_name == id) { - enum shape_type shape_type; - shape_type = (enum shape_type)shape->type; - - switch (shape_type) { - case SHAPE_IVAR: - RUBY_ASSERT(shape->next_iv_index > 0); - *value = shape->next_iv_index - 1; - return true; - case SHAPE_ROOT: - case SHAPE_T_OBJECT: - return false; - case SHAPE_OBJ_TOO_COMPLEX: - case SHAPE_FROZEN: - rb_bug("Ivar should not exist on transition"); - } - } - } + shape = rb_shape_get_parent(shape); } + return false; } +static bool +shape_cache_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) +{ + if (shape->ancestor_index && shape->next_iv_index >= ANCESTOR_CACHE_THRESHOLD) { + redblack_node_t *node = redblack_find(shape->ancestor_index, id); + if (node) { + rb_shape_t *shape = redblack_value(node); + *value = shape->next_iv_index - 1; + +#if RUBY_DEBUG + attr_index_t shape_tree_index; + RUBY_ASSERT(shape_get_iv_index(shape, id, &shape_tree_index)); + RUBY_ASSERT(shape_tree_index == *value); +#endif + + return true; + } + + /* Verify the cache is correct by checking that this instance variable + * does not exist in the shape tree either. */ + RUBY_ASSERT(!shape_get_iv_index(shape, id, value)); + } + + return false; +} + +bool +rb_shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) +{ + // It doesn't make sense to ask for the index of an IV that's stored + // on an object that is "too complex" as it uses a hash for storing IVs + RUBY_ASSERT(rb_shape_id(shape) != OBJ_TOO_COMPLEX_SHAPE_ID); + + if (!shape_cache_get_iv_index(shape, id, value)) { + return shape_get_iv_index(shape, id, value); + } + + return true; +} + void rb_shape_set_shape(VALUE obj, rb_shape_t* shape) { From 1b7376423d3bf79ba37856eae8f45d59ecc9c170 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 01:18:48 +0900 Subject: [PATCH 26/78] Omit test_session_reuse_but_expire if OpenSSL 3.2.0 --- test/net/http/test_https.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/http/test_https.rb b/test/net/http/test_https.rb index 5f9e066b6bedbf..89d500118db24b 100644 --- a/test/net/http/test_https.rb +++ b/test/net/http/test_https.rb @@ -167,7 +167,7 @@ def test_session_reuse def test_session_reuse_but_expire # FIXME: The new_session_cb is known broken for clients in OpenSSL 1.1.0h. omit if OpenSSL::OPENSSL_LIBRARY_VERSION.include?('OpenSSL 1.1.0h') - omit 'failing on MinGW' if /mingw/ =~ RUBY_PLATFORM + omit if OpenSSL::OPENSSL_LIBRARY_VERSION.include?('OpenSSL 3.2.0') http = Net::HTTP.new(HOST, config("port")) http.use_ssl = true From 87c3deacf431dd1ee4538ba15db54f884fa8a918 Mon Sep 17 00:00:00 2001 From: Jeremy Evans Date: Mon, 26 Aug 2019 20:30:59 -0700 Subject: [PATCH 27/78] [ruby/resolv] Support a :use_ipv6 option to Resolv#initialize When set, supports returning IPv6 results even if there is no public IPv6 address for the system. Implements Ruby Feature #14922 https://github.com/ruby/resolv/commit/09d141de38 --- lib/resolv.rb | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/lib/resolv.rb b/lib/resolv.rb index 0943edc40086b4..68790225ca1972 100644 --- a/lib/resolv.rb +++ b/lib/resolv.rb @@ -84,8 +84,8 @@ def self.each_name(address, &proc) ## # Creates a new Resolv using +resolvers+. - def initialize(resolvers=[Hosts.new, DNS.new]) - @resolvers = resolvers + def initialize(resolvers=nil, use_ipv6: nil) + @resolvers = resolvers || [Hosts.new, DNS.new(DNS::Config.default_config_hash.merge(use_ipv6: use_ipv6))] end ## @@ -410,6 +410,11 @@ def each_address(name) end def use_ipv6? # :nodoc: + use_ipv6 = @config.use_ipv6? + unless use_ipv6.nil? + return use_ipv6 + end + begin list = Socket.ip_address_list rescue NotImplementedError @@ -1008,6 +1013,7 @@ def lazy_initialize @mutex.synchronize { unless @initialized @nameserver_port = [] + @use_ipv6 = nil @search = nil @ndots = 1 case @config_info @@ -1032,6 +1038,9 @@ def lazy_initialize if config_hash.include? :nameserver_port @nameserver_port = config_hash[:nameserver_port].map {|ns, port| [ns, (port || Port)] } end + if config_hash.include? :use_ipv6 + @use_ipv6 = config_hash[:use_ipv6] + end @search = config_hash[:search] if config_hash.include? :search @ndots = config_hash[:ndots] if config_hash.include? :ndots @raise_timeout_errors = config_hash[:raise_timeout_errors] @@ -1088,6 +1097,10 @@ def nameserver_port @nameserver_port end + def use_ipv6? + @use_ipv6 + end + def generate_candidates(name) candidates = nil name = Name.create(name) From 003f06bde40008159e871411cae5a928121490ba Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 10:03:45 +0900 Subject: [PATCH 28/78] [ruby/resolv] Close leaked FD https://github.com/ruby/resolv/commit/49aefa3bba --- test/resolv/test_dns.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/resolv/test_dns.rb b/test/resolv/test_dns.rb index 63ab66657c9c8f..20c3408cd6b9ca 100644 --- a/test/resolv/test_dns.rb +++ b/test/resolv/test_dns.rb @@ -653,5 +653,7 @@ def test_unreachable_server config[:raise_timeout_errors] = true r = Resolv.new([Resolv::DNS.new(config)]) assert_raise(Resolv::ResolvError) { r.getaddresses('www.google.com') } + ensure + sock&.close end end From 7fe7b7bc5a7a3d79280c9dbf2a2383d386386b0f Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 00:28:36 +0900 Subject: [PATCH 29/78] Fix portability of bignum in ISeq Binary Format - Unless `sizeof(BDIGIT) == 4`, (8-byte integer not available), the size to be loaded was wrong. - Since `BDIGIT`s are dumped as raw binary, the loaded byte order was inverted unless little-endian. --- compile.c | 8 ++++++-- test/ruby/test_iseq.rb | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/compile.c b/compile.c index 6038a2272378bf..7e8b6c37b8af0d 100644 --- a/compile.c +++ b/compile.c @@ -12859,8 +12859,12 @@ ibf_load_object_bignum(const struct ibf_load *load, const struct ibf_object_head const struct ibf_object_bignum *bignum = IBF_OBJBODY(struct ibf_object_bignum, offset); int sign = bignum->slen > 0; ssize_t len = sign > 0 ? bignum->slen : -1 * bignum->slen; - VALUE obj = rb_integer_unpack(bignum->digits, len * 2, 2, 0, - INTEGER_PACK_LITTLE_ENDIAN | (sign == 0 ? INTEGER_PACK_NEGATIVE : 0)); + const int big_unpack_flags = /* c.f. rb_big_unpack() */ + INTEGER_PACK_LSWORD_FIRST | + INTEGER_PACK_NATIVE_BYTE_ORDER; + VALUE obj = rb_integer_unpack(bignum->digits, len, sizeof(BDIGIT), 0, + big_unpack_flags | + (sign == 0 ? INTEGER_PACK_NEGATIVE : 0)); if (header->internal) rb_obj_hide(obj); if (header->frozen) rb_obj_freeze(obj); return obj; diff --git a/test/ruby/test_iseq.rb b/test/ruby/test_iseq.rb index 6a1a1ea8c18711..f05d067ac218b1 100644 --- a/test/ruby/test_iseq.rb +++ b/test/ruby/test_iseq.rb @@ -791,4 +791,11 @@ def test_loading_kwargs_memory_leak end end; end + + def test_ibf_bignum + iseq = RubyVM::InstructionSequence.compile("0x0"+"_0123_4567_89ab_cdef"*5) + expected = iseq.eval + result = RubyVM::InstructionSequence.load_from_binary(iseq.to_binary).eval + assert_equal expected, result, proc {sprintf("expected: %x, result: %x", expected, result)} + end end From e81c380c0fb3611d5507bfa169b0437a8712ea7c Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 16:04:04 +0900 Subject: [PATCH 30/78] Constify `RUBY_REFERENCES_START` tables --- include/ruby/internal/gc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/ruby/internal/gc.h b/include/ruby/internal/gc.h index 2a5180671ce8df..68472c6454e8d5 100644 --- a/include/ruby/internal/gc.h +++ b/include/ruby/internal/gc.h @@ -47,7 +47,7 @@ RBIMPL_SYMBOL_EXPORT_BEGIN() #define REF_EDGE(s, p) (offsetof(struct s, p)) #define REFS_LIST_PTR(l) ((RUBY_DATA_FUNC)l) #define RUBY_REF_END SIZE_MAX -#define RUBY_REFERENCES_START(t) static size_t t[] = { +#define RUBY_REFERENCES_START(t) static const size_t t[] = { #define RUBY_REFERENCES_END RUBY_REF_END, }; /* gc.c */ From 087a919ea4a6e151aa65209ebbebc9a907079172 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 19:18:57 +0900 Subject: [PATCH 31/78] [DOC] Fix markup RDoc is not markdown. --- doc/timezones.rdoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/timezones.rdoc b/doc/timezones.rdoc index d59efba37d402c..c5af6653f9db2d 100644 --- a/doc/timezones.rdoc +++ b/doc/timezones.rdoc @@ -107,7 +107,7 @@ which will be called if defined: === Timezone Names If the class (the receiver of class methods, or the class of the receiver -of instance methods) has `find_timezone` singleton method, this method is +of instance methods) has +find_timezone+ singleton method, this method is called to achieve the corresponding timezone object from a timezone name. For example, using {Timezone}[https://github.com/panthomakos/timezone]: @@ -128,4 +128,4 @@ Or, using {TZInfo}[https://tzinfo.github.io]: TimeWithTZInfo.now(in: "America/New_York") #=> 2023-12-25 00:00:00 -0500 TimeWithTZInfo.new("2023-12-25 America/New_York") #=> 2023-12-25 00:00:00 -0500 -You can define this method per subclasses, or on the toplevel `Time` class. +You can define this method per subclasses, or on the toplevel Time class. From 0bced53a8a11055d33160aa3f023b4b957e9e497 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 19:37:56 +0900 Subject: [PATCH 32/78] [DOC] Remove extra `+` which is not a part of keyword argument name --- timev.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timev.rb b/timev.rb index db3a1925cec639..6295e16bde9195 100644 --- a/timev.rb +++ b/timev.rb @@ -270,7 +270,7 @@ def self.now(in: nil) # Time.at(secs, -1000000000, :nanosecond) # => 2000-12-31 23:59:58 -0600 # # - # Optional keyword argument +in: zone specifies the timezone + # Optional keyword argument in: zone specifies the timezone # for the returned time: # # Time.at(secs, in: '+12:00') # => 2001-01-01 17:59:59 +1200 From 9cd086ba4b559153864ab924723a665a4ddfb5d8 Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Sun, 26 Nov 2023 11:07:46 +0000 Subject: [PATCH 33/78] [ruby/irb] Support disabling pager (https://github.com/ruby/irb/pull/783) With either `IRB.conf[:USE_PAGER] = false` or `--no-pager` commnad line flag. I decided use `--no-pager` instead of `--use-pager` because it matches with Pry and git's command line flags. https://github.com/ruby/irb/commit/df1c3b9042 --- lib/irb/init.rb | 3 +++ lib/irb/lc/help-message | 1 + lib/irb/pager.rb | 2 +- test/irb/test_cmd.rb | 5 +---- test/irb/test_debug_cmd.rb | 6 ++++-- test/irb/test_irb.rb | 6 ++---- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/irb/init.rb b/lib/irb/init.rb index 4df285ce64b298..9704e36cb10745 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -82,6 +82,7 @@ def IRB.init_config(ap_path) @CONF[:USE_LOADER] = false @CONF[:IGNORE_SIGINT] = true @CONF[:IGNORE_EOF] = false + @CONF[:USE_PAGER] = true @CONF[:EXTRA_DOC_DIRS] = [] @CONF[:ECHO] = nil @CONF[:ECHO_ON_ASSIGNMENT] = nil @@ -285,6 +286,8 @@ def IRB.parse_opts(argv: ::ARGV) end when "--noinspect" @CONF[:INSPECT_MODE] = false + when "--no-pager" + @CONF[:USE_PAGER] = false when "--singleline", "--readline", "--legacy" @CONF[:USE_SINGLELINE] = true when "--nosingleline", "--noreadline" diff --git a/lib/irb/lc/help-message b/lib/irb/lc/help-message index c7846b755df96d..37347306e8497a 100644 --- a/lib/irb/lc/help-message +++ b/lib/irb/lc/help-message @@ -22,6 +22,7 @@ Usage: irb.rb [options] [programfile] [arguments] Show truncated result on assignment (default). --inspect Use 'inspect' for output. --noinspect Don't use 'inspect' for output. + --no-pager Don't use pager. --multiline Use multiline editor module (default). --nomultiline Don't use multiline editor module. --singleline Use single line editor module. diff --git a/lib/irb/pager.rb b/lib/irb/pager.rb index 119515078ba25d..e38d97e3c7d347 100644 --- a/lib/irb/pager.rb +++ b/lib/irb/pager.rb @@ -18,7 +18,7 @@ def page_content(content) end def page - if STDIN.tty? && pager = setup_pager + if IRB.conf[:USE_PAGER] && STDIN.tty? && pager = setup_pager begin pid = pager.pid yield pager diff --git a/test/irb/test_cmd.rb b/test/irb/test_cmd.rb index 219710c9210f34..62ef7a5b70e2a7 100644 --- a/test/irb/test_cmd.rb +++ b/test/irb/test_cmd.rb @@ -23,9 +23,6 @@ def setup save_encodings IRB.instance_variable_get(:@CONF).clear @is_win = (RbConfig::CONFIG['host_os'] =~ /mswin|msys|mingw|cygwin|bccwin|wince|emc/) - STDIN.singleton_class.define_method :tty? do - false - end end def teardown @@ -34,13 +31,13 @@ def teardown Dir.chdir(@pwd) FileUtils.rm_rf(@tmpdir) restore_encodings - STDIN.singleton_class.remove_method :tty? end def execute_lines(*lines, conf: {}, main: self, irb_path: nil) IRB.init_config(nil) IRB.conf[:VERBOSE] = false IRB.conf[:PROMPT_MODE] = :SIMPLE + IRB.conf[:USE_PAGER] = false IRB.conf.merge!(conf) input = TestInputMethod.new(lines) irb = IRB::Irb.new(IRB::WorkSpace.new(main), input) diff --git a/test/irb/test_debug_cmd.rb b/test/irb/test_debug_cmd.rb index d669c174e6ef6a..53d40f7297ff27 100644 --- a/test/irb/test_debug_cmd.rb +++ b/test/irb/test_debug_cmd.rb @@ -346,9 +346,11 @@ def test_help_command_is_delegated_to_the_debugger end def test_show_cmds_display_different_content_when_debugger_is_enabled + write_rc <<~RUBY + IRB.conf[:USE_PAGER] = false + RUBY + write_ruby <<~'ruby' - # disable pager - STDIN.singleton_class.define_method(:tty?) { false } binding.irb ruby diff --git a/test/irb/test_irb.rb b/test/irb/test_irb.rb index b32e857c1e81a2..e6eb3d5da1f620 100644 --- a/test/irb/test_irb.rb +++ b/test/irb/test_irb.rb @@ -7,8 +7,7 @@ module TestIRB class InputTest < IntegrationTestCase def test_symbol_aliases_are_handled_correctly write_rc <<~RUBY - # disable pager - STDIN.singleton_class.define_method(:tty?) { false } + IRB.conf[:USE_PAGER] = false RUBY write_ruby <<~'RUBY' @@ -27,8 +26,7 @@ class Foo def test_symbol_aliases_are_handled_correctly_with_singleline_mode write_rc <<~RUBY - # disable pager - STDIN.singleton_class.define_method(:tty?) { false } + IRB.conf[:USE_PAGER] = false IRB.conf[:USE_SINGLELINE] = true RUBY From 688faa93f03142b632b8eb0de0946f4e86845ebc Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 26 Nov 2023 21:49:18 +0900 Subject: [PATCH 34/78] [DOC] Fix markup in declarative marking API document - RDoc is not markdown, use `+` and `_` for code and variables respectively, instead of backquotes. - Remove useless backslashes. --- doc/extension.rdoc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/extension.rdoc b/doc/extension.rdoc index 50892725999ae8..b95ed033e4a4f3 100644 --- a/doc/extension.rdoc +++ b/doc/extension.rdoc @@ -779,26 +779,26 @@ approach to marking and reference updating is provided, by declaring offset references to the VALUES in your struct. Doing this allows the Ruby GC to support marking these references and GC -compaction without the need to define the `dmark` and `dcompact` callbacks. +compaction without the need to define the +dmark+ and +dcompact+ callbacks. You must define a static list of VALUE pointers to the offsets within your struct where the references are located, and set the "data" member to point to -this reference list. The reference list must end with `END_REFS` +this reference list. The reference list must end with +RUBY_END_REFS+. Some Macros have been provided to make edge referencing easier: -* RUBY_TYPED_DECL_MARKING =A flag that can be set on the `ruby_data_type_t` to indicate that references are being declared as edges. +* RUBY_TYPED_DECL_MARKING =A flag that can be set on the +ruby_data_type_t+ to indicate that references are being declared as edges. -* RUBY_REFERENCES_START(ref_list_name) - Define `ref_list_name` as a list of references +* RUBY_REFERENCES_START(ref_list_name) - Define _ref_list_name_ as a list of references * RUBY_REFERENCES_END - Mark the end of the references list. This will take care of terminating the list correctly -* RUBY_REF_EDGE\(struct, member\) - Declare `member` as a VALUE edge from `struct`. Use this after `RUBY_REFERENCES_START` +* RUBY_REF_EDGE(struct, member) - Declare _member_ as a VALUE edge from _struct_. Use this after +RUBY_REFERENCES_START+ * +REFS_LIST_PTR+ - Coerce the reference list into a format that can be - accepted by the existing `dmark` interface. + accepted by the existing +dmark+ interface. -The example below is from `Dir` (defined in `dir.c`) +The example below is from Dir (defined in +dir.c+) // The struct being wrapped. Notice this contains 3 members of which the second // is a VALUE reference to another ruby object. From cc5d1bf026bcc5b4929a4f9d5e32d2fa5730348c Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Sun, 26 Nov 2023 17:07:41 +0000 Subject: [PATCH 35/78] [ruby/irb] Display aliases in help message (https://github.com/ruby/irb/pull/788) Similar to Pry, it displays user-defined aliases in the help message with a dedicated section. With the current default aliases, it looks like: ``` ...other sections... Aliases $ Alias for `show_source` @ Alias for `whereami` ``` https://github.com/ruby/irb/commit/2a0eacc891 --- lib/irb/cmd/show_cmds.rb | 6 ++++++ lib/irb/context.rb | 14 +++++++++++++- lib/irb/init.rb | 4 ---- test/irb/test_cmd.rb | 10 ++++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/irb/cmd/show_cmds.rb b/lib/irb/cmd/show_cmds.rb index 7d6b3ec2668b6f..a8d899e4ace17e 100644 --- a/lib/irb/cmd/show_cmds.rb +++ b/lib/irb/cmd/show_cmds.rb @@ -16,6 +16,12 @@ def execute(*args) commands_info = IRB::ExtendCommandBundle.all_commands_info commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } + user_aliases = irb_context.instance_variable_get(:@user_aliases) + + commands_grouped_by_categories["Aliases"] = user_aliases.map do |alias_name, target| + { display_name: alias_name, description: "Alias for `#{target}`" } + end + if irb_context.with_debugger # Remove the original "Debugging" category commands_grouped_by_categories.delete("Debugging") diff --git a/lib/irb/context.rb b/lib/irb/context.rb index a7b8ca2c26506b..3442fbf4da5cf9 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -146,9 +146,21 @@ def initialize(irb, workspace = nil, input_method = nil) @newline_before_multiline_output = true end - @command_aliases = IRB.conf[:COMMAND_ALIASES] + @user_aliases = IRB.conf[:COMMAND_ALIASES].dup + @command_aliases = @user_aliases.merge(KEYWORD_ALIASES) end + # because all input will eventually be evaluated as Ruby code, + # command names that conflict with Ruby keywords need special workaround + # we can remove them once we implemented a better command system for IRB + KEYWORD_ALIASES = { + :break => :irb_break, + :catch => :irb_catch, + :next => :irb_next, + }.freeze + + private_constant :KEYWORD_ALIASES + private def build_completor completor_type = IRB.conf[:COMPLETOR] case completor_type diff --git a/lib/irb/init.rb b/lib/irb/init.rb index 9704e36cb10745..b69f68d53061e7 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -189,10 +189,6 @@ def IRB.init_config(ap_path) # Symbol aliases :'$' => :show_source, :'@' => :whereami, - # Keyword aliases - :break => :irb_break, - :catch => :irb_catch, - :next => :irb_next, } end diff --git a/test/irb/test_cmd.rb b/test/irb/test_cmd.rb index 62ef7a5b70e2a7..55373c2e8aa3b2 100644 --- a/test/irb/test_cmd.rb +++ b/test/irb/test_cmd.rb @@ -680,6 +680,16 @@ def test_show_cmds assert_match(/List all available commands and their description/, out) assert_match(/Start the debugger of debug\.gem/, out) end + + def test_show_cmds_list_user_aliases + out, err = execute_lines( + "show_cmds\n" + ) + + assert_empty err + assert_match(/\$\s+Alias for `show_source`/, out) + assert_match(/@\s+Alias for `whereami`/, out) + end end class LsTest < CommandTestCase From 08308fe3e8fd51f4445be9408a418d9ac6960921 Mon Sep 17 00:00:00 2001 From: Samuel Giddins Date: Mon, 18 Sep 2023 19:18:21 -0700 Subject: [PATCH 36/78] [rubygems/rubygems] Reduce allocations when installing gems with bundler ``` ==> memprof.after.txt <== Total allocated: 1.13 MB (2352 objects) Total retained: 10.08 kB (78 objects) ==> memprof.before.txt <== Total allocated: 46.27 MB (38439 objects) Total retained: 9.94 kB (75 objects) ``` Yes, we were allocating 45MB of arrays in `dependencies_installed?`, it was accidentally cubic. https://github.com/rubygems/rubygems/commit/13ab874388 --- lib/bundler/installer/parallel_installer.rb | 13 +++++++++---- .../bundler/installer/spec_installation_spec.rb | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/bundler/installer/parallel_installer.rb b/lib/bundler/installer/parallel_installer.rb index 11a90b36cb62a3..02ae7d535f8d76 100644 --- a/lib/bundler/installer/parallel_installer.rb +++ b/lib/bundler/installer/parallel_installer.rb @@ -42,8 +42,7 @@ def ignorable_dependency?(dep) # Checks installed dependencies against spec's dependencies to make # sure needed dependencies have been installed. - def dependencies_installed?(all_specs) - installed_specs = all_specs.select(&:installed?).map(&:name) + def dependencies_installed?(installed_specs) dependencies.all? {|d| installed_specs.include? d.name } end @@ -183,8 +182,14 @@ def require_tree_for_spec(spec) # previously installed specifications. We continue until all specs # are installed. def enqueue_specs - @specs.select(&:ready_to_enqueue?).each do |spec| - if spec.dependencies_installed? @specs + installed_specs = {} + @specs.each do |spec| + next unless spec.installed? + installed_specs[spec.name] = true + end + + @specs.each do |spec| + if spec.ready_to_enqueue? && spec.dependencies_installed?(installed_specs) spec.state = :enqueued worker_pool.enq spec end diff --git a/spec/bundler/bundler/installer/spec_installation_spec.rb b/spec/bundler/bundler/installer/spec_installation_spec.rb index e63ef26cb31631..c0ba05ad30ebcd 100644 --- a/spec/bundler/bundler/installer/spec_installation_spec.rb +++ b/spec/bundler/bundler/installer/spec_installation_spec.rb @@ -47,7 +47,8 @@ def a_spec.full_name all_specs = dependencies + [instance_double("SpecInstallation", :spec => "gamma", :name => "gamma", :installed? => false, :all_dependencies => [], :type => :production)] spec = described_class.new(dep) allow(spec).to receive(:all_dependencies).and_return(dependencies) - expect(spec.dependencies_installed?(all_specs)).to be_truthy + installed_specs = all_specs.select(&:installed?).map {|s| [s.name, true] }.to_h + expect(spec.dependencies_installed?(installed_specs)).to be_truthy end end @@ -59,7 +60,8 @@ def a_spec.full_name all_specs = dependencies + [instance_double("SpecInstallation", :spec => "gamma", :name => "gamma", :installed? => false, :all_dependencies => [], :type => :production)] spec = described_class.new(dep) allow(spec).to receive(:all_dependencies).and_return(dependencies) - expect(spec.dependencies_installed?(all_specs)).to be_falsey + installed_specs = all_specs.select(&:installed?).map {|s| [s.name, true] }.to_h + expect(spec.dependencies_installed?(installed_specs)).to be_falsey end end end From 52c7e43b8797de0d042d05ecf6fc4a883c5c372f Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Sat, 25 Nov 2023 17:52:15 +0100 Subject: [PATCH 37/78] [rubygems/rubygems] Add missing --prefer-local to Synopsis in bundle-install.1.ronn https://github.com/rubygems/rubygems/commit/e956c5bbe4 --- lib/bundler/man/bundle-add.1 | 2 +- lib/bundler/man/bundle-binstubs.1 | 2 +- lib/bundler/man/bundle-cache.1 | 2 +- lib/bundler/man/bundle-check.1 | 2 +- lib/bundler/man/bundle-clean.1 | 2 +- lib/bundler/man/bundle-config.1 | 2 +- lib/bundler/man/bundle-console.1 | 2 +- lib/bundler/man/bundle-doctor.1 | 2 +- lib/bundler/man/bundle-exec.1 | 2 +- lib/bundler/man/bundle-gem.1 | 2 +- lib/bundler/man/bundle-help.1 | 2 +- lib/bundler/man/bundle-info.1 | 2 +- lib/bundler/man/bundle-init.1 | 2 +- lib/bundler/man/bundle-inject.1 | 2 +- lib/bundler/man/bundle-install.1 | 4 ++-- lib/bundler/man/bundle-install.1.ronn | 1 + lib/bundler/man/bundle-list.1 | 2 +- lib/bundler/man/bundle-lock.1 | 2 +- lib/bundler/man/bundle-open.1 | 2 +- lib/bundler/man/bundle-outdated.1 | 2 +- lib/bundler/man/bundle-platform.1 | 2 +- lib/bundler/man/bundle-plugin.1 | 2 +- lib/bundler/man/bundle-pristine.1 | 2 +- lib/bundler/man/bundle-remove.1 | 2 +- lib/bundler/man/bundle-show.1 | 2 +- lib/bundler/man/bundle-update.1 | 2 +- lib/bundler/man/bundle-version.1 | 2 +- lib/bundler/man/bundle-viz.1 | 2 +- lib/bundler/man/bundle.1 | 2 +- lib/bundler/man/gemfile.5 | 2 +- 30 files changed, 31 insertions(+), 30 deletions(-) diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index 17f03fc290f627..ce0d40aac36e66 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-ADD" "1" "October 2023" "" "" +.TH "BUNDLE\-ADD" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index 00cbda104a9be7..ed87f993b98098 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-BINSTUBS" "1" "October 2023" "" "" +.TH "BUNDLE\-BINSTUBS" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index 14250e589aa382..c504c5daf05a87 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-CACHE" "1" "October 2023" "" "" +.TH "BUNDLE\-CACHE" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index cb70661591d93f..e514f45ba1cffe 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-CHECK" "1" "October 2023" "" "" +.TH "BUNDLE\-CHECK" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index 76450a35dc5813..46ac22550653be 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-CLEAN" "1" "October 2023" "" "" +.TH "BUNDLE\-CLEAN" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 34654301b26533..68c5aa0125b82e 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-CONFIG" "1" "October 2023" "" "" +.TH "BUNDLE\-CONFIG" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index a223558c686bbd..992e8461d7462b 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-CONSOLE" "1" "October 2023" "" "" +.TH "BUNDLE\-CONSOLE" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index 143d9b700f44e9..c3e69531368436 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-DOCTOR" "1" "October 2023" "" "" +.TH "BUNDLE\-DOCTOR" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 91454985f21ecf..e6b8de6edb8dee 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-EXEC" "1" "October 2023" "" "" +.TH "BUNDLE\-EXEC" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index 825c46fd473452..31f41b7b6a46f6 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-GEM" "1" "October 2023" "" "" +.TH "BUNDLE\-GEM" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index 26309a2e68472d..a6057becc92133 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-HELP" "1" "October 2023" "" "" +.TH "BUNDLE\-HELP" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index d6049aa6678669..b03e7897e04092 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-INFO" "1" "October 2023" "" "" +.TH "BUNDLE\-INFO" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index 3514ae40e1c3bb..a44ce87ed40456 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-INIT" "1" "October 2023" "" "" +.TH "BUNDLE\-INIT" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index 21fb8734fedd53..347ef5afc7b76f 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-INJECT" "1" "October 2023" "" "" +.TH "BUNDLE\-INJECT" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index 70dccc60788bab..a774b660a90a13 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,13 +1,13 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-INSTALL" "1" "October 2023" "" "" +.TH "BUNDLE\-INSTALL" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile . .SH "SYNOPSIS" -\fBbundle install\fR [\-\-binstubs[=DIRECTORY]] [\-\-clean] [\-\-deployment] [\-\-frozen] [\-\-full\-index] [\-\-gemfile=GEMFILE] [\-\-jobs=NUMBER] [\-\-local] [\-\-no\-cache] [\-\-no\-prune] [\-\-path PATH] [\-\-quiet] [\-\-redownload] [\-\-retry=NUMBER] [\-\-shebang] [\-\-standalone[=GROUP[ GROUP\.\.\.]]] [\-\-system] [\-\-trust\-policy=POLICY] [\-\-with=GROUP[ GROUP\.\.\.]] [\-\-without=GROUP[ GROUP\.\.\.]] +\fBbundle install\fR [\-\-binstubs[=DIRECTORY]] [\-\-clean] [\-\-deployment] [\-\-frozen] [\-\-full\-index] [\-\-gemfile=GEMFILE] [\-\-jobs=NUMBER] [\-\-local] [\-\-no\-cache] [\-\-no\-prune] [\-\-path PATH] [\-\-prefer\-local] [\-\-quiet] [\-\-redownload] [\-\-retry=NUMBER] [\-\-shebang] [\-\-standalone[=GROUP[ GROUP\.\.\.]]] [\-\-system] [\-\-trust\-policy=POLICY] [\-\-with=GROUP[ GROUP\.\.\.]] [\-\-without=GROUP[ GROUP\.\.\.]] . .SH "DESCRIPTION" Install the gems specified in your Gemfile(5)\. If this is the first time you run bundle install (and a \fBGemfile\.lock\fR does not exist), Bundler will fetch all remote sources, resolve dependencies and install all needed gems\. diff --git a/lib/bundler/man/bundle-install.1.ronn b/lib/bundler/man/bundle-install.1.ronn index be9ed0f97422ac..ac0014e24e5320 100644 --- a/lib/bundler/man/bundle-install.1.ronn +++ b/lib/bundler/man/bundle-install.1.ronn @@ -14,6 +14,7 @@ bundle-install(1) -- Install the dependencies specified in your Gemfile [--no-cache] [--no-prune] [--path PATH] + [--prefer-local] [--quiet] [--redownload] [--retry=NUMBER] diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index f66755b19a57e3..e6bd01de33a3b1 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-LIST" "1" "October 2023" "" "" +.TH "BUNDLE\-LIST" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index 783e97f9f2a4ad..55b08c8e802b44 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-LOCK" "1" "October 2023" "" "" +.TH "BUNDLE\-LOCK" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index 8fb26f98629729..fc2f0f86b2623f 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-OPEN" "1" "October 2023" "" "" +.TH "BUNDLE\-OPEN" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index a8afc5cc7474de..6b5e3b4cf4ccaf 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-OUTDATED" "1" "October 2023" "" "" +.TH "BUNDLE\-OUTDATED" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 6c0451ebac99b6..d8fc413f14bbdc 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-PLATFORM" "1" "October 2023" "" "" +.TH "BUNDLE\-PLATFORM" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index e295c30622a1ac..d452ce84bc2806 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-PLUGIN" "1" "October 2023" "" "" +.TH "BUNDLE\-PLUGIN" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index 6422ea95175bbd..0f8cf1c0647d2e 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-PRISTINE" "1" "October 2023" "" "" +.TH "BUNDLE\-PRISTINE" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index ebfb17ffb15f0c..458c838cad553c 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-REMOVE" "1" "October 2023" "" "" +.TH "BUNDLE\-REMOVE" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index ac64aee6b63c1d..2c1cc3ca40499c 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-SHOW" "1" "October 2023" "" "" +.TH "BUNDLE\-SHOW" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 37a289ddf9c3ca..144542c4227d97 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-UPDATE" "1" "October 2023" "" "" +.TH "BUNDLE\-UPDATE" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index ab29c247630633..b4381df9e3f96d 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-VERSION" "1" "October 2023" "" "" +.TH "BUNDLE\-VERSION" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index 3ae31a0fbac86d..ee5301561d9ad0 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE\-VIZ" "1" "October 2023" "" "" +.TH "BUNDLE\-VIZ" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index d89a8cb1ee94b5..b5e5bcb2ad0073 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "BUNDLE" "1" "October 2023" "" "" +.TH "BUNDLE" "1" "November 2023" "" "" . .SH "NAME" \fBbundle\fR \- Ruby Dependency Management diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index 4480bb625afc9c..04693486b85388 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "GEMFILE" "5" "October 2023" "" "" +.TH "GEMFILE" "5" "November 2023" "" "" . .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs From 67ee91a3058e9d918add38ed02205e7383f00fad Mon Sep 17 00:00:00 2001 From: Mau Magnaguagno Date: Sun, 26 Nov 2023 04:07:14 -0300 Subject: [PATCH 38/78] [ruby/psych] Prefer each_char in Psych::Visitors::Visitor::ToRuby#deserialize Use safe navigation operator with each_char to remove empty strings and improve readability. https://github.com/ruby/psych/commit/5fe714b216 --- ext/psych/lib/psych/visitors/to_ruby.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/psych/lib/psych/visitors/to_ruby.rb b/ext/psych/lib/psych/visitors/to_ruby.rb index 8614251ca9e6c3..f0fda9bdbcc4a0 100644 --- a/ext/psych/lib/psych/visitors/to_ruby.rb +++ b/ext/psych/lib/psych/visitors/to_ruby.rb @@ -101,7 +101,7 @@ def deserialize o source = $1 options = 0 lang = nil - ($2 || '').split('').each do |option| + $2&.each_char do |option| case option when 'x' then options |= Regexp::EXTENDED when 'i' then options |= Regexp::IGNORECASE From 60803e192eebd1048f28ea5ba2a2f2c753424dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 23 Nov 2023 22:06:19 +0100 Subject: [PATCH 39/78] [rubygems/rubygems] Remove no longer necessary workaround for old RubyGems https://github.com/rubygems/rubygems/commit/ed4eaefac0 --- spec/bundler/commands/config_spec.rb | 4 ++-- spec/bundler/install/gemfile/groups_spec.rb | 2 +- spec/bundler/lock/lockfile_spec.rb | 2 +- spec/bundler/other/major_deprecation_spec.rb | 2 +- spec/bundler/realworld/double_check_spec.rb | 4 ++-- spec/bundler/runtime/inline_spec.rb | 16 ++++++++-------- spec/bundler/runtime/load_spec.rb | 2 +- spec/bundler/runtime/platform_spec.rb | 2 +- spec/bundler/runtime/require_spec.rb | 2 +- spec/bundler/runtime/setup_spec.rb | 12 ++++++------ spec/bundler/support/helpers.rb | 2 +- spec/bundler/support/path.rb | 7 ------- spec/bundler/support/rubygems_version_manager.rb | 3 +-- 13 files changed, 26 insertions(+), 34 deletions(-) diff --git a/spec/bundler/commands/config_spec.rb b/spec/bundler/commands/config_spec.rb index ede93f99eb826c..99f9423c22bd6d 100644 --- a/spec/bundler/commands/config_spec.rb +++ b/spec/bundler/commands/config_spec.rb @@ -443,7 +443,7 @@ expect(err).to be_empty ruby(<<~RUBY) - require "#{entrypoint}" + require "bundler" print Bundler.settings.mirror_for("https://rails-assets.org") RUBY expect(out).to eq("https://rails-assets.org/") @@ -451,7 +451,7 @@ bundle "config set mirror.all http://localhost:9293" ruby(<<~RUBY) - require "#{entrypoint}" + require "bundler" print Bundler.settings.mirror_for("https://rails-assets.org") RUBY expect(out).to eq("http://localhost:9293/") diff --git a/spec/bundler/install/gemfile/groups_spec.rb b/spec/bundler/install/gemfile/groups_spec.rb index 734e012e849b73..5dd39ee921c218 100644 --- a/spec/bundler/install/gemfile/groups_spec.rb +++ b/spec/bundler/install/gemfile/groups_spec.rb @@ -349,7 +349,7 @@ G ruby <<-R - require "#{entrypoint}" + require "bundler" Bundler.setup :default Bundler.require :default puts RACK diff --git a/spec/bundler/lock/lockfile_spec.rb b/spec/bundler/lock/lockfile_spec.rb index c7c16f707723ba..455315dab7ca81 100644 --- a/spec/bundler/lock/lockfile_spec.rb +++ b/spec/bundler/lock/lockfile_spec.rb @@ -1733,7 +1733,7 @@ def set_lockfile_mtime_to_known_value expect do ruby <<-RUBY - require '#{entrypoint}' + require 'bundler' Bundler.setup RUBY end.not_to change { File.mtime(bundled_app_lock) } diff --git a/spec/bundler/other/major_deprecation_spec.rb b/spec/bundler/other/major_deprecation_spec.rb index 293c04d60a953c..baec67ec340def 100644 --- a/spec/bundler/other/major_deprecation_spec.rb +++ b/spec/bundler/other/major_deprecation_spec.rb @@ -503,7 +503,7 @@ G ruby <<-RUBY - require '#{entrypoint}' + require 'bundler' Bundler.setup Bundler.setup diff --git a/spec/bundler/realworld/double_check_spec.rb b/spec/bundler/realworld/double_check_spec.rb index d7f28d10bb28a2..cfeba26e53e38f 100644 --- a/spec/bundler/realworld/double_check_spec.rb +++ b/spec/bundler/realworld/double_check_spec.rb @@ -25,9 +25,9 @@ RUBY cmd = <<-RUBY - require "#{entrypoint}" + require "bundler" require "#{spec_dir}/support/artifice/vcr" - require "#{entrypoint}/inline" + require "bundler/inline" gemfile(true) do source "https://rubygems.org" gem "rails", path: "." diff --git a/spec/bundler/runtime/inline_spec.rb b/spec/bundler/runtime/inline_spec.rb index 6fcbb68cb2adfa..6dffd6d32ba17a 100644 --- a/spec/bundler/runtime/inline_spec.rb +++ b/spec/bundler/runtime/inline_spec.rb @@ -2,7 +2,7 @@ RSpec.describe "bundler/inline#gemfile" do def script(code, options = {}) - requires = ["#{entrypoint}/inline"] + requires = ["bundler/inline"] requires.unshift "#{spec_dir}/support/artifice/" + options.delete(:artifice) if options.key?(:artifice) requires = requires.map {|r| "require '#{r}'" }.join("\n") ruby("#{requires}\n\n" + code, options) @@ -95,7 +95,7 @@ def script(code, options = {}) it "lets me use my own ui object" do script <<-RUBY, :artifice => "endpoint" - require '#{entrypoint}' + require 'bundler' class MyBundlerUI < Bundler::UI::Shell def confirm(msg, newline = nil) puts "CONFIRMED!" @@ -114,7 +114,7 @@ def confirm(msg, newline = nil) it "has an option for quiet installation" do script <<-RUBY, :artifice => "endpoint" - require '#{entrypoint}/inline' + require 'bundler/inline' gemfile(true, :quiet => true) do source "https://notaserver.com" @@ -140,7 +140,7 @@ def confirm(msg, newline = nil) it "does not mutate the option argument" do script <<-RUBY - require '#{entrypoint}' + require 'bundler' options = { :ui => Bundler::UI::Shell.new } gemfile(false, options) do source "#{file_uri_for(gem_repo1)}" @@ -259,7 +259,7 @@ def confirm(msg, newline = nil) system_gems "rack-1.0.0" script <<-RUBY - require '#{entrypoint}' + require 'bundler' ui = Bundler::UI::Shell.new ui.level = "confirm" @@ -279,7 +279,7 @@ def confirm(msg, newline = nil) system_gems "rack-1.0.0" script <<-RUBY - require '#{entrypoint}' + require 'bundler' ui = Bundler::UI::Shell.new ui.level = "confirm" gemfile(true, ui: ui) do @@ -302,7 +302,7 @@ def confirm(msg, newline = nil) system_gems "rack-1.0.0" script <<-RUBY - require '#{entrypoint}' + require 'bundler' ui = Bundler::UI::Shell.new ui.level = "confirm" gemfile(true, ui: ui) do @@ -339,7 +339,7 @@ def confirm(msg, newline = nil) end script <<-RUBY - require '#{entrypoint}' + require 'bundler' ui = Bundler::UI::Shell.new ui.level = "confirm" gemfile(true, ui: ui) do diff --git a/spec/bundler/runtime/load_spec.rb b/spec/bundler/runtime/load_spec.rb index 96a22a46cc9992..f28ffd94607ff5 100644 --- a/spec/bundler/runtime/load_spec.rb +++ b/spec/bundler/runtime/load_spec.rb @@ -82,7 +82,7 @@ G ruby <<-RUBY - require "#{entrypoint}" + require "bundler" Bundler.setup :default Bundler.require :default puts RACK diff --git a/spec/bundler/runtime/platform_spec.rb b/spec/bundler/runtime/platform_spec.rb index 31d93a559faf07..82120f75b2ceb5 100644 --- a/spec/bundler/runtime/platform_spec.rb +++ b/spec/bundler/runtime/platform_spec.rb @@ -22,7 +22,7 @@ ruby <<-R begin - require '#{entrypoint}' + require 'bundler' Bundler.ui.silence { Bundler.setup } rescue Bundler::GemNotFound => e puts "WIN" diff --git a/spec/bundler/runtime/require_spec.rb b/spec/bundler/runtime/require_spec.rb index e59fa564f60fe3..61dbd303f7c262 100644 --- a/spec/bundler/runtime/require_spec.rb +++ b/spec/bundler/runtime/require_spec.rb @@ -199,7 +199,7 @@ G cmd = <<-RUBY - require '#{entrypoint}' + require 'bundler' Bundler.require RUBY ruby(cmd) diff --git a/spec/bundler/runtime/setup_spec.rb b/spec/bundler/runtime/setup_spec.rb index 6c414ccb5d406b..859b7a890a5183 100644 --- a/spec/bundler/runtime/setup_spec.rb +++ b/spec/bundler/runtime/setup_spec.rb @@ -194,7 +194,7 @@ def clean_load_path(lp) G ruby <<-R - require '#{entrypoint}' + require 'bundler' begin Bundler.setup @@ -441,7 +441,7 @@ def clean_load_path(lp) break_git! ruby <<-R - require "#{entrypoint}" + require "bundler" begin Bundler.setup @@ -1187,7 +1187,7 @@ def lock_with(bundler_version = nil) context "is not present" do it "does not change the lock" do lockfile lock_with(nil) - ruby "require '#{entrypoint}/setup'" + ruby "require 'bundler/setup'" expect(lockfile).to eq lock_with(nil) end end @@ -1206,7 +1206,7 @@ def lock_with(bundler_version = nil) it "does not change the lock" do system_gems "bundler-1.10.1" lockfile lock_with("1.10.1") - ruby "require '#{entrypoint}/setup'" + ruby "require 'bundler/setup'" expect(lockfile).to eq lock_with("1.10.1") end end @@ -1304,7 +1304,7 @@ def lock_with(ruby_version = nil) bundle :install ruby <<-RUBY - require '#{entrypoint}/setup' + require 'bundler/setup' puts defined?(::Digest) ? "Digest defined" : "Digest undefined" require 'digest' RUBY @@ -1314,7 +1314,7 @@ def lock_with(ruby_version = nil) it "does not load Psych" do gemfile "source \"#{file_uri_for(gem_repo1)}\"" ruby <<-RUBY - require '#{entrypoint}/setup' + require 'bundler/setup' puts defined?(Psych::VERSION) ? Psych::VERSION : "undefined" require 'psych' puts Psych::VERSION diff --git a/spec/bundler/support/helpers.rb b/spec/bundler/support/helpers.rb index d63cf1e839e5fc..f873220f149bc4 100644 --- a/spec/bundler/support/helpers.rb +++ b/spec/bundler/support/helpers.rb @@ -60,7 +60,7 @@ def exitstatus def run(cmd, *args) opts = args.last.is_a?(Hash) ? args.pop : {} groups = args.map(&:inspect).join(", ") - setup = "require '#{entrypoint}' ; Bundler.ui.silence { Bundler.setup(#{groups}) }" + setup = "require 'bundler' ; Bundler.ui.silence { Bundler.setup(#{groups}) }" ruby([setup, cmd].join(" ; "), opts) end diff --git a/spec/bundler/support/path.rb b/spec/bundler/support/path.rb index 8b9c0e12909444..23b8cc4180b521 100644 --- a/spec/bundler/support/path.rb +++ b/spec/bundler/support/path.rb @@ -225,13 +225,6 @@ def lib_dir root.join("lib") end - # Sometimes rubygems version under test does not include - # https://github.com/rubygems/rubygems/pull/2728 and will not always end up - # activating the current bundler. In that case, require bundler absolutely. - def entrypoint - Gem.rubygems_version < Gem::Version.new("3.1.a") ? "#{lib_dir}/bundler" : "bundler" - end - def global_plugin_gem(*args) home ".bundle", "plugin", "gems", *args end diff --git a/spec/bundler/support/rubygems_version_manager.rb b/spec/bundler/support/rubygems_version_manager.rb index 5653601ae89176..cb670d60e560de 100644 --- a/spec/bundler/support/rubygems_version_manager.rb +++ b/spec/bundler/support/rubygems_version_manager.rb @@ -30,11 +30,10 @@ def assert_system_features_not_loaded! rubygems_default_path = rubygems_path + "/defaults" bundler_path = rubylibdir + "/bundler" - bundler_exemptions = Gem.rubygems_version < Gem::Version.new("3.2.0") ? [bundler_path + "/errors.rb"] : [] bad_loaded_features = $LOADED_FEATURES.select do |loaded_feature| (loaded_feature.start_with?(rubygems_path) && !loaded_feature.start_with?(rubygems_default_path)) || - (loaded_feature.start_with?(bundler_path) && !bundler_exemptions.any? {|bundler_exemption| loaded_feature.start_with?(bundler_exemption) }) + loaded_feature.start_with?(bundler_path) end errors = if bad_loaded_features.any? From bd4bd61650403121dc1fc29fab850d048d851a55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 23 Nov 2023 22:53:35 +0100 Subject: [PATCH 40/78] [rubygems/rubygems] Simplify remembered flags deprecation message Configuration is now local by default. https://github.com/rubygems/rubygems/commit/6bc7709aa8 --- lib/bundler/cli.rb | 2 +- spec/bundler/other/major_deprecation_spec.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index 2938f30e97e62f..942f2984d4934c 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -888,7 +888,7 @@ def flag_deprecation(name, flag_name, option) Bundler::SharedHelpers.major_deprecation 2, "The `#{flag_name}` flag is deprecated because it relies on being " \ "remembered across bundler invocations, which bundler will no longer " \ - "do in future versions. Instead please use `bundle config set --local #{name.tr("-", "_")} " \ + "do in future versions. Instead please use `bundle config set #{name.tr("-", "_")} " \ "'#{value}'`, and stop using this flag" end end diff --git a/spec/bundler/other/major_deprecation_spec.rb b/spec/bundler/other/major_deprecation_spec.rb index baec67ec340def..f6d532529942d8 100644 --- a/spec/bundler/other/major_deprecation_spec.rb +++ b/spec/bundler/other/major_deprecation_spec.rb @@ -125,7 +125,7 @@ expect(deprecations).to include( "The `--path` flag is deprecated because it relies on being " \ "remembered across bundler invocations, which bundler will no " \ - "longer do in future versions. Instead please use `bundle config set --local " \ + "longer do in future versions. Instead please use `bundle config set " \ "path 'vendor/bundle'`, and stop using this flag" ) end @@ -147,7 +147,7 @@ expect(deprecations).to include( "The `--path` flag is deprecated because it relies on being " \ "remembered across bundler invocations, which bundler will no " \ - "longer do in future versions. Instead please use `bundle config set --local " \ + "longer do in future versions. Instead please use `bundle config set " \ "path 'vendor/bundle'`, and stop using this flag" ) end @@ -395,7 +395,7 @@ "The `#{flag_name}` flag is deprecated because it relies on " \ "being remembered across bundler invocations, which bundler " \ "will no longer do in future versions. Instead please use " \ - "`bundle config set --local #{option_name} '#{value}'`, and stop using this flag" + "`bundle config set #{option_name} '#{value}'`, and stop using this flag" ) end From fe57be5a2ea80655e1ece52518e3fe72058a0ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 23 Nov 2023 23:01:35 +0100 Subject: [PATCH 41/78] [rubygems/rubygems] Avoid some unnecessary quotes in remember flag deprecation message https://github.com/rubygems/rubygems/commit/3fd627e486 --- lib/bundler/cli.rb | 3 ++- spec/bundler/other/major_deprecation_spec.rb | 22 ++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index 942f2984d4934c..adc228193b70bd 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -884,12 +884,13 @@ def flag_deprecation(name, flag_name, option) value = options[name] value = value.join(" ").to_s if option.type == :array + value = "'#{value}'" unless option.type == :boolean Bundler::SharedHelpers.major_deprecation 2, "The `#{flag_name}` flag is deprecated because it relies on being " \ "remembered across bundler invocations, which bundler will no longer " \ "do in future versions. Instead please use `bundle config set #{name.tr("-", "_")} " \ - "'#{value}'`, and stop using this flag" + "#{value}`, and stop using this flag" end end end diff --git a/spec/bundler/other/major_deprecation_spec.rb b/spec/bundler/other/major_deprecation_spec.rb index f6d532529942d8..2d01b64537b555 100644 --- a/spec/bundler/other/major_deprecation_spec.rb +++ b/spec/bundler/other/major_deprecation_spec.rb @@ -370,16 +370,16 @@ end { - "clean" => ["clean", true], - "deployment" => ["deployment", true], - "frozen" => ["frozen", true], - "no-deployment" => ["deployment", false], - "no-prune" => ["no_prune", true], - "path" => ["path", "vendor/bundle"], - "shebang" => ["shebang", "ruby27"], - "system" => ["system", true], - "without" => ["without", "development"], - "with" => ["with", "development"], + "clean" => ["clean", "true"], + "deployment" => ["deployment", "true"], + "frozen" => ["frozen", "true"], + "no-deployment" => ["deployment", "false"], + "no-prune" => ["no_prune", "true"], + "path" => ["path", "'vendor/bundle'"], + "shebang" => ["shebang", "'ruby27'"], + "system" => ["system", "true"], + "without" => ["without", "'development'"], + "with" => ["with", "'development'"], }.each do |name, expectations| option_name, value = *expectations flag_name = "--#{name}" @@ -395,7 +395,7 @@ "The `#{flag_name}` flag is deprecated because it relies on " \ "being remembered across bundler invocations, which bundler " \ "will no longer do in future versions. Instead please use " \ - "`bundle config set #{option_name} '#{value}'`, and stop using this flag" + "`bundle config set #{option_name} #{value}`, and stop using this flag" ) end From e00d7b61827303ed3c7759757e9d0a90b80415cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 23 Nov 2023 22:46:35 +0100 Subject: [PATCH 42/78] [rubygems/rubygems] Keep a single copy of the remembered flag deprecation message https://github.com/rubygems/rubygems/commit/cb4e26eabc --- lib/bundler/cli.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index adc228193b70bd..8099b74ba1227f 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -458,11 +458,7 @@ def fund bundle without having to download any additional gems. D def cache - SharedHelpers.major_deprecation 2, - "The `--all` flag is deprecated because it relies on being " \ - "remembered across bundler invocations, which bundler will no longer " \ - "do in future versions. Instead please use `bundle config set cache_all true`, " \ - "and stop using this flag" if ARGV.include?("--all") + print_remembered_flag_deprecation("--all", "cache_all", "true") if ARGV.include?("--all") SharedHelpers.major_deprecation 2, "The `--path` flag is deprecated because its semantics are unclear. " \ @@ -886,11 +882,15 @@ def flag_deprecation(name, flag_name, option) value = value.join(" ").to_s if option.type == :array value = "'#{value}'" unless option.type == :boolean + print_remembered_flag_deprecation(flag_name, name.tr("-", "_"), value) + end + + def print_remembered_flag_deprecation(flag_name, option_name, option_value) Bundler::SharedHelpers.major_deprecation 2, "The `#{flag_name}` flag is deprecated because it relies on being " \ "remembered across bundler invocations, which bundler will no longer " \ - "do in future versions. Instead please use `bundle config set #{name.tr("-", "_")} " \ - "#{value}`, and stop using this flag" + "do in future versions. Instead please use `bundle config set #{option_name} " \ + "#{option_value}`, and stop using this flag" end end end From 56ac1b0e1435a425eb7451e10c0606a009d9113a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 23 Nov 2023 23:04:44 +0100 Subject: [PATCH 43/78] [rubygems/rubygems] Fix advice in `bundle install --system` deprecation https://github.com/rubygems/rubygems/commit/59a66e3560 --- lib/bundler/cli.rb | 4 +++- spec/bundler/other/major_deprecation_spec.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index 8099b74ba1227f..1f7afe72d852cc 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -250,10 +250,12 @@ def remove(*gems) def install SharedHelpers.major_deprecation(2, "The `--force` option has been renamed to `--redownload`") if ARGV.include?("--force") - %w[clean deployment frozen no-prune path shebang system without with].each do |option| + %w[clean deployment frozen no-prune path shebang without with].each do |option| remembered_flag_deprecation(option) end + print_remembered_flag_deprecation("--system", "path.system", "true") if ARGV.include?("--system") + remembered_negative_flag_deprecation("no-deployment") require_relative "cli/install" diff --git a/spec/bundler/other/major_deprecation_spec.rb b/spec/bundler/other/major_deprecation_spec.rb index 2d01b64537b555..7cc19262536465 100644 --- a/spec/bundler/other/major_deprecation_spec.rb +++ b/spec/bundler/other/major_deprecation_spec.rb @@ -377,7 +377,7 @@ "no-prune" => ["no_prune", "true"], "path" => ["path", "'vendor/bundle'"], "shebang" => ["shebang", "'ruby27'"], - "system" => ["system", "true"], + "system" => ["path.system", "true"], "without" => ["without", "'development'"], "with" => ["with", "'development'"], }.each do |name, expectations| From 794c879d19689df8ced5537ecaacdbe5313f2a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Thu, 23 Nov 2023 23:13:14 +0100 Subject: [PATCH 44/78] [rubygems/rubygems] Don't remember `--jobs` flag https://github.com/rubygems/rubygems/commit/9ab1136036 --- lib/bundler/settings.rb | 20 +++++++++++++++++++- spec/bundler/commands/install_spec.rb | 10 ++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index 0dd92b1ad99f36..8885b22e080d4a 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -46,6 +46,20 @@ class Settings update_requires_all_flag ].freeze + REMEMBERED_KEYS = %w[ + bin + cache_all + clean + deployment + frozen + no_prune + path + shebang + path.system + without + with + ].freeze + NUMBER_KEYS = %w[ jobs redirect @@ -115,7 +129,7 @@ def [](name) end def set_command_option(key, value) - if Bundler.feature_flag.forget_cli_options? + if !is_remembered(key) || Bundler.feature_flag.forget_cli_options? temporary(key => value) value else @@ -374,6 +388,10 @@ def is_array(key) ARRAY_KEYS.include?(self.class.key_to_s(key)) end + def is_remembered(key) + REMEMBERED_KEYS.include?(self.class.key_to_s(key)) + end + def is_credential(key) key == "gem.push_key" end diff --git a/spec/bundler/commands/install_spec.rb b/spec/bundler/commands/install_spec.rb index 4456af25725814..bed24f06185314 100644 --- a/spec/bundler/commands/install_spec.rb +++ b/spec/bundler/commands/install_spec.rb @@ -1288,4 +1288,14 @@ def run expect(err).to include("Could not find compatible versions") end end + + context "when --jobs option given" do + before do + install_gemfile "source \"#{file_uri_for(gem_repo1)}\"", :jobs => 1 + end + + it "does not save the flag to config" do + expect(bundled_app(".bundle/config")).not_to exist + end + end end From 71a8daecd9ad10ba8aa0d66131e326722f1d78ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20=C5=A0im=C3=A1nek?= Date: Tue, 27 Jun 2023 02:36:59 +0200 Subject: [PATCH 45/78] Opaque Etags for compact index requests This changes the CompactIndexClient to store etags received from the compact index in separate files rather than relying on the MD5 checksum of the file as the etag. Smoothes the upgrade from md5 etags to opaque by generating them when no etag file exists. This should reduce the initial impact of changing the caching behavior by reducing cache misses when the MD5 etag is the same. Eventually, the MD5 behavior should be retired and the etag should be considered completely opaque with no assumption that MD5 would match. --- lib/bundler/compact_index_client.rb | 21 +- lib/bundler/compact_index_client/cache.rb | 30 ++- .../compact_index_client/cache_file.rb | 153 ++++++++++++++ lib/bundler/compact_index_client/updater.rb | 165 +++++++-------- lib/bundler/fetcher/compact_index.rb | 4 +- lib/bundler/shared_helpers.rb | 15 ++ .../compact_index_client/updater_spec.rb | 199 +++++++++++++++++- .../install/gems/compact_index_spec.rb | 125 +++++++++-- .../compact_index_checksum_mismatch.rb | 4 +- .../compact_index_concurrent_download.rb | 7 +- .../artifice/compact_index_etag_match.rb | 16 ++ .../artifice/compact_index_partial_update.rb | 2 +- ...compact_index_partial_update_bad_digest.rb | 40 ++++ ...rtial_update_no_digest_not_incremental.rb} | 12 +- .../artifice/compact_index_range_ignored.rb | 40 ++++ .../support/artifice/helpers/compact_index.rb | 14 +- 16 files changed, 702 insertions(+), 145 deletions(-) create mode 100644 lib/bundler/compact_index_client/cache_file.rb create mode 100644 spec/bundler/support/artifice/compact_index_etag_match.rb create mode 100644 spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb rename spec/bundler/support/artifice/{compact_index_partial_update_no_etag_not_incremental.rb => compact_index_partial_update_no_digest_not_incremental.rb} (72%) create mode 100644 spec/bundler/support/artifice/compact_index_range_ignored.rb diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb index 127a50e81080aa..68e0d7e0d51ffc 100644 --- a/lib/bundler/compact_index_client.rb +++ b/lib/bundler/compact_index_client.rb @@ -5,7 +5,13 @@ module Bundler class CompactIndexClient + # NOTE: MD5 is here not because we expect a server to respond with it, but + # because we use it to generate the etag on first request during the upgrade + # to the compact index client that uses opaque etags saved to files. + # Remove once 2.5.0 has been out for a while. + SUPPORTED_DIGESTS = { "sha-256" => :SHA256, "md5" => :MD5 }.freeze DEBUG_MUTEX = Thread::Mutex.new + def self.debug return unless ENV["DEBUG_COMPACT_INDEX"] DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } @@ -14,6 +20,7 @@ def self.debug class Error < StandardError; end require_relative "compact_index_client/cache" + require_relative "compact_index_client/cache_file" require_relative "compact_index_client/updater" attr_reader :directory @@ -54,13 +61,13 @@ def sequentially def names Bundler::CompactIndexClient.debug { "/names" } - update(@cache.names_path, "names") + update("names", @cache.names_path, @cache.names_etag_path) @cache.names end def versions Bundler::CompactIndexClient.debug { "/versions" } - update(@cache.versions_path, "versions") + update("versions", @cache.versions_path, @cache.versions_etag_path) versions, @info_checksums_by_name = @cache.versions versions end @@ -76,36 +83,36 @@ def dependencies(names) def update_and_parse_checksums! Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } return @info_checksums_by_name if @parsed_checksums - update(@cache.versions_path, "versions") + update("versions", @cache.versions_path, @cache.versions_etag_path) @info_checksums_by_name = @cache.checksums @parsed_checksums = true end private - def update(local_path, remote_path) + def update(remote_path, local_path, local_etag_path) Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } unless synchronize { @endpoints.add?(remote_path) } Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } return end - @updater.update(local_path, url(remote_path)) + @updater.update(url(remote_path), local_path, local_etag_path) end def update_info(name) Bundler::CompactIndexClient.debug { "update_info(#{name})" } path = @cache.info_path(name) - checksum = @updater.checksum_for_file(path) unless existing = @info_checksums_by_name[name] Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } return end + checksum = SharedHelpers.checksum_for_file(path, :MD5) if checksum == existing Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } return end Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } - update(path, "info/#{name}") + update("info/#{name}", path, @cache.info_etag_path(name)) end def url(path) diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb index b5607c875139af..5efdf18eba5516 100644 --- a/lib/bundler/compact_index_client/cache.rb +++ b/lib/bundler/compact_index_client/cache.rb @@ -9,11 +9,8 @@ class Cache def initialize(directory) @directory = Pathname.new(directory).expand_path - info_roots.each do |dir| - SharedHelpers.filesystem_access(dir) do - FileUtils.mkdir_p(dir) - end - end + info_roots.each {|dir| mkdir(dir) } + mkdir(info_etag_root) end def names @@ -24,6 +21,10 @@ def names_path directory.join("names") end + def names_etag_path + directory.join("names.etag") + end + def versions versions_by_name = Hash.new {|hash, key| hash[key] = [] } info_checksums_by_name = {} @@ -49,6 +50,10 @@ def versions_path directory.join("versions") end + def versions_etag_path + directory.join("versions.etag") + end + def checksums checksums = {} @@ -76,8 +81,19 @@ def info_path(name) end end + def info_etag_path(name) + name = name.to_s + info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") + end + private + def mkdir(dir) + SharedHelpers.filesystem_access(dir) do + FileUtils.mkdir_p(dir) + end + end + def lines(path) return [] unless path.file? lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n") @@ -96,6 +112,10 @@ def info_roots directory.join("info-special-characters"), ] end + + def info_etag_root + directory.join("info-etags") + end end end end diff --git a/lib/bundler/compact_index_client/cache_file.rb b/lib/bundler/compact_index_client/cache_file.rb new file mode 100644 index 00000000000000..5988bc91b3bac4 --- /dev/null +++ b/lib/bundler/compact_index_client/cache_file.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +require_relative "../vendored_fileutils" +require "rubygems/package" + +module Bundler + class CompactIndexClient + # write cache files in a way that is robust to concurrent modifications + # if digests are given, the checksums will be verified + class CacheFile + DEFAULT_FILE_MODE = 0o644 + private_constant :DEFAULT_FILE_MODE + + class Error < RuntimeError; end + class ClosedError < Error; end + + class DigestMismatchError < Error + def initialize(digests, expected_digests) + super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}." + end + end + + # Initialize with a copy of the original file, then yield the instance. + def self.copy(path, &block) + new(path) do |file| + file.initialize_digests + + SharedHelpers.filesystem_access(path, :read) do + path.open("rb") do |s| + file.open {|f| IO.copy_stream(s, f) } + end + end + + yield file + end + end + + # Write data to a temp file, then replace the original file with it verifying the digests if given. + def self.write(path, data, digests = nil) + return unless data + new(path) do |file| + file.digests = digests + file.write(data) + end + end + + attr_reader :original_path, :path + + def initialize(original_path, &block) + @original_path = original_path + @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE + @path = original_path.sub(/$/, ".#{$$}.tmp") + return unless block_given? + begin + yield self + ensure + close + end + end + + def size + path.size + end + + # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys. + def initialize_digests(keys = nil) + @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup + @digests.transform_values! {|algo_class| SharedHelpers.digest(algo_class).new } + end + + # reset the digests so they don't contain any previously read data + def reset_digests + @digests&.each_value(&:reset) + end + + # set the digests that will be verified at the end + def digests=(expected_digests) + @expected_digests = expected_digests + + if @expected_digests.nil? + @digests = nil + elsif @digests + @digests = @digests.slice(*@expected_digests.keys) + else + initialize_digests(@expected_digests.keys) + end + end + + # remove this method when we stop generating md5 digests for legacy etags + def md5 + @digests && @digests["md5"] + end + + def digests? + @digests&.any? + end + + # Open the temp file for writing, reusing original permissions, yielding the IO object. + def open(write_mode = "wb", perm = @perm, &block) + raise ClosedError, "Cannot reopen closed file" if @closed + SharedHelpers.filesystem_access(path, :write) do + path.open(write_mode, perm) do |f| + yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f + end + end + end + + # Returns false without appending when no digests since appending is too error prone to do without digests. + def append(data) + return false unless digests? + open("a") {|f| f.write data } + verify && commit + end + + def write(data) + reset_digests + open {|f| f.write data } + commit! + end + + def commit! + verify || raise(DigestMismatchError.new(@base64digests, @expected_digests)) + commit + end + + # Verify the digests, returning true on match, false on mismatch. + def verify + return true unless @expected_digests && digests? + @base64digests = @digests.transform_values!(&:base64digest) + @digests = nil + @base64digests.all? {|algo, digest| @expected_digests[algo] == digest } + end + + # Replace the original file with the temp file without verifying digests. + # The file is permanently closed. + def commit + raise ClosedError, "Cannot commit closed file" if @closed + SharedHelpers.filesystem_access(original_path, :write) do + FileUtils.mv(path, original_path) + end + @closed = true + end + + # Remove the temp file without replacing the original file. + # The file is permanently closed. + def close + return if @closed + FileUtils.remove_file(path) if @path&.file? + @closed = true + end + end + end +end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb index 3b75d5c129b121..c4686fad7d74df 100644 --- a/lib/bundler/compact_index_client/updater.rb +++ b/lib/bundler/compact_index_client/updater.rb @@ -1,20 +1,11 @@ # frozen_string_literal: true -require_relative "../vendored_fileutils" - module Bundler class CompactIndexClient class Updater - class MisMatchedChecksumError < Error - def initialize(path, server_checksum, local_checksum) - @path = path - @server_checksum = server_checksum - @local_checksum = local_checksum - end - - def message - "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \ - "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})." + class MismatchedChecksumError < Error + def initialize(path, message) + super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}" end end @@ -22,100 +13,102 @@ def initialize(fetcher) @fetcher = fetcher end - def update(local_path, remote_path, retrying = nil) - headers = {} - - local_temp_path = local_path.sub(/$/, ".#{$$}") - local_temp_path = local_temp_path.sub(/$/, ".retrying") if retrying - local_temp_path = local_temp_path.sub(/$/, ".tmp") - - # first try to fetch any new bytes on the existing file - if retrying.nil? && local_path.file? - copy_file local_path, local_temp_path + def update(remote_path, local_path, etag_path) + append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path) + rescue CacheFile::DigestMismatchError => e + raise MismatchedChecksumError.new(remote_path, e.message) + rescue Zlib::GzipFile::Error + raise Bundler::HTTPError + end - headers["If-None-Match"] = etag_for(local_temp_path) - headers["Range"] = - if local_temp_path.size.nonzero? - # Subtract a byte to ensure the range won't be empty. - # Avoids 416 (Range Not Satisfiable) responses. - "bytes=#{local_temp_path.size - 1}-" - else - "bytes=#{local_temp_path.size}-" - end - end + private - response = @fetcher.call(remote_path, headers) - return nil if response.is_a?(Net::HTTPNotModified) + def append(remote_path, local_path, etag_path) + return false unless local_path.file? && local_path.size.nonzero? - content = response.body + CacheFile.copy(local_path) do |file| + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + etag ||= generate_etag(etag_path, file) # Remove this after 2.5.0 has been out for a while. - etag = (response["ETag"] || "").gsub(%r{\AW/}, "") - correct_response = SharedHelpers.filesystem_access(local_temp_path) do - if response.is_a?(Net::HTTPPartialContent) && local_temp_path.size.nonzero? - local_temp_path.open("a") {|f| f << slice_body(content, 1..-1) } + # Subtract a byte to ensure the range won't be empty. + # Avoids 416 (Range Not Satisfiable) responses. + response = @fetcher.call(remote_path, request_headers(etag, file.size - 1)) + break true if response.is_a?(Net::HTTPNotModified) - etag_for(local_temp_path) == etag + file.digests = parse_digests(response) + # server may ignore Range and return the full response + if response.is_a?(Net::HTTPPartialContent) + break false unless file.append(response.body.byteslice(1..-1)) else - local_temp_path.open("wb") {|f| f << content } - - etag.length.zero? || etag_for(local_temp_path) == etag + file.write(response.body) end + CacheFile.write(etag_path, etag(response)) + true end + end - if correct_response - SharedHelpers.filesystem_access(local_path) do - FileUtils.mv(local_temp_path, local_path) - end - return nil - end + # request without range header to get the full file or a 304 Not Modified + def replace(remote_path, local_path, etag_path) + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + response = @fetcher.call(remote_path, request_headers(etag)) + return true if response.is_a?(Net::HTTPNotModified) + CacheFile.write(local_path, response.body, parse_digests(response)) + CacheFile.write(etag_path, etag(response)) + end - if retrying - raise MisMatchedChecksumError.new(remote_path, etag, etag_for(local_temp_path)) - end + def request_headers(etag, range_start = nil) + headers = {} + headers["Range"] = "bytes=#{range_start}-" if range_start + headers["If-None-Match"] = etag if etag + headers + end - update(local_path, remote_path, :retrying) - rescue Zlib::GzipFile::Error - raise Bundler::HTTPError - ensure - FileUtils.remove_file(local_temp_path) if File.exist?(local_temp_path) + def etag_for_request(etag_path) + etag_path.read.tap(&:chomp!) if etag_path.file? end - def etag_for(path) - sum = checksum_for_file(path) - sum ? %("#{sum}") : nil + # When first releasing this opaque etag feature, we want to generate the old MD5 etag + # based on the content of the file. After that it will always use the saved opaque etag. + # This transparently saves existing users with good caches from updating a bunch of files. + # Remove this behavior after 2.5.0 has been out for a while. + def generate_etag(etag_path, file) + etag = file.md5.hexdigest + CacheFile.write(etag_path, etag) + etag end - def slice_body(body, range) - body.byteslice(range) + def etag(response) + return unless response["ETag"] + etag = response["ETag"].delete_prefix("W/") + return if etag.delete_prefix!('"') && !etag.delete_suffix!('"') + etag end - def checksum_for_file(path) - return nil unless path.file? - # This must use File.read instead of Digest.file().hexdigest - # because we need to preserve \n line endings on windows when calculating - # the checksum - SharedHelpers.filesystem_access(path, :read) do - File.open(path, "rb") do |f| - digest = SharedHelpers.digest(:MD5).new - buf = String.new(:capacity => 16_384, :encoding => Encoding::BINARY) - digest << buf while f.read(16_384, buf) - digest.hexdigest - end + # Unwraps and returns a Hash of digest algorithms and base64 values + # according to RFC 8941 Structured Field Values for HTTP. + # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence + # Ignores unsupported algorithms. + def parse_digests(response) + return unless header = response["Repr-Digest"] || response["Digest"] + digests = {} + header.split(",") do |param| + algorithm, value = param.split("=", 2) + algorithm.strip! + algorithm.downcase! + next unless SUPPORTED_DIGESTS.key?(algorithm) + next unless value = byte_sequence(value) + digests[algorithm] = value end + digests.empty? ? nil : digests end - private - - def copy_file(source, dest) - SharedHelpers.filesystem_access(source, :read) do - File.open(source, "r") do |s| - SharedHelpers.filesystem_access(dest, :write) do - File.open(dest, "wb", s.stat.mode) do |f| - IO.copy_stream(s, f) - end - end - end - end + # Unwrap surrounding colons (byte sequence) + # The wrapping characters must be matched or we return nil. + # Also handles quotes because right now rubygems.org sends them. + def byte_sequence(value) + return if value.delete_prefix!(":") && !value.delete_suffix!(":") + return if value.delete_prefix!('"') && !value.delete_suffix!('"') + value end end end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index dc30443e27a542..f0ba30c7ca687b 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -13,7 +13,7 @@ def self.compact_index_request(method_name) undef_method(method_name) define_method(method_name) do |*args, &blk| method.bind(self).call(*args, &blk) - rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e + rescue NetworkDownError, CompactIndexClient::Updater::MismatchedChecksumError => e raise HTTPError, e.message rescue AuthenticationRequiredError, BadAuthenticationError # Fail since we got a 401 from the server. @@ -62,7 +62,7 @@ def available? end # Read info file checksums out of /versions, so we can know if gems are up to date compact_index_client.update_and_parse_checksums! - rescue CompactIndexClient::Updater::MisMatchedChecksumError => e + rescue CompactIndexClient::Updater::MismatchedChecksumError => e Bundler.ui.debug(e.message) nil end diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb index cccc2b63d9ee65..fa7db1c09d42d8 100644 --- a/lib/bundler/shared_helpers.rb +++ b/lib/bundler/shared_helpers.rb @@ -193,6 +193,21 @@ def digest(name) Digest(name) end + def checksum_for_file(path, digest) + return unless path.file? + # This must use File.read instead of Digest.file().hexdigest + # because we need to preserve \n line endings on windows when calculating + # the checksum + SharedHelpers.filesystem_access(path, :read) do + File.open(path, "rb") do |f| + digest = SharedHelpers.digest(digest).new + buf = String.new(:capacity => 16_384, :encoding => Encoding::BINARY) + digest << buf while f.read(16_384, buf) + digest.hexdigest + end + end + end + def write_to_gemfile(gemfile_path, contents) filesystem_access(gemfile_path) {|g| File.open(g, "w") {|file| file.puts contents } } end diff --git a/spec/bundler/bundler/compact_index_client/updater_spec.rb b/spec/bundler/bundler/compact_index_client/updater_spec.rb index fe417e392067c9..430e536ac0dfdb 100644 --- a/spec/bundler/bundler/compact_index_client/updater_spec.rb +++ b/spec/bundler/bundler/compact_index_client/updater_spec.rb @@ -6,21 +6,202 @@ require "tmpdir" RSpec.describe Bundler::CompactIndexClient::Updater do + subject(:updater) { described_class.new(fetcher) } + let(:fetcher) { double(:fetcher) } - let(:local_path) { Pathname.new Dir.mktmpdir("localpath") } + let(:local_path) { Pathname.new(Dir.mktmpdir("localpath")).join("versions") } + let(:etag_path) { Pathname.new(Dir.mktmpdir("localpath-etags")).join("versions.etag") } let(:remote_path) { double(:remote_path) } - let!(:updater) { described_class.new(fetcher) } + let(:full_body) { "abc123" } + let(:response) { double(:response, :body => full_body, :is_a? => false) } + let(:digest) { Digest::SHA256.base64digest(full_body) } + + context "when the local path does not exist" do + before do + allow(response).to receive(:[]).with("Repr-Digest") { nil } + allow(response).to receive(:[]).with("Digest") { nil } + allow(response).to receive(:[]).with("ETag") { "thisisanetag" } + end + + it "downloads the file without attempting append" do + expect(fetcher).to receive(:call).once.with(remote_path, {}) { response } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("thisisanetag") + end + + it "fails immediately on bad checksum" do + expect(fetcher).to receive(:call).once.with(remote_path, {}) { response } + allow(response).to receive(:[]).with("Repr-Digest") { "sha-256=:baddigest:" } + + expect do + updater.update(remote_path, local_path, etag_path) + end.to raise_error(Bundler::CompactIndexClient::Updater::MismatchedChecksumError) + end + end + + context "when the local path exists" do + let(:local_body) { "abc" } + + before do + local_path.open("w") {|f| f.write(local_body) } + end + + context "with an etag" do + before do + etag_path.open("w") {|f| f.write("LocalEtag") } + end + + let(:headers) do + { + "If-None-Match" => "LocalEtag", + "Range" => "bytes=2-", + } + end + + it "does nothing if etags match" do + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { false } + allow(response).to receive(:is_a?).with(Net::HTTPNotModified) { true } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq("abc") + expect(etag_path.read).to eq("LocalEtag") + end + + it "appends the file if etags do not match" do + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + allow(response).to receive(:[]).with("Repr-Digest") { "sha-256=:#{digest}:" } + allow(response).to receive(:[]).with("ETag") { "NewEtag" } + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { true } + allow(response).to receive(:is_a?).with(Net::HTTPNotModified) { false } + allow(response).to receive(:body) { "c123" } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("NewEtag") + end + + it "replaces the file if response ignores range" do + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + allow(response).to receive(:[]).with("Repr-Digest") { "sha-256=:#{digest}:" } + allow(response).to receive(:[]).with("ETag") { "NewEtag" } + allow(response).to receive(:body) { full_body } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("NewEtag") + end + + it "tries the request again if the partial response fails digest check" do + allow(response).to receive(:[]).with("Repr-Digest") { "sha-256=:baddigest:" } + allow(response).to receive(:body) { "the beginning of the file changed" } + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { true } + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + + full_response = double(:full_response, :body => full_body, :is_a? => false) + allow(full_response).to receive(:[]).with("Repr-Digest") { "sha-256=:#{digest}:" } + allow(full_response).to receive(:[]).with("ETag") { "NewEtag" } + expect(fetcher).to receive(:call).once.with(remote_path, { "If-None-Match" => "LocalEtag" }).and_return(full_response) + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("NewEtag") + end + end + + context "without an etag file" do + let(:headers) do + { + "Range" => "bytes=2-", + # This MD5 feature should be deleted after sufficient time has passed since release. + # From then on, requests that still don't have a saved etag will be made without this header. + "If-None-Match" => Digest::MD5.hexdigest(local_body), + } + end + + it "saves only the etag_path if generated etag matches" do + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { false } + allow(response).to receive(:is_a?).with(Net::HTTPNotModified) { true } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq("abc") + expect(etag_path.read).to eq(headers["If-None-Match"]) + end + + it "appends the file" do + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + allow(response).to receive(:[]).with("Repr-Digest") { "sha-256=:#{digest}:" } + allow(response).to receive(:[]).with("ETag") { "OpaqueEtag" } + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { true } + allow(response).to receive(:is_a?).with(Net::HTTPNotModified) { false } + allow(response).to receive(:body) { "c123" } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("OpaqueEtag") + end + + it "replaces the file on full file response that ignores range request" do + expect(fetcher).to receive(:call).once.with(remote_path, headers).and_return(response) + allow(response).to receive(:[]).with("Repr-Digest") { nil } + allow(response).to receive(:[]).with("Digest") { nil } + allow(response).to receive(:[]).with("ETag") { "OpaqueEtag" } + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { false } + allow(response).to receive(:is_a?).with(Net::HTTPNotModified) { false } + allow(response).to receive(:body) { full_body } + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("OpaqueEtag") + end + + it "tries the request again if the partial response fails digest check" do + allow(response).to receive(:[]).with("Repr-Digest") { "sha-256=:baddigest:" } + allow(response).to receive(:body) { "the beginning of the file changed" } + allow(response).to receive(:is_a?).with(Net::HTTPPartialContent) { true } + expect(fetcher).to receive(:call).once.with(remote_path, headers) do + # During the failed first request, we simulate another process writing the etag. + # This ensures the second request doesn't generate the md5 etag again but just uses whatever is written. + etag_path.open("w") {|f| f.write("LocalEtag") } + response + end + + full_response = double(:full_response, :body => full_body, :is_a? => false) + allow(full_response).to receive(:[]).with("Repr-Digest") { "sha-256=:#{digest}:" } + allow(full_response).to receive(:[]).with("ETag") { "NewEtag" } + expect(fetcher).to receive(:call).once.with(remote_path, { "If-None-Match" => "LocalEtag" }).and_return(full_response) + + updater.update(remote_path, local_path, etag_path) + + expect(local_path.read).to eq(full_body) + expect(etag_path.read).to eq("NewEtag") + end + end + end context "when the ETag header is missing" do # Regression test for https://github.com/rubygems/bundler/issues/5463 - let(:response) { double(:response, :body => "abc123") } + let(:response) { double(:response, :body => full_body) } it "treats the response as an update" do - expect(response).to receive(:[]).with("ETag") { nil } + allow(response).to receive(:[]).with("Repr-Digest") { nil } + allow(response).to receive(:[]).with("Digest") { nil } + allow(response).to receive(:[]).with("ETag") { nil } expect(fetcher).to receive(:call) { response } - updater.update(local_path, remote_path) + updater.update(remote_path, local_path, etag_path) end end @@ -31,7 +212,7 @@ expect(fetcher).to receive(:call).and_raise(Zlib::GzipFile::Error) expect do - updater.update(local_path, remote_path) + updater.update(remote_path, local_path, etag_path) end.to raise_error(Bundler::HTTPError) end end @@ -46,10 +227,12 @@ begin $VERBOSE = false Encoding.default_internal = "ASCII" - expect(response).to receive(:[]).with("ETag") { nil } + allow(response).to receive(:[]).with("Repr-Digest") { nil } + allow(response).to receive(:[]).with("Digest") { nil } + allow(response).to receive(:[]).with("ETag") { nil } expect(fetcher).to receive(:call) { response } - updater.update(local_path, remote_path) + updater.update(remote_path, local_path, etag_path) ensure Encoding.default_internal = previous_internal_encoding $VERBOSE = old_verbose diff --git a/spec/bundler/install/gems/compact_index_spec.rb b/spec/bundler/install/gems/compact_index_spec.rb index b2ed9565a42fe0..b383614410b5c7 100644 --- a/spec/bundler/install/gems/compact_index_spec.rb +++ b/spec/bundler/install/gems/compact_index_spec.rb @@ -157,9 +157,8 @@ bundle :install, :verbose => true, :artifice => "compact_index_checksum_mismatch" expect(out).to include("Fetching gem metadata from #{source_uri}") - expect(out).to include <<-'WARN' -The checksum of /versions does not match the checksum provided by the server! Something is wrong (local checksum is "\"d41d8cd98f00b204e9800998ecf8427e\"", was expecting "\"123\""). - WARN + expect(out).to include("The checksum of /versions does not match the checksum provided by the server!") + expect(out).to include('Calculated checksums {"sha-256"=>"8KfZiM/fszVkqhP/m5s9lvE6M9xKu4I1bU4Izddp5Ms="} did not match expected {"sha-256"=>"ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}') expect(the_bundle).to include_gems "rack 1.0.0" end @@ -169,10 +168,12 @@ gem "rack" G - versions = File.join(Bundler.rubygems.user_home, ".bundle", "cache", "compact_index", - "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions") - FileUtils.mkdir_p(File.dirname(versions)) - FileUtils.touch(versions) + versions = Pathname.new(Bundler.rubygems.user_home).join( + ".bundle", "cache", "compact_index", + "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions" + ) + versions.dirname.mkpath + versions.write("created_at") FileUtils.chmod("-r", versions) bundle :install, :artifice => "compact_index", :raise_on_error => false @@ -772,23 +773,83 @@ def start end end - it "performs partial update with a non-empty range" do + it "performs update with etag not-modified" do + versions_etag = Pathname.new(Bundler.rubygems.user_home).join( + ".bundle", "cache", "compact_index", + "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions.etag" + ) + expect(versions_etag.file?).to eq(false) + gemfile <<-G source "#{source_uri}" gem 'rack', '0.9.1' G - # Initial install creates the cached versions file + # Initial install creates the cached versions file and etag file bundle :install, :artifice => "compact_index" + expect(versions_etag.file?).to eq(true) + previous_content = versions_etag.binread + # Update the Gemfile so we can check subsequent install was successful gemfile <<-G source "#{source_uri}" gem 'rack', '1.0.0' G - # Second install should make only a partial request to /versions - bundle :install, :artifice => "compact_index_partial_update" + # Second install should match etag + bundle :install, :artifice => "compact_index_etag_match" + + expect(versions_etag.binread).to eq(previous_content) + expect(the_bundle).to include_gems "rack 1.0.0" + end + + it "performs full update when range is ignored" do + gemfile <<-G + source "#{source_uri}" + gem 'rack', '0.9.1' + G + + # Initial install creates the cached versions file and etag file + bundle :install, :artifice => "compact_index" + + gemfile <<-G + source "#{source_uri}" + gem 'rack', '1.0.0' + G + + versions = Pathname.new(Bundler.rubygems.user_home).join( + ".bundle", "cache", "compact_index", + "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions" + ) + # Modify the cached file. The ranged request will be based on this but, + # in this test, the range is ignored so this gets overwritten, allowing install. + versions.write "ruining this file" + + bundle :install, :artifice => "compact_index_range_ignored" + + expect(the_bundle).to include_gems "rack 1.0.0" + end + + it "performs partial update with a non-empty range" do + build_repo4 do + build_gem "rack", "0.9.1" + end + + # Initial install creates the cached versions file + install_gemfile <<-G, :artifice => "compact_index", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + source "#{source_uri}" + gem 'rack', '0.9.1' + G + + update_repo4 do + build_gem "rack", "1.0.0" + end + + install_gemfile <<-G, :artifice => "compact_index_partial_update", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + source "#{source_uri}" + gem 'rack', '1.0.0' + G expect(the_bundle).to include_gems "rack 1.0.0" end @@ -799,19 +860,43 @@ def start gem 'rack' G - # Create an empty file to trigger a partial download - versions = File.join(Bundler.rubygems.user_home, ".bundle", "cache", "compact_index", - "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions") - FileUtils.mkdir_p(File.dirname(versions)) - FileUtils.touch(versions) + # Create a partial cache versions file + versions = Pathname.new(Bundler.rubygems.user_home).join( + ".bundle", "cache", "compact_index", + "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions" + ) + versions.dirname.mkpath + versions.write("created_at") bundle :install, :artifice => "compact_index_concurrent_download" - expect(File.read(versions)).to start_with("created_at") + expect(versions.read).to start_with("created_at") + expect(the_bundle).to include_gems "rack 1.0.0" + end + + it "performs a partial update that fails digest check, then a full update" do + build_repo4 do + build_gem "rack", "0.9.1" + end + + install_gemfile <<-G, :artifice => "compact_index", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + source "#{source_uri}" + gem 'rack', '0.9.1' + G + + update_repo4 do + build_gem "rack", "1.0.0" + end + + install_gemfile <<-G, :artifice => "compact_index_partial_update_bad_digest", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + source "#{source_uri}" + gem 'rack', '1.0.0' + G + expect(the_bundle).to include_gems "rack 1.0.0" end - it "performs full update if server endpoints serve partial content responses but don't have incremental content and provide no Etag" do + it "performs full update if server endpoints serve partial content responses but don't have incremental content and provide no digest" do build_repo4 do build_gem "rack", "0.9.1" end @@ -825,7 +910,7 @@ def start build_gem "rack", "1.0.0" end - install_gemfile <<-G, :artifice => "compact_index_partial_update_no_etag_not_incremental", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } + install_gemfile <<-G, :artifice => "compact_index_partial_update_no_digest_not_incremental", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s } source "#{source_uri}" gem 'rack', '1.0.0' G @@ -847,7 +932,7 @@ def start expected_rack_info_content = File.read(rake_info_path) # Modify the cache files. We expect them to be reset to the normal ones when we re-run :install - File.open(rake_info_path, "w") {|f| f << (expected_rack_info_content + "this is different") } + File.open(rake_info_path, "a") {|f| f << "this is different" } # Update the Gemfile so the next install does its normal things gemfile <<-G diff --git a/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb b/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb index a6545b9ee4b574..83b147d2aecc96 100644 --- a/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb +++ b/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb @@ -4,10 +4,10 @@ class CompactIndexChecksumMismatch < CompactIndexAPI get "/versions" do - headers "ETag" => quote("123") + headers "Repr-Digest" => "sha-256=:ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=:" headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" content_type "text/plain" - body "" + body "content does not match the checksum" end end diff --git a/spec/bundler/support/artifice/compact_index_concurrent_download.rb b/spec/bundler/support/artifice/compact_index_concurrent_download.rb index 35548f278c8232..5d55b8a72b0d8c 100644 --- a/spec/bundler/support/artifice/compact_index_concurrent_download.rb +++ b/spec/bundler/support/artifice/compact_index_concurrent_download.rb @@ -7,11 +7,12 @@ class CompactIndexConcurrentDownload < CompactIndexAPI versions = File.join(Bundler.rubygems.user_home, ".bundle", "cache", "compact_index", "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions") - # Verify the original (empty) content hasn't been deleted, e.g. on a retry - File.binread(versions) == "" || raise("Original file should be present and empty") + # Verify the original content hasn't been deleted, e.g. on a retry + data = File.binread(versions) + data == "created_at" || raise("Original file should be present with expected content") # Verify this is only requested once for a partial download - env["HTTP_RANGE"] || raise("Missing Range header for expected partial download") + env["HTTP_RANGE"] == "bytes=#{data.bytesize - 1}-" || raise("Missing Range header for expected partial download") # Overwrite the file in parallel, which should be then overwritten # after a successful download to prevent corruption diff --git a/spec/bundler/support/artifice/compact_index_etag_match.rb b/spec/bundler/support/artifice/compact_index_etag_match.rb new file mode 100644 index 00000000000000..08d7b5ec539129 --- /dev/null +++ b/spec/bundler/support/artifice/compact_index_etag_match.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative "helpers/compact_index" + +class CompactIndexEtagMatch < CompactIndexAPI + get "/versions" do + raise "ETag header should be present" unless env["HTTP_IF_NONE_MATCH"] + headers "ETag" => env["HTTP_IF_NONE_MATCH"] + status 304 + body "" + end +end + +require_relative "helpers/artifice" + +Artifice.activate_with(CompactIndexEtagMatch) diff --git a/spec/bundler/support/artifice/compact_index_partial_update.rb b/spec/bundler/support/artifice/compact_index_partial_update.rb index 8c73011346b2ce..f111d91ef94ba8 100644 --- a/spec/bundler/support/artifice/compact_index_partial_update.rb +++ b/spec/bundler/support/artifice/compact_index_partial_update.rb @@ -23,7 +23,7 @@ def not_modified?(_checksum) # Verify that a partial request is made, starting from the index of the # final byte of the cached file. unless env["HTTP_RANGE"] == "bytes=#{File.binread(cached_versions_path).bytesize - 1}-" - raise("Range header should be present, and start from the index of the final byte of the cache.") + raise("Range header should be present, and start from the index of the final byte of the cache. #{env["HTTP_RANGE"].inspect}") end etag_response do diff --git a/spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb b/spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb new file mode 100644 index 00000000000000..72cb579ad4cac4 --- /dev/null +++ b/spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative "helpers/compact_index" + +# The purpose of this Artifice is to test that an incremental response is invalidated +# and a second request is issued for the full content. +class CompactIndexPartialUpdateBadDigest < CompactIndexAPI + def partial_update_bad_digest + response_body = yield + if request.env["HTTP_RANGE"] + headers "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest("wrong digest on ranged request")}:" + else + headers "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest(response_body)}:" + end + headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" + content_type "text/plain" + requested_range_for(response_body) + end + + get "/versions" do + partial_update_bad_digest do + file = tmp("versions.list") + FileUtils.rm_f(file) + file = CompactIndex::VersionsFile.new(file.to_s) + file.create(gems) + file.contents([], :calculate_info_checksums => true) + end + end + + get "/info/:name" do + partial_update_bad_digest do + gem = gems.find {|g| g.name == params[:name] } + CompactIndex.info(gem ? gem.versions : []) + end + end +end + +require_relative "helpers/artifice" + +Artifice.activate_with(CompactIndexPartialUpdateBadDigest) diff --git a/spec/bundler/support/artifice/compact_index_partial_update_no_etag_not_incremental.rb b/spec/bundler/support/artifice/compact_index_partial_update_no_digest_not_incremental.rb similarity index 72% rename from spec/bundler/support/artifice/compact_index_partial_update_no_etag_not_incremental.rb rename to spec/bundler/support/artifice/compact_index_partial_update_no_digest_not_incremental.rb index 20546ba4c3a80b..ce275037417122 100644 --- a/spec/bundler/support/artifice/compact_index_partial_update_no_etag_not_incremental.rb +++ b/spec/bundler/support/artifice/compact_index_partial_update_no_digest_not_incremental.rb @@ -2,8 +2,10 @@ require_relative "helpers/compact_index" -class CompactIndexPartialUpdateNoEtagNotIncremental < CompactIndexAPI - def partial_update_no_etag +# The purpose of this Artifice is to test that an incremental response is ignored +# when the digest is not present to verify that the partial response is valid. +class CompactIndexPartialUpdateNoDigestNotIncremental < CompactIndexAPI + def partial_update_no_digest response_body = yield headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" content_type "text/plain" @@ -11,7 +13,7 @@ def partial_update_no_etag end get "/versions" do - partial_update_no_etag do + partial_update_no_digest do file = tmp("versions.list") FileUtils.rm_f(file) file = CompactIndex::VersionsFile.new(file.to_s) @@ -25,7 +27,7 @@ def partial_update_no_etag end get "/info/:name" do - partial_update_no_etag do + partial_update_no_digest do gem = gems.find {|g| g.name == params[:name] } lines = CompactIndex.info(gem ? gem.versions : []).split("\n") @@ -37,4 +39,4 @@ def partial_update_no_etag require_relative "helpers/artifice" -Artifice.activate_with(CompactIndexPartialUpdateNoEtagNotIncremental) +Artifice.activate_with(CompactIndexPartialUpdateNoDigestNotIncremental) diff --git a/spec/bundler/support/artifice/compact_index_range_ignored.rb b/spec/bundler/support/artifice/compact_index_range_ignored.rb new file mode 100644 index 00000000000000..2303682c1f21f7 --- /dev/null +++ b/spec/bundler/support/artifice/compact_index_range_ignored.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative "helpers/compact_index" + +class CompactIndexRangeIgnored < CompactIndexAPI + # Stub the server to not return 304 so that we don't bypass all the logic + def not_modified?(_checksum) + false + end + + get "/versions" do + cached_versions_path = File.join( + Bundler.rubygems.user_home, ".bundle", "cache", "compact_index", + "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions" + ) + + # Verify a cached copy of the versions file exists + unless File.binread(cached_versions_path).size > 0 + raise("Cached versions file should be present and have content") + end + + # Verify that a partial request is made, starting from the index of the + # final byte of the cached file. + unless env.delete("HTTP_RANGE") + raise("Expected client to write the full response on the first try") + end + + etag_response do + file = tmp("versions.list") + FileUtils.rm_f(file) + file = CompactIndex::VersionsFile.new(file.to_s) + file.create(gems) + file.contents + end + end +end + +require_relative "helpers/artifice" + +Artifice.activate_with(CompactIndexRangeIgnored) diff --git a/spec/bundler/support/artifice/helpers/compact_index.rb b/spec/bundler/support/artifice/helpers/compact_index.rb index dd9e94ef9b1ef8..8e7acb41a90442 100644 --- a/spec/bundler/support/artifice/helpers/compact_index.rb +++ b/spec/bundler/support/artifice/helpers/compact_index.rb @@ -4,6 +4,7 @@ $LOAD_PATH.unshift Dir[Spec::Path.base_system_gem_path.join("gems/compact_index*/lib")].first.to_s require "compact_index" +require "digest" class CompactIndexAPI < Endpoint helpers do @@ -17,9 +18,10 @@ def load_spec(name, version, platform, gem_repo) def etag_response response_body = yield - checksum = Digest(:MD5).hexdigest(response_body) - return if not_modified?(checksum) - headers "ETag" => quote(checksum) + etag = Digest::MD5.hexdigest(response_body) + return if not_modified?(etag) + headers "ETag" => quote(etag) + headers "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest(response_body)}:" headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" content_type "text/plain" requested_range_for(response_body) @@ -29,11 +31,11 @@ def etag_response raise end - def not_modified?(checksum) + def not_modified?(etag) etags = parse_etags(request.env["HTTP_IF_NONE_MATCH"]) - return unless etags.include?(checksum) - headers "ETag" => quote(checksum) + return unless etags.include?(etag) + headers "ETag" => quote(etag) status 304 body "" end From a07d84b63c7785f6f1a0c5f6933c0b7fa87df1d8 Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Mon, 27 Nov 2023 10:34:36 +0000 Subject: [PATCH 46/78] [ruby/irb] Hide debugger hint after the input is submitted (https://github.com/ruby/irb/pull/789) If `output_modifier_proc`'s `complete` arg is true, it means the input is submitted. In that case, debugger hint doesn't provide value to users and adds noise to the output. So we hide it in such case. https://github.com/ruby/irb/commit/f86d9dbe2f --- lib/irb/debug.rb | 2 +- test/irb/yamatanooroti/test_rendering.rb | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/irb/debug.rb b/lib/irb/debug.rb index 514395605f8c03..1ec2335a8e9705 100644 --- a/lib/irb/debug.rb +++ b/lib/irb/debug.rb @@ -64,7 +64,7 @@ def DEBUGGER__.capture_frames(*args) unless output.strip.empty? cmd = output.split(/\s/, 2).first - if DEBUGGER__.commands.key?(cmd) + if !complete && DEBUGGER__.commands.key?(cmd) output = output.sub(/\n$/, " # debug command\n") end end diff --git a/test/irb/yamatanooroti/test_rendering.rb b/test/irb/yamatanooroti/test_rendering.rb index f9b4befbf5c8dc..ca8176dfe67c5b 100644 --- a/test/irb/yamatanooroti/test_rendering.rb +++ b/test/irb/yamatanooroti/test_rendering.rb @@ -389,11 +389,15 @@ def test_debug_integration_hints_debugger_commands script.close start_terminal(40, 80, %W{ruby -I#{@pwd}/lib #{script.to_path}}, startup_message: 'start IRB') write("debug\n") - write("n") + write("pp 1\n") + write("pp 1") close screen = result.join("\n").sub(/\n*\z/, "\n") - assert_include(screen, "irb:rdbg(main):002> n # debug command") + # submitted input shouldn't contain hint + assert_include(screen, "irb:rdbg(main):002> pp 1\n") + # unsubmitted input should contain hint + assert_include(screen, "irb:rdbg(main):003> pp 1 # debug command\n") ensure File.unlink(script) if script end From 1c3088117a6b22b6aa07fb8c9c28028851786c1b Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Fri, 24 Nov 2023 13:14:56 +0100 Subject: [PATCH 47/78] [ruby/stringio] Do not compile the C extension on TruffleRuby * Before this it was compiled but not used, because TruffleRuby has a stringio.rb in stdlib and .rb has precedence over .so. In fact that extension never worked on TruffleRuby, because rb_io_extract_modeenc() has never been defined on TruffleRuby. * So this just skip compiling the extension since compilation of it now fails: https://github.com/ruby/openssl/issues/699 https://github.com/ruby/stringio/commit/d791b63df6 --- ext/stringio/extconf.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ext/stringio/extconf.rb b/ext/stringio/extconf.rb index ad8650dce2cda4..553732f79cdba6 100644 --- a/ext/stringio/extconf.rb +++ b/ext/stringio/extconf.rb @@ -1,3 +1,7 @@ # frozen_string_literal: false require 'mkmf' -create_makefile('stringio') +if RUBY_ENGINE == 'ruby' + create_makefile('stringio') +else + File.write('Makefile', dummy_makefile("").join) +end From d7165d88ec715f4c73b6fcbbbd6b185145359757 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Nov 2023 10:45:50 -0500 Subject: [PATCH 48/78] Implement Write Barriers on Enumerator::Yielder --- enumerator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enumerator.c b/enumerator.c index 94f8490ca49dc0..602735119b600f 100644 --- a/enumerator.c +++ b/enumerator.c @@ -1302,7 +1302,7 @@ static const rb_data_type_t yielder_data_type = { NULL, yielder_compact, }, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; static struct yielder * @@ -1341,7 +1341,7 @@ yielder_init(VALUE obj, VALUE proc) rb_raise(rb_eArgError, "unallocated yielder"); } - ptr->proc = proc; + RB_OBJ_WRITE(obj, &ptr->proc, proc); return obj; } From 2dadd17c780108d9f98989578149a091ff2c0f50 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Nov 2023 10:49:28 -0500 Subject: [PATCH 49/78] Implement Write Barriers on Enumerator::Generator --- enumerator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/enumerator.c b/enumerator.c index 602735119b600f..8e86176c90a2e2 100644 --- a/enumerator.c +++ b/enumerator.c @@ -1434,7 +1434,7 @@ static const rb_data_type_t generator_data_type = { NULL, generator_compact, }, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; static struct generator * @@ -1474,7 +1474,7 @@ generator_init(VALUE obj, VALUE proc) rb_raise(rb_eArgError, "unallocated generator"); } - ptr->proc = proc; + RB_OBJ_WRITE(obj, &ptr->proc, proc); return obj; } @@ -1522,7 +1522,7 @@ generator_init_copy(VALUE obj, VALUE orig) rb_raise(rb_eArgError, "unallocated generator"); } - ptr1->proc = ptr0->proc; + RB_OBJ_WRITE(obj, &ptr1->proc, ptr0->proc); return obj; } @@ -1689,7 +1689,7 @@ lazy_generator_init(VALUE enumerator, VALUE procs) lazy_init_block, rb_ary_new3(2, obj, procs)); gen_ptr = generator_ptr(generator); - gen_ptr->obj = obj; + RB_OBJ_WRITE(generator, &gen_ptr->obj, obj); return generator; } From 2e4a0a4d90205224703a958fc9e4de61f6436aae Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 22 Nov 2023 10:54:32 -0500 Subject: [PATCH 50/78] Implement Write Barriers on Enumerator::Producer --- enumerator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/enumerator.c b/enumerator.c index 8e86176c90a2e2..a90a5943970d9f 100644 --- a/enumerator.c +++ b/enumerator.c @@ -2945,7 +2945,7 @@ static const rb_data_type_t producer_data_type = { producer_memsize, producer_compact, }, - 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_EMBEDDABLE + 0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_EMBEDDABLE }; static struct producer * @@ -2985,8 +2985,8 @@ producer_init(VALUE obj, VALUE init, VALUE proc) rb_raise(rb_eArgError, "unallocated producer"); } - ptr->init = init; - ptr->proc = proc; + RB_OBJ_WRITE(obj, &ptr->init, init); + RB_OBJ_WRITE(obj, &ptr->proc, proc); return obj; } From ca4755b59a7bdaaffbaef05474c8d8ee187e8d7d Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 28 Nov 2023 00:13:05 +0900 Subject: [PATCH 51/78] [Bug #20023] Resurrect fake string feature name before raising --- load.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/load.c b/load.c index 3dd5f044118e8c..780edf27a74844 100644 --- a/load.c +++ b/load.c @@ -935,7 +935,7 @@ load_unlock(rb_vm_t *vm, const char *ftptr, int done) } } -static VALUE rb_require_string_internal(VALUE fname); +static VALUE rb_require_string_internal(VALUE fname, bool resurrect); /* * call-seq: @@ -998,7 +998,7 @@ rb_f_require_relative(VALUE obj, VALUE fname) rb_loaderror("cannot infer basepath"); } base = rb_file_dirname(base); - return rb_require_string_internal(rb_file_absolute_path(fname, base)); + return rb_require_string_internal(rb_file_absolute_path(fname, base), false); } typedef int (*feature_func)(rb_vm_t *vm, const char *feature, const char *ext, int rb, int expanded, const char **fn); @@ -1336,11 +1336,11 @@ ruby_require_internal(const char *fname, unsigned int len) VALUE rb_require_string(VALUE fname) { - return rb_require_string_internal(FilePathValue(fname)); + return rb_require_string_internal(FilePathValue(fname), false); } static VALUE -rb_require_string_internal(VALUE fname) +rb_require_string_internal(VALUE fname, bool resurrect) { rb_execution_context_t *ec = GET_EC(); int result = require_internal(ec, fname, 1, RTEST(ruby_verbose)); @@ -1349,6 +1349,7 @@ rb_require_string_internal(VALUE fname) EC_JUMP_TAG(ec, result); } if (result < 0) { + if (resurrect) fname = rb_str_resurrect(fname); load_failed(fname); } @@ -1360,7 +1361,7 @@ rb_require(const char *fname) { struct RString fake; VALUE str = rb_setup_fake_str(&fake, fname, strlen(fname), 0); - return rb_require_string_internal(str); + return rb_require_string_internal(str, true); } static int From 7835ebce97a6e6132d2bc7bdbef115f3f47cc6c2 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 27 Nov 2023 10:23:31 -0500 Subject: [PATCH 52/78] Set compaction after major GC has been determined do_full_mark can change in gc_start, so we want to set auto-compaction only after do_full_mark has been properly set. --- gc.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gc.c b/gc.c index 2d7eb4de93af65..7daddd536439c3 100644 --- a/gc.c +++ b/gc.c @@ -9409,14 +9409,6 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) /* reason may be clobbered, later, so keep set immediate_sweep here */ objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); - /* Explicitly enable compaction (GC.compact) */ - if (do_full_mark && ruby_enable_autocompact) { - objspace->flags.during_compacting = TRUE; - } - else { - objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT); - } - if (!heap_allocated_pages) return FALSE; /* heap is not ready */ if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */ @@ -9464,6 +9456,14 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) objspace->flags.during_incremental_marking = do_full_mark; } + /* Explicitly enable compaction (GC.compact) */ + if (do_full_mark && ruby_enable_autocompact) { + objspace->flags.during_compacting = TRUE; + } + else { + objspace->flags.during_compacting = !!(reason & GPR_FLAG_COMPACT); + } + if (!GC_ENABLE_LAZY_SWEEP || objspace->flags.dont_incremental) { objspace->flags.immediate_sweep = TRUE; } From 196c4aeb766a66b3557ddab61086db58c7a08226 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 27 Nov 2023 20:15:54 +0900 Subject: [PATCH 53/78] [ruby/rdoc] Place a space between certain character class letters only https://github.com/ruby/rdoc/commit/1f568e049d --- lib/rdoc/markup/parser.rb | 2 +- lib/rdoc/markup/to_html.rb | 4 +++- lib/rdoc/text.rb | 6 ++++++ test/rdoc/test_rdoc_markup_to_html.rb | 28 +++++++++++++++++++++++---- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/lib/rdoc/markup/parser.rb b/lib/rdoc/markup/parser.rb index 0029df7e65e7f9..2ad4a658089acf 100644 --- a/lib/rdoc/markup/parser.rb +++ b/lib/rdoc/markup/parser.rb @@ -218,7 +218,7 @@ def build_paragraph margin break if peek_token.first == :BREAK - data << ' ' if skip :NEWLINE + data << ' ' if skip :NEWLINE and /#{SPACE_SEPARATED_LETTER_CLASS}\z/o.match?(data) else unget break diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index 6c9f5733a2dd3c..fb38924a0401b9 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -202,7 +202,9 @@ def accept_block_quote block_quote def accept_paragraph paragraph @res << "\n

" text = paragraph.text @hard_break - text = text.gsub(/\r?\n/, ' ') + text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K\r?\n(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) { + defined?($2) && ' ' + } @res << to_html(text) @res << "

\n" end diff --git a/lib/rdoc/text.rb b/lib/rdoc/text.rb index 0bc4aba4281d9e..6f1a2b8d1544df 100644 --- a/lib/rdoc/text.rb +++ b/lib/rdoc/text.rb @@ -309,4 +309,10 @@ def wrap(txt, line_len = 76) res.join.strip end + ## + # Character class to be separated by a space when concatenating + # lines. + + SPACE_SEPARATED_LETTER_CLASS = /[\p{Nd}\p{Lc}\p{Pc}]/ + end diff --git a/test/rdoc/test_rdoc_markup_to_html.rb b/test/rdoc/test_rdoc_markup_to_html.rb index 6897c8132ef355..2dd8cf922dc11f 100644 --- a/test/rdoc/test_rdoc_markup_to_html.rb +++ b/test/rdoc/test_rdoc_markup_to_html.rb @@ -257,7 +257,7 @@ def accept_paragraph_br end def accept_paragraph_break - assert_equal "\n

hello
world

\n", @to.res.join + assert_equal "\n

hello
world

\n", @to.res.join end def accept_paragraph_i @@ -391,11 +391,31 @@ def test_accept_heading_pipe end def test_accept_paragraph_newline - @to.start_accepting + hellos = ["hello", "\u{393 3b5 3b9 3ac} \u{3c3 3bf 3c5}"] + worlds = ["world", "\u{3ba 3cc 3c3 3bc 3bf 3c2}"] + ohayo, sekai = %W"\u{304a 306f 3088 3046} \u{4e16 754c}" + + hellos.product(worlds) do |hello, world| + @to.start_accepting + @to.accept_paragraph para("#{hello}\n", "#{world}\n") + assert_equal "\n

#{hello} #{world}

\n", @to.res.join + end + + hellos.each do |hello| + @to.start_accepting + @to.accept_paragraph para("#{hello}\n", "#{sekai}\n") + assert_equal "\n

#{hello}#{sekai}

\n", @to.res.join + end - @to.accept_paragraph para("hello\n", "world\n") + worlds.each do |world| + @to.start_accepting + @to.accept_paragraph para("#{ohayo}\n", "#{world}\n") + assert_equal "\n

#{ohayo}#{world}

\n", @to.res.join + end - assert_equal "\n

hello world

\n", @to.res.join + @to.start_accepting + @to.accept_paragraph para("#{ohayo}\n", "#{sekai}\n") + assert_equal "\n

#{ohayo}#{sekai}

\n", @to.res.join end def test_accept_heading_output_decoration From 8427a8a655e2a04bfdc6a645ec967405d3617137 Mon Sep 17 00:00:00 2001 From: KJ Tsanaktsidis Date: Mon, 27 Nov 2023 17:33:08 +1100 Subject: [PATCH 54/78] Fix flaky "Expected 499 to be >= 500" assertion in test_gc_compact.rb There have been some sproradically flaky tests related to GC compaction, which fail with: 1) Failure: TestGCCompact#test_moving_hashes_down_size_pools [/test/ruby/test_gc_compact.rb:442]: Expected 499 to be >= 500. What's happening here, is that, _sometimes_, depending on very unlucky combinations of machine things, one of the expected-to-be-moved hashes might be found on the machine stack during GC, and thus pinned. One factor which seems to make this _more_ likely is that GCC 11 on Ubuntu 22.04 seems to want to allocate 440 bytes of stack space for `gc_start`, which is much more than it actually uses on the common code path. The result is that there are some 50-odd VALUE-sized cells "live" on the stack which may well contain valid heap pointers from previous function calls, and will need to be pinned. This is, of course, totally normal and expected; Ruby's GC is conservative and if there is the possibility that a VALUE might be live on the machine stack, it can't be moved. However, it does make these tests flaky. This commit "fixes" the tests by performing the work in a fiber; the fiber goes out of scope and should be collected by the call to verify_compaction_references, so there should be no references to the to-be-moved objects floating around on the machine stack. Fixes [#20021] --- test/ruby/test_gc_compact.rb | 67 ++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index 8a3f1f145d5492..6d26811cbb93b3 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -317,14 +317,16 @@ def test_moving_arrays_down_size_pools GC.verify_compaction_references(expand_heap: true, toward: :empty) - arys = ARY_COUNT.times.map do - ary = "abbbbbbbbbb".chars - ary.uniq! - end + Fiber.new { + $arys = ARY_COUNT.times.map do + ary = "abbbbbbbbbb".chars + ary.uniq! + end + }.resume stats = GC.verify_compaction_references(expand_heap: true, toward: :empty) assert_operator(stats.dig(:moved_down, :T_ARRAY) || 0, :>=, ARY_COUNT) - refute_empty(arys.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) + refute_empty($arys.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) end; end @@ -337,22 +339,23 @@ def test_moving_arrays_up_size_pools GC.verify_compaction_references(expand_heap: true, toward: :empty) - ary = "hello".chars - arys = ARY_COUNT.times.map do - x = [] - ary.each { |e| x << e } - x - end + Fiber.new { + ary = "hello".chars + $arys = ARY_COUNT.times.map do + x = [] + ary.each { |e| x << e } + x + end + }.resume stats = GC.verify_compaction_references(expand_heap: true, toward: :empty) assert_operator(stats.dig(:moved_up, :T_ARRAY) || 0, :>=, ARY_COUNT) - refute_empty(arys.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) + refute_empty($arys.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) end; end def test_moving_objects_between_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 - omit "Flaky on Solaris" if /solaris/i =~ RUBY_PLATFORM assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) begin; @@ -368,16 +371,18 @@ def add_ivars GC.verify_compaction_references(expand_heap: true, toward: :empty) - ary = OBJ_COUNT.times.map { Foo.new } - ary.each(&:add_ivars) + Fiber.new { + $ary = OBJ_COUNT.times.map { Foo.new } + $ary.each(&:add_ivars) - GC.start - Foo.new.add_ivars + GC.start + Foo.new.add_ivars + }.resume stats = GC.verify_compaction_references(expand_heap: true, toward: :empty) assert_operator(stats.dig(:moved_up, :T_OBJECT) || 0, :>=, OBJ_COUNT) - refute_empty(ary.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) + refute_empty($ary.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) end; end @@ -390,13 +395,15 @@ def test_moving_strings_up_size_pools GC.verify_compaction_references(expand_heap: true, toward: :empty) - str = "a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] - ary = STR_COUNT.times.map { "" << str } + Fiber.new { + str = "a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + $ary = STR_COUNT.times.map { "" << str } + }.resume stats = GC.verify_compaction_references(expand_heap: true, toward: :empty) assert_operator(stats[:moved_up][:T_STRING], :>=, STR_COUNT) - refute_empty(ary.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) + refute_empty($ary.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) end; end @@ -409,12 +416,14 @@ def test_moving_strings_down_size_pools GC.verify_compaction_references(expand_heap: true, toward: :empty) - ary = STR_COUNT.times.map { ("a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).squeeze! } + Fiber.new { + $ary = STR_COUNT.times.map { ("a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).squeeze! } + }.resume stats = GC.verify_compaction_references(expand_heap: true, toward: :empty) assert_operator(stats[:moved_down][:T_STRING], :>=, STR_COUNT) - refute_empty(ary.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) + refute_empty($ary.keep_if { |o| ObjectSpace.dump(o).include?('"embedded":true') }) end; end @@ -422,10 +431,6 @@ def test_moving_hashes_down_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 # AR and ST hashes are in the same size pool on 32 bit omit unless RbConfig::SIZEOF["uint64_t"] <= RbConfig::SIZEOF["void*"] - # This test fails on Solaris SPARC with the following error and I can't figure out why: - # TestGCCompact#test_moving_hashes_down_size_pools - # Expected 499 to be >= 500. - omit if /sparc-solaris/ =~ RUBY_PLATFORM assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV) begin; @@ -433,9 +438,11 @@ def test_moving_hashes_down_size_pools GC.verify_compaction_references(expand_heap: true, toward: :empty) - base_hash = { a: 1, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8 } - ary = HASH_COUNT.times.map { base_hash.dup } - ary.each { |h| h[:i] = 9 } + Fiber.new { + base_hash = { a: 1, b: 2, c: 3, d: 4, e: 5, f: 6, g: 7, h: 8 } + $ary = HASH_COUNT.times.map { base_hash.dup } + $ary.each_with_index { |h, i| h[:i] = 9 } + }.resume stats = GC.verify_compaction_references(expand_heap: true, toward: :empty) From 7973eb7c3f6531a41adb9de63d1b3c4d5d4b7d03 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 28 Nov 2023 01:06:47 +0900 Subject: [PATCH 55/78] [ruby/rdoc] [DOC] Slightly decorate `em` and `strong` https://github.com/ruby/rdoc/commit/2161157205 --- lib/rdoc/generator/template/darkfish/css/rdoc.css | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/rdoc/generator/template/darkfish/css/rdoc.css b/lib/rdoc/generator/template/darkfish/css/rdoc.css index f845d6cecba0bd..2cc55e03b1b0f6 100644 --- a/lib/rdoc/generator/template/darkfish/css/rdoc.css +++ b/lib/rdoc/generator/template/darkfish/css/rdoc.css @@ -87,6 +87,17 @@ pre { border-radius: 0.2em; } +em { + text-decoration-color: rgba(52, 48, 64, 0.25); + text-decoration-line: underline; + text-decoration-style: dotted; +} + +strong, +em { + background-color: rgba(158, 178, 255, 0.1); +} + table { margin: 0; border-spacing: 0; From e3875dd0f8f11d9dbdc25b400f387c406b799cb5 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 27 Nov 2023 11:13:47 -0500 Subject: [PATCH 56/78] Don't incremental mark when GC stressful Incremental marking prevents the GC from fully executing, so it may fail to catch certain bugs. --- gc.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gc.c b/gc.c index 7daddd536439c3..eada017888ab85 100644 --- a/gc.c +++ b/gc.c @@ -9404,7 +9404,6 @@ static int gc_start(rb_objspace_t *objspace, unsigned int reason) { unsigned int do_full_mark = !!(reason & GPR_FLAG_FULL_MARK); - unsigned int immediate_mark = reason & GPR_FLAG_IMMEDIATE_MARK; /* reason may be clobbered, later, so keep set immediate_sweep here */ objspace->flags.immediate_sweep = !!(reason & GPR_FLAG_IMMEDIATE_SWEEP); @@ -9449,7 +9448,9 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) reason |= GPR_FLAG_MAJOR_BY_FORCE; /* GC by CAPI, METHOD, and so on. */ } - if (objspace->flags.dont_incremental || immediate_mark) { + if (objspace->flags.dont_incremental || + reason & GPR_FLAG_IMMEDIATE_MARK || + ruby_gc_stressful) { objspace->flags.during_incremental_marking = FALSE; } else { From 23a7714343b372234972ef0dacf774d07fe65ced Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 24 Nov 2023 13:18:00 +0100 Subject: [PATCH 57/78] Refactor and fix the GVL instrumentation API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This entirely changes how it is tested. Rather than to use counters we now record the timeline of events with associated threads which makes it much easier to assert that certains events are only preceded by a specific event, and makes it much easier to debug unexpected timelines. Co-Authored-By: Étienne Barrié Co-Authored-By: JP Camara Co-Authored-By: John Hawthorn --- .../thread/instrumentation/instrumentation.c | 213 ++++++++++++------ include/ruby/thread.h | 7 +- test/-ext-/thread/test_instrumentation_api.rb | 209 +++++++++++++---- thread.c | 4 - thread_pthread.c | 45 +++- 5 files changed, 345 insertions(+), 133 deletions(-) diff --git a/ext/-test-/thread/instrumentation/instrumentation.c b/ext/-test-/thread/instrumentation/instrumentation.c index 4d76f4cbb3b6d9..9703c045a6243e 100644 --- a/ext/-test-/thread/instrumentation/instrumentation.c +++ b/ext/-test-/thread/instrumentation/instrumentation.c @@ -2,85 +2,136 @@ #include "ruby/atomic.h" #include "ruby/thread.h" -static rb_atomic_t started_count = 0; -static rb_atomic_t ready_count = 0; -static rb_atomic_t resumed_count = 0; -static rb_atomic_t suspended_count = 0; -static rb_atomic_t exited_count = 0; - #ifndef RB_THREAD_LOCAL_SPECIFIER # define RB_THREAD_LOCAL_SPECIFIER #endif -static RB_THREAD_LOCAL_SPECIFIER unsigned int local_ready_count = 0; -static RB_THREAD_LOCAL_SPECIFIER unsigned int local_resumed_count = 0; -static RB_THREAD_LOCAL_SPECIFIER unsigned int local_suspended_count = 0; - static VALUE last_thread = Qnil; +static VALUE timeline_value = Qnil; + +struct thread_event { + VALUE thread; + rb_event_flag_t event; +}; + +#define MAX_EVENTS 1024 +static struct thread_event event_timeline[MAX_EVENTS]; +static rb_atomic_t timeline_cursor; + +static void +event_timeline_gc_mark(void *ptr) { + rb_atomic_t cursor; + for (cursor = 0; cursor < timeline_cursor; cursor++) { + rb_gc_mark(event_timeline[cursor].thread); + } +} + +static const rb_data_type_t event_timeline_type = { + "TestThreadInstrumentation/event_timeline", + {event_timeline_gc_mark, NULL, NULL,}, + 0, 0, + RUBY_TYPED_FREE_IMMEDIATELY, +}; + +static void +reset_timeline(void) +{ + timeline_cursor = 0; + memset(event_timeline, 0, sizeof(struct thread_event) * MAX_EVENTS); +} + +static rb_event_flag_t +find_last_event(VALUE thread) +{ + rb_atomic_t cursor = timeline_cursor; + if (cursor) { + do { + if (event_timeline[cursor].thread == thread){ + return event_timeline[cursor].event; + } + cursor--; + } while (cursor > 0); + } + return 0; +} + +static const char * +event_name(rb_event_flag_t event) +{ + switch (event) { + case RUBY_INTERNAL_THREAD_EVENT_STARTED: + return "started"; + case RUBY_INTERNAL_THREAD_EVENT_READY: + return "ready"; + case RUBY_INTERNAL_THREAD_EVENT_RESUMED: + return "resumed"; + case RUBY_INTERNAL_THREAD_EVENT_SUSPENDED: + return "suspended"; + case RUBY_INTERNAL_THREAD_EVENT_EXITED: + return "exited"; + } + return "no-event"; +} + +NORETURN(static void unexpected(const char *format, const char *event_name, VALUE thread)); + +static void +unexpected(const char *format, const char *event_name, VALUE thread) +{ +#if 0 + fprintf(stderr, "----------------\n"); + fprintf(stderr, format, event_name, thread); + fprintf(stderr, "\n"); + rb_backtrace(); + fprintf(stderr, "----------------\n"); +#else + rb_bug(format, event_name, thread); +#endif +} static void ex_callback(rb_event_flag_t event, const rb_internal_thread_event_data_t *event_data, void *user_data) { + rb_event_flag_t last_event = find_last_event(event_data->thread); + switch (event) { case RUBY_INTERNAL_THREAD_EVENT_STARTED: - last_thread = event_data->thread; - RUBY_ATOMIC_INC(started_count); + if (last_event != 0) { + unexpected("TestThreadInstrumentation: `started` event can't be preceded by `%s` (thread=%"PRIxVALUE")", event_name(last_event), event_data->thread); + } break; case RUBY_INTERNAL_THREAD_EVENT_READY: - RUBY_ATOMIC_INC(ready_count); - local_ready_count++; + if (last_event != 0 && last_event != RUBY_INTERNAL_THREAD_EVENT_STARTED && last_event != RUBY_INTERNAL_THREAD_EVENT_SUSPENDED) { + unexpected("TestThreadInstrumentation: `ready` must be preceded by `started` or `suspended`, got: `%s` (thread=%"PRIxVALUE")", event_name(last_event), event_data->thread); + } break; case RUBY_INTERNAL_THREAD_EVENT_RESUMED: - RUBY_ATOMIC_INC(resumed_count); - local_resumed_count++; + if (last_event != 0 && last_event != RUBY_INTERNAL_THREAD_EVENT_READY) { + unexpected("TestThreadInstrumentation: `resumed` must be preceded by `ready`, got: `%s` (thread=%"PRIxVALUE")", event_name(last_event), event_data->thread); + } break; case RUBY_INTERNAL_THREAD_EVENT_SUSPENDED: - RUBY_ATOMIC_INC(suspended_count); - local_suspended_count++; + if (last_event != 0 && last_event != RUBY_INTERNAL_THREAD_EVENT_RESUMED) { + unexpected("TestThreadInstrumentation: `suspended` must be preceded by `resumed`, got: `%s` (thread=%"PRIxVALUE")", event_name(last_event), event_data->thread); + } break; case RUBY_INTERNAL_THREAD_EVENT_EXITED: - RUBY_ATOMIC_INC(exited_count); + if (last_event != 0 && last_event != RUBY_INTERNAL_THREAD_EVENT_RESUMED && last_event != RUBY_INTERNAL_THREAD_EVENT_SUSPENDED) { + unexpected("TestThreadInstrumentation: `exited` must be preceded by `resumed` or `suspended`, got: `%s` (thread=%"PRIxVALUE")", event_name(last_event), event_data->thread); + } break; } -} - -static rb_internal_thread_event_hook_t * single_hook = NULL; -static VALUE -thread_counters(VALUE thread) -{ - VALUE array = rb_ary_new2(5); - rb_ary_push(array, UINT2NUM(started_count)); - rb_ary_push(array, UINT2NUM(ready_count)); - rb_ary_push(array, UINT2NUM(resumed_count)); - rb_ary_push(array, UINT2NUM(suspended_count)); - rb_ary_push(array, UINT2NUM(exited_count)); - return array; -} + rb_atomic_t cursor = RUBY_ATOMIC_FETCH_ADD(timeline_cursor, 1); + if (cursor >= MAX_EVENTS) { + rb_bug("TestThreadInstrumentation: ran out of event_timeline space"); + } -static VALUE -thread_local_counters(VALUE thread) -{ - VALUE array = rb_ary_new2(3); - rb_ary_push(array, UINT2NUM(local_ready_count)); - rb_ary_push(array, UINT2NUM(local_resumed_count)); - rb_ary_push(array, UINT2NUM(local_suspended_count)); - return array; + event_timeline[cursor].thread = event_data->thread; + event_timeline[cursor].event = event; } -static VALUE -thread_reset_counters(VALUE thread) -{ - RUBY_ATOMIC_SET(started_count, 0); - RUBY_ATOMIC_SET(ready_count, 0); - RUBY_ATOMIC_SET(resumed_count, 0); - RUBY_ATOMIC_SET(suspended_count, 0); - RUBY_ATOMIC_SET(exited_count, 0); - local_ready_count = 0; - local_resumed_count = 0; - local_suspended_count = 0; - return Qtrue; -} +static rb_internal_thread_event_hook_t * single_hook = NULL; static VALUE thread_register_callback(VALUE thread) @@ -98,6 +149,26 @@ thread_register_callback(VALUE thread) return Qnil; } +static VALUE +event_symbol(rb_event_flag_t event) +{ + switch (event) { + case RUBY_INTERNAL_THREAD_EVENT_STARTED: + return rb_id2sym(rb_intern("started")); + case RUBY_INTERNAL_THREAD_EVENT_READY: + return rb_id2sym(rb_intern("ready")); + case RUBY_INTERNAL_THREAD_EVENT_RESUMED: + return rb_id2sym(rb_intern("resumed")); + case RUBY_INTERNAL_THREAD_EVENT_SUSPENDED: + return rb_id2sym(rb_intern("suspended")); + case RUBY_INTERNAL_THREAD_EVENT_EXITED: + return rb_id2sym(rb_intern("exited")); + default: + rb_bug("TestThreadInstrumentation: Unexpected event"); + break; + } +} + static VALUE thread_unregister_callback(VALUE thread) { @@ -106,7 +177,18 @@ thread_unregister_callback(VALUE thread) single_hook = NULL; } - return Qnil; + VALUE events = rb_ary_new_capa(timeline_cursor); + rb_atomic_t cursor; + for (cursor = 0; cursor < timeline_cursor; cursor++) { + VALUE pair = rb_ary_new_capa(2); + rb_ary_push(pair, event_timeline[cursor].thread); + rb_ary_push(pair, event_symbol(event_timeline[cursor].event)); + rb_ary_push(events, pair); + } + + reset_timeline(); + + return events; } static VALUE @@ -125,31 +207,16 @@ thread_register_and_unregister_callback(VALUE thread) return Qtrue; } -static VALUE -thread_last_spawned(VALUE mod) -{ - return last_thread; -} - -static VALUE -thread_set_last_spawned(VALUE mod, VALUE value) -{ - return last_thread = value; -} - void Init_instrumentation(void) { VALUE mBug = rb_define_module("Bug"); VALUE klass = rb_define_module_under(mBug, "ThreadInstrumentation"); + rb_global_variable(&timeline_value); + timeline_value = TypedData_Wrap_Struct(0, &event_timeline_type, 0); + rb_global_variable(&last_thread); - rb_define_singleton_method(klass, "counters", thread_counters, 0); - rb_define_singleton_method(klass, "local_counters", thread_local_counters, 0); - rb_define_singleton_method(klass, "reset_counters", thread_reset_counters, 0); rb_define_singleton_method(klass, "register_callback", thread_register_callback, 0); rb_define_singleton_method(klass, "unregister_callback", thread_unregister_callback, 0); rb_define_singleton_method(klass, "register_and_unregister_callbacks", thread_register_and_unregister_callback, 0); - - rb_define_singleton_method(klass, "last_spawned_thread", thread_last_spawned, 0); - rb_define_singleton_method(klass, "last_spawned_thread=", thread_set_last_spawned, 1); } diff --git a/include/ruby/thread.h b/include/ruby/thread.h index f6eea65b702883..f11cc190868686 100644 --- a/include/ruby/thread.h +++ b/include/ruby/thread.h @@ -243,9 +243,12 @@ typedef struct rb_internal_thread_event_hook rb_internal_thread_event_hook_t; * @param[in] events A set of events that `func` should run. * @param[in] data Passed as-is to `func`. * @return An opaque pointer to the hook, to unregister it later. - * @note This functionality is a noop on Windows. + * @note This functionality is a noop on Windows and WebAssembly. * @note The callback will be called without the GVL held, except for the * RESUMED event. + * @note Callbacks are not guaranteed to be executed on the native threads + * that corresponds to the Ruby thread. To identify which Ruby thread + * the event refers to, you must use `event_data->thread`. * @warning This function MUST not be called from a thread event callback. */ rb_internal_thread_event_hook_t *rb_internal_thread_add_event_hook( @@ -258,7 +261,7 @@ rb_internal_thread_event_hook_t *rb_internal_thread_add_event_hook( * * @param[in] hook. The hook to unregister. * @return Wether the hook was found and unregistered. - * @note This functionality is a noop on Windows. + * @note This functionality is a noop on Windows and WebAssembly. * @warning This function MUST not be called from a thread event callback. */ bool rb_internal_thread_remove_event_hook( diff --git a/test/-ext-/thread/test_instrumentation_api.rb b/test/-ext-/thread/test_instrumentation_api.rb index 208d11de85b5c5..e2df02f81a28a6 100644 --- a/test/-ext-/thread/test_instrumentation_api.rb +++ b/test/-ext-/thread/test_instrumentation_api.rb @@ -7,75 +7,200 @@ def setup require '-test-/thread/instrumentation' - Thread.list.each do |thread| - if thread != Thread.current - thread.kill - thread.join rescue nil - end - end - assert_equal [Thread.current], Thread.list - - Bug::ThreadInstrumentation.reset_counters - Bug::ThreadInstrumentation::register_callback + cleanup_threads end def teardown return if /mswin|mingw|bccwin/ =~ RUBY_PLATFORM - Bug::ThreadInstrumentation::unregister_callback - Bug::ThreadInstrumentation.last_spawned_thread = nil + Bug::ThreadInstrumentation.unregister_callback + cleanup_threads end THREADS_COUNT = 3 - def test_thread_instrumentation - threads = threaded_cpu_work - assert_equal [false] * THREADS_COUNT, threads.map(&:status) - counters = Bug::ThreadInstrumentation.counters - assert_join_counters(counters) - assert_global_join_counters(counters) + def test_single_thread_timeline + thread = nil + full_timeline = record do + thread = Thread.new { 1 + 1 } + thread.join + end + assert_equal %i(started ready resumed suspended exited), timeline_for(thread, full_timeline) + ensure + thread&.kill + end + + def test_muti_thread_timeline + threads = nil + full_timeline = record do + threads = threaded_cpu_work + fib(20) + assert_equal [false] * THREADS_COUNT, threads.map(&:status) + end + + + threads.each do |thread| + timeline = timeline_for(thread, full_timeline) + assert_consistent_timeline(timeline) + end + + timeline = timeline_for(Thread.current, full_timeline) + assert_consistent_timeline(timeline) + ensure + threads&.each(&:kill) + end + + def test_join_suspends # Bug #18900 + thread = other_thread = nil + full_timeline = record do + other_thread = Thread.new { sleep 0.3 } + thread = Thread.new { other_thread.join } + thread.join + end + + timeline = timeline_for(thread, full_timeline) + assert_consistent_timeline(timeline) + assert_equal %i(started ready resumed suspended ready resumed suspended exited), timeline + ensure + other_thread&.kill + thread&.kill + end + + def test_io_release_gvl + r, w = IO.pipe + thread = nil + full_timeline = record do + thread = Thread.new do + w.write("Hello\n") + end + thread.join + end + + timeline = timeline_for(thread, full_timeline) + assert_consistent_timeline(timeline) + assert_equal %i(started ready resumed suspended ready resumed suspended exited), timeline + ensure + r&.close + w&.close + end + + def test_queue_releases_gvl + queue1 = Queue.new + queue2 = Queue.new + + thread = nil + + full_timeline = record do + thread = Thread.new do + queue1 << true + queue2.pop + end + + queue1.pop + queue2 << true + thread.join + end + + timeline = timeline_for(thread, full_timeline) + assert_consistent_timeline(timeline) + assert_equal %i(started ready resumed suspended ready resumed suspended exited), timeline end - def test_join_counters # Bug #18900 - thr = Thread.new { fib(30) } - Bug::ThreadInstrumentation.reset_counters - thr.join - assert_join_counters(Bug::ThreadInstrumentation.local_counters) + def test_thread_blocked_forever + mutex = Mutex.new + mutex.lock + thread = nil + + full_timeline = record do + thread = Thread.new do + mutex.lock + end + 10.times { Thread.pass } + sleep 0.1 + end + + mutex.unlock + thread.join + + timeline = timeline_for(thread, full_timeline) + assert_consistent_timeline(timeline) + assert_equal %i(started ready resumed), timeline end def test_thread_instrumentation_fork_safe skip "No fork()" unless Process.respond_to?(:fork) - thread_statuses = counters = nil + thread_statuses = full_timeline = nil IO.popen("-") do |read_pipe| if read_pipe thread_statuses = Marshal.load(read_pipe) - counters = Marshal.load(read_pipe) + full_timeline = Marshal.load(read_pipe) else - Bug::ThreadInstrumentation.reset_counters threads = threaded_cpu_work Marshal.dump(threads.map(&:status), STDOUT) - Marshal.dump(Bug::ThreadInstrumentation.counters, STDOUT) + full_timeline = Bug::ThreadInstrumentation.unregister_callback.map { |t, e| [t.to_s, e ] } + Marshal.dump(full_timeline, STDOUT) end end assert_predicate $?, :success? assert_equal [false] * THREADS_COUNT, thread_statuses - assert_join_counters(counters) - assert_global_join_counters(counters) + thread_names = full_timeline.map(&:first).uniq + thread_names.each do |thread_name| + assert_consistent_timeline(timeline_for(thread_name, full_timeline)) + end end def test_thread_instrumentation_unregister - Bug::ThreadInstrumentation::unregister_callback assert Bug::ThreadInstrumentation::register_and_unregister_callbacks end - def test_thread_instrumentation_event_data - assert_nil Bug::ThreadInstrumentation.last_spawned_thread - thr = Thread.new{ }.join - assert_same thr, Bug::ThreadInstrumentation.last_spawned_thread + private + + def record + Bug::ThreadInstrumentation.register_callback + yield + ensure + timeline = Bug::ThreadInstrumentation.unregister_callback + if $! + raise + else + return timeline + end + end + + def assert_consistent_timeline(events) + refute_predicate events, :empty? + + previous_event = nil + events.each do |event| + refute_equal :exited, previous_event, "`exited` must be the final event: #{events.inspect}" + case event + when :started + assert_nil previous_event, "`started` must be the first event: #{events.inspect}" + when :ready + unless previous_event.nil? + assert %i(started suspended).include?(previous_event), "`ready` must be preceded by `started` or `suspended`: #{events.inspect}" + end + when :resumed + unless previous_event.nil? + assert_equal :ready, previous_event, "`resumed` must be preceded by `ready`: #{events.inspect}" + end + when :suspended + unless previous_event.nil? + assert_equal :resumed, previous_event, "`suspended` must be preceded by `resumed`: #{events.inspect}" + end + when :exited + unless previous_event.nil? + assert %i(resumed suspended).include?(previous_event), "`exited` must be preceded by `resumed` or `suspended`: #{events.inspect}" + end + end + previous_event = event + end end - private + def timeline_for(thread, timeline) + timeline.select { |t, _| t == thread }.map(&:last) + end def fib(n = 20) return n if n <= 1 @@ -86,13 +211,13 @@ def threaded_cpu_work(size = 20) THREADS_COUNT.times.map { Thread.new { fib(size) } }.each(&:join) end - def assert_join_counters(counters) - counters.each_with_index do |c, i| - assert_operator c, :>, 0, "Call counters[#{i}]: #{counters.inspect}" + def cleanup_threads + Thread.list.each do |thread| + if thread != Thread.current + thread.kill + thread.join rescue nil + end end - end - - def assert_global_join_counters(counters) - assert_equal THREADS_COUNT, counters.first + assert_equal [Thread.current], Thread.list end end diff --git a/thread.c b/thread.c index 6c255ae15917ac..1e160d6ab9253e 100644 --- a/thread.c +++ b/thread.c @@ -5406,10 +5406,6 @@ Init_Thread(void) // it assumes blocked by thread_sched_to_waiting(). // thread_sched_to_running(sched, th); -#ifdef RB_INTERNAL_THREAD_HOOK - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th); -#endif - th->pending_interrupt_queue = rb_ary_hidden_new(0); th->pending_interrupt_queue_checked = 0; th->pending_interrupt_mask_stack = rb_ary_hidden_new(0); diff --git a/thread_pthread.c b/thread_pthread.c index c7319678c47253..f7619df46aa304 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -266,7 +266,34 @@ rb_native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, u static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL; static void rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th); -#define RB_INTERNAL_THREAD_HOOK(event, th) if (rb_internal_thread_event_hooks) { rb_thread_execute_hooks(event, th); } + +#if 0 +static const char * +event_name(rb_event_flag_t event) +{ + switch (event) { + case RUBY_INTERNAL_THREAD_EVENT_STARTED: + return "STARTED"; + case RUBY_INTERNAL_THREAD_EVENT_READY: + return "READY"; + case RUBY_INTERNAL_THREAD_EVENT_RESUMED: + return "RESUMED"; + case RUBY_INTERNAL_THREAD_EVENT_SUSPENDED: + return "SUSPENDED"; + case RUBY_INTERNAL_THREAD_EVENT_EXITED: + return "EXITED"; + } + return "no-event"; +} + +#define RB_INTERNAL_THREAD_HOOK(event, th) \ + if (UNLIKELY(rb_internal_thread_event_hooks)) { \ + fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \ + rb_thread_execute_hooks(event, th); \ + } +#else +#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); } +#endif static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */ @@ -958,9 +985,7 @@ thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, static void thread_sched_to_waiting_common0(struct rb_thread_sched *sched, rb_thread_t *th, bool to_dead) { - if (rb_internal_thread_event_hooks) { - rb_thread_execute_hooks(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - } + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); if (!to_dead) native_thread_dedicated_inc(th->vm, th->ractor, th->nt); @@ -975,9 +1000,8 @@ static void thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th) { RUBY_DEBUG_LOG("dedicated:%d", th->nt->dedicated); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th); - thread_sched_to_waiting_common0(sched, th, true); + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th); } // running -> dead @@ -1007,8 +1031,6 @@ thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th) static void thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th) { - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); - thread_sched_lock(sched, th); { thread_sched_to_waiting_common(sched, th); @@ -1046,6 +1068,7 @@ ubf_waiting(void *ptr) // not sleeping yet. } else { + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); thread_sched_to_ready_common(sched, th, true, false); } } @@ -1089,6 +1112,7 @@ thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th) thread_sched_lock(sched, th); { if (!ccan_list_empty(&sched->readyq)) { + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th)); bool can_direct_transfer = !th_has_dedicated_nt(th); thread_sched_to_ready_common(sched, th, false, can_direct_transfer); @@ -2148,8 +2172,6 @@ native_thread_create_dedicated(rb_thread_t *th) static void call_thread_start_func_2(rb_thread_t *th) { - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th); - #if defined USE_NATIVE_THREAD_INIT native_thread_init_stack(th); thread_start_func_2(th, th->ec->machine.stack_start); @@ -2309,6 +2331,7 @@ native_thread_create(rb_thread_t *th) { VM_ASSERT(th->nt == 0); RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt); + RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th); if (!th->ractor->threads.sched.enable_mn_threads) { th->has_dedicated_nt = 1; @@ -3232,7 +3255,6 @@ static void native_sleep(rb_thread_t *th, rb_hrtime_t *rel) { struct rb_thread_sched *sched = TH_SCHED(th); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th); RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0); if (rel) { @@ -3248,7 +3270,6 @@ native_sleep(rb_thread_t *th, rb_hrtime_t *rel) } RUBY_DEBUG_LOG("wakeup"); - RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th); } // thread internal event hooks (only for pthread) From acab060c17a21bd79f384e3e055aaa115c5dc235 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Nov 2023 18:17:51 +0100 Subject: [PATCH 58/78] Update to ruby/mspec@9f83eea --- .../mspec/lib/mspec/runner/actions/timeout.rb | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/spec/mspec/lib/mspec/runner/actions/timeout.rb b/spec/mspec/lib/mspec/runner/actions/timeout.rb index 499001c95265e1..1200926872b520 100644 --- a/spec/mspec/lib/mspec/runner/actions/timeout.rb +++ b/spec/mspec/lib/mspec/runner/actions/timeout.rb @@ -48,11 +48,12 @@ def start show_backtraces if MSpec.subprocesses.empty? - exit 2 + exit! 2 else # Do not exit but signal the subprocess so we can get their output MSpec.subprocesses.each do |pid| - Process.kill :SIGTERM, pid + kill_wait_one_second :SIGTERM, pid + hard_kill :SIGKILL, pid end @fail = true @current_state = nil @@ -80,7 +81,7 @@ def after(state = nil) if @fail STDERR.puts "\n\nThe last example #{@error_message}. See above for the subprocess stacktrace." - exit 2 + exit! 2 end end @@ -89,12 +90,28 @@ def finish @thread.join end + private def hard_kill(signal, pid) + begin + Process.kill signal, pid + rescue Errno::ESRCH + # Process already terminated + end + end + + private def kill_wait_one_second(signal, pid) + begin + Process.kill signal, pid + sleep 1 + rescue Errno::ESRCH + # Process already terminated + end + end + private def show_backtraces java_stacktraces = -> pid { if RUBY_ENGINE == 'truffleruby' || RUBY_ENGINE == 'jruby' STDERR.puts 'Java stacktraces:' - Process.kill :SIGQUIT, pid - sleep 1 + kill_wait_one_second :SIGQUIT, pid end } @@ -118,8 +135,7 @@ def finish if RUBY_ENGINE == 'truffleruby' STDERR.puts "\nRuby backtraces:" - Process.kill :SIGALRM, pid - sleep 1 + kill_wait_one_second :SIGALRM, pid else STDERR.puts "Don't know how to print backtraces of a subprocess on #{RUBY_ENGINE}" end From cc05a60c16b69b6156396f9e6a009f94421fe1b4 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 27 Nov 2023 18:17:52 +0100 Subject: [PATCH 59/78] Update to ruby/spec@c3206f6 --- spec/ruby/core/array/assoc_spec.rb | 12 +++++ spec/ruby/core/array/rassoc_spec.rb | 12 +++++ spec/ruby/core/exception/full_message_spec.rb | 43 +++++++++++++++ spec/ruby/core/file/new_spec.rb | 2 +- spec/ruby/core/file/open_spec.rb | 6 +-- spec/ruby/core/kernel/Integer_spec.rb | 15 ++++++ spec/ruby/core/method/parameters_spec.rb | 31 ++++++++++- spec/ruby/core/proc/parameters_spec.rb | 24 +++++++-- .../ruby/language/numbered_parameters_spec.rb | 13 +++++ spec/ruby/language/singleton_class_spec.rb | 7 +++ spec/ruby/library/date/iso8601_spec.rb | 21 ++++++++ spec/ruby/library/date/shared/parse.rb | 4 +- spec/ruby/library/date/shared/parse_eu.rb | 8 +-- spec/ruby/library/date/shared/parse_us.rb | 8 +-- .../library/digest/instance/shared/update.rb | 2 +- spec/ruby/library/etc/uname_spec.rb | 14 +++++ .../fixed_length_secure_compare_spec.rb | 42 +++++++++++++++ .../library/openssl/kdf/pbkdf2_hmac_spec.rb | 17 ------ .../library/openssl/secure_compare_spec.rb | 38 +++++++++++++ .../x509/{name => store}/verify_spec.rb | 2 +- spec/ruby/library/yaml/dump_spec.rb | 14 +++-- spec/ruby/library/yaml/dump_stream_spec.rb | 3 +- spec/ruby/library/yaml/fixtures/common.rb | 4 -- spec/ruby/library/yaml/load_file_spec.rb | 13 +++-- spec/ruby/library/yaml/load_stream_spec.rb | 3 +- spec/ruby/library/yaml/parse_file_spec.rb | 8 +-- spec/ruby/library/yaml/parse_spec.rb | 7 +-- .../ruby/library/yaml/shared/each_document.rb | 3 +- spec/ruby/library/yaml/shared/load.rb | 10 ++-- spec/ruby/library/yaml/to_yaml_spec.rb | 20 +++++-- spec/ruby/optional/capi/array_spec.rb | 34 ++++++++++++ spec/ruby/optional/capi/ext/array_spec.c | 34 ++++++++++++ spec/ruby/optional/capi/ext/encoding_spec.c | 3 ++ spec/ruby/optional/capi/ext/object_spec.c | 22 ++------ spec/ruby/optional/capi/ext/rbasic_spec.c | 54 ++++++++++++------- spec/ruby/optional/capi/ext/string_spec.c | 22 ++------ spec/ruby/optional/capi/ext/thread_spec.c | 4 +- spec/ruby/optional/capi/ext/typed_data_spec.c | 12 +++++ spec/ruby/optional/capi/rbasic_spec.rb | 2 + spec/ruby/optional/capi/shared/rbasic.rb | 1 - spec/ruby/optional/capi/typed_data_spec.rb | 12 +++++ spec/ruby/shared/sizedqueue/enque.rb | 2 +- 42 files changed, 480 insertions(+), 128 deletions(-) create mode 100644 spec/ruby/library/etc/uname_spec.rb create mode 100644 spec/ruby/library/openssl/fixed_length_secure_compare_spec.rb create mode 100644 spec/ruby/library/openssl/secure_compare_spec.rb rename spec/ruby/library/openssl/x509/{name => store}/verify_spec.rb (98%) delete mode 100644 spec/ruby/library/yaml/fixtures/common.rb diff --git a/spec/ruby/core/array/assoc_spec.rb b/spec/ruby/core/array/assoc_spec.rb index af95528281872f..f0be3de7957807 100644 --- a/spec/ruby/core/array/assoc_spec.rb +++ b/spec/ruby/core/array/assoc_spec.rb @@ -37,4 +37,16 @@ a.assoc(s1.first).should equal(s1) a.assoc(s2.first).should equal(s2) end + + it "calls to_ary on non-array elements" do + s1 = [1, 2] + s2 = ArraySpecs::ArrayConvertible.new(2, 3) + a = [s1, s2] + + s1.should_not_receive(:to_ary) + a.assoc(s1.first).should equal(s1) + + a.assoc(2).should == [2, 3] + s2.called.should equal(:to_ary) + end end diff --git a/spec/ruby/core/array/rassoc_spec.rb b/spec/ruby/core/array/rassoc_spec.rb index 62fbd40611fe2a..ed2851f1952cdb 100644 --- a/spec/ruby/core/array/rassoc_spec.rb +++ b/spec/ruby/core/array/rassoc_spec.rb @@ -35,4 +35,16 @@ def o.==(other); other == 'foobar'; end [[1, :foobar, o], [2, o, 1], [3, mock('foo')]].rassoc(key).should == [2, o, 1] end + + it "does not call to_ary on non-array elements" do + s1 = [1, 2] + s2 = ArraySpecs::ArrayConvertible.new(2, 3) + a = [s1, s2] + + s1.should_not_receive(:to_ary) + a.rassoc(2).should equal(s1) + + s2.should_not_receive(:to_ary) + a.rassoc(3).should equal(nil) + end end diff --git a/spec/ruby/core/exception/full_message_spec.rb b/spec/ruby/core/exception/full_message_spec.rb index e15649ca756159..4fad369936b285 100644 --- a/spec/ruby/core/exception/full_message_spec.rb +++ b/spec/ruby/core/exception/full_message_spec.rb @@ -46,6 +46,49 @@ full_message[0].should.end_with?("': Some runtime error (RuntimeError)\n") end + describe "includes details about whether an exception was handled" do + describe "RuntimeError" do + it "should report as unhandled if message is empty" do + err = RuntimeError.new("") + + err.full_message.should =~ /unhandled exception/ + err.full_message(highlight: true).should =~ /unhandled exception/ + err.full_message(highlight: false).should =~ /unhandled exception/ + end + + it "should not report as unhandled if the message is not empty" do + err = RuntimeError.new("non-empty") + + err.full_message.should !~ /unhandled exception/ + err.full_message(highlight: true).should !~ /unhandled exception/ + err.full_message(highlight: false).should !~ /unhandled exception/ + end + + it "should not report as unhandled if the message is nil" do + err = RuntimeError.new(nil) + + err.full_message.should !~ /unhandled exception/ + err.full_message(highlight: true).should !~ /unhandled exception/ + err.full_message(highlight: false).should !~ /unhandled exception/ + end + + it "should not report as unhandled if the message is not specified" do + err = RuntimeError.new() + + err.full_message.should !~ /unhandled exception/ + err.full_message(highlight: true).should !~ /unhandled exception/ + err.full_message(highlight: false).should !~ /unhandled exception/ + end + end + + describe "generic Error" do + it "should not report as unhandled in any event" do + StandardError.new("").full_message.should !~ /unhandled exception/ + StandardError.new("non-empty").full_message.should !~ /unhandled exception/ + end + end + end + it "shows the exception class at the end of the first line of the message when the message contains multiple lines" do begin line = __LINE__; raise "first line\nsecond line" diff --git a/spec/ruby/core/file/new_spec.rb b/spec/ruby/core/file/new_spec.rb index 3e2641aed3e19d..1e82a070b10dff 100644 --- a/spec/ruby/core/file/new_spec.rb +++ b/spec/ruby/core/file/new_spec.rb @@ -100,7 +100,7 @@ File.should.exist?(@file) end - it "raises an Errorno::EEXIST if the file exists when create a new file with File::CREAT|File::EXCL" do + it "raises an Errno::EEXIST if the file exists when create a new file with File::CREAT|File::EXCL" do -> { @fh = File.new(@file, File::CREAT|File::EXCL) }.should raise_error(Errno::EEXIST) end diff --git a/spec/ruby/core/file/open_spec.rb b/spec/ruby/core/file/open_spec.rb index 4c41f70e12b3f0..6bfc16bbf97782 100644 --- a/spec/ruby/core/file/open_spec.rb +++ b/spec/ruby/core/file/open_spec.rb @@ -354,7 +354,7 @@ end end - it "raises an Errorno::EEXIST if the file exists when open with File::CREAT|File::EXCL" do + it "raises an Errno::EEXIST if the file exists when open with File::CREAT|File::EXCL" do -> { File.open(@file, File::CREAT|File::EXCL) do |f| f.puts("writing") @@ -423,7 +423,7 @@ }.should raise_error(IOError) end - it "raises an Errorno::EEXIST if the file exists when open with File::RDONLY|File::TRUNC" do + it "raises an Errno::EEXIST if the file exists when open with File::RDONLY|File::TRUNC" do -> { File.open(@file, File::RDONLY|File::TRUNC) do |f| f.puts("writing").should == nil @@ -441,7 +441,7 @@ }.should raise_error(Errno::EINVAL) end - it "raises an Errorno::EEXIST if the file exists when open with File::RDONLY|File::TRUNC" do + it "raises an Errno::EEXIST if the file exists when open with File::RDONLY|File::TRUNC" do -> { File.open(@file, File::RDONLY|File::TRUNC) do |f| f.puts("writing").should == nil diff --git a/spec/ruby/core/kernel/Integer_spec.rb b/spec/ruby/core/kernel/Integer_spec.rb index c37733e88d213d..74dd3e0dd2ef27 100644 --- a/spec/ruby/core/kernel/Integer_spec.rb +++ b/spec/ruby/core/kernel/Integer_spec.rb @@ -586,6 +586,21 @@ Integer("777", obj).should == 0777 end + # https://bugs.ruby-lang.org/issues/19349 + ruby_version_is ''...'3.3' do + it "ignores the base if it is not an integer and does not respond to #to_i" do + Integer("777", "8").should == 777 + end + end + + ruby_version_is '3.3' do + it "raises a TypeError if it is not an integer and does not respond to #to_i" do + -> { + Integer("777", "8") + }.should raise_error(TypeError, "no implicit conversion of String into Integer") + end + end + describe "when passed exception: false" do describe "and valid argument" do it "returns an Integer number" do diff --git a/spec/ruby/core/method/parameters_spec.rb b/spec/ruby/core/method/parameters_spec.rb index 0f730fe0135a47..7d2b37fac75194 100644 --- a/spec/ruby/core/method/parameters_spec.rb +++ b/spec/ruby/core/method/parameters_spec.rb @@ -7,6 +7,7 @@ def one_key(a: 1); end def one_keyrest(**a); end def one_keyreq(a:); end + def one_nokey(**nil); end def one_splat_one_req(*a,b); end def one_splat_two_req(*a,b,c); end @@ -15,6 +16,7 @@ def one_splat_one_req_with_block(*a,b,&blk); end def one_opt_with_stabby(a=-> b { true }); end def one_unnamed_splat(*); end + def one_unnamed_keyrest(**); end def one_splat_one_block(*args, &block) local_is_not_parameter = {} @@ -178,6 +180,11 @@ def underscore_parameters(_, _, _ = 1, *_, _:, _: 2, **_, &_); end m.parameters.should == [[:keyreq,:a]] end + it "returns [[:nokey]] for a method with a single **nil parameter" do + m = MethodSpecs::Methods.instance_method(:one_nokey) + m.parameters.should == [[:nokey]] + end + it "works with ->(){} as the value of an optional argument" do m = MethodSpecs::Methods.instance_method(:one_opt_with_stabby) m.parameters.should == [[:opt,:a]] @@ -225,10 +232,15 @@ def underscore_parameters(_, _, _ = 1, *_, _:, _: 2, **_, &_); end end ruby_version_is '3.2' do - it "adds * rest arg for \"star\" argument" do + it "adds rest arg with name * for \"star\" argument" do m = MethodSpecs::Methods.new m.method(:one_unnamed_splat).parameters.should == [[:rest, :*]] end + + it "adds keyrest arg with ** as a name for \"double star\" argument" do + m = MethodSpecs::Methods.new + m.method(:one_unnamed_keyrest).parameters.should == [[:keyrest, :**]] + end end ruby_version_is ''...'3.2' do @@ -236,6 +248,23 @@ def underscore_parameters(_, _, _ = 1, *_, _:, _: 2, **_, &_); end m = MethodSpecs::Methods.new m.method(:one_unnamed_splat).parameters.should == [[:rest]] end + + it "adds nameless keyrest arg for \"double star\" argument" do + m = MethodSpecs::Methods.new + m.method(:one_unnamed_keyrest).parameters.should == [[:keyrest]] + end + end + + ruby_version_is '3.1' do + it "adds block arg with name & for anonymous block argument" do + object = Object.new + + eval(<<~RUBY).should == [[:block, :&]] + def object.foo(&) + end + object.method(:foo).parameters + RUBY + end end it "returns the args and block for a splat and block argument" do diff --git a/spec/ruby/core/proc/parameters_spec.rb b/spec/ruby/core/proc/parameters_spec.rb index 1ffaf173157798..2a4dcc36b351f1 100644 --- a/spec/ruby/core/proc/parameters_spec.rb +++ b/spec/ruby/core/proc/parameters_spec.rb @@ -21,7 +21,7 @@ end ruby_version_is "3.2" do - it "sets the first element of each sub-Array to :req if argument would be required if a lambda if lambda keyword used" do + it "sets the first element of each sub-Array to :req for required argument if lambda keyword used" do proc {|x| }.parameters(lambda: true).first.first.should == :req proc {|y,*x| }.parameters(lambda: true).first.first.should == :req end @@ -91,19 +91,33 @@ proc {|&block| }.parameters.first.last.should == :block end - it "ignores unnamed rest args" do + it "ignores unnamed rest arguments" do -> x {}.parameters.should == [[:req, :x]] end ruby_version_is '3.2' do - it "adds * rest arg for \"star\" argument" do - -> x, * {}.parameters.should == [[:req, :x], [:rest, :*]] + it "adds rest arg with name * for \"star\" argument" do + -> * {}.parameters.should == [[:rest, :*]] + end + + it "adds keyrest arg with ** as a name for \"double star\" argument" do + -> ** {}.parameters.should == [[:keyrest, :**]] end end ruby_version_is ''...'3.2' do it "adds nameless rest arg for \"star\" argument" do - -> x, * {}.parameters.should == [[:req, :x], [:rest]] + -> * {}.parameters.should == [[:rest]] + end + + it "adds nameless keyrest arg for \"double star\" argument" do + -> ** {}.parameters.should == [[:keyrest]] + end + end + + ruby_version_is '3.1' do + it "adds block arg with name & for anonymous block argument" do + eval('-> & {}.parameters').should == [[:block, :&]] end end diff --git a/spec/ruby/language/numbered_parameters_spec.rb b/spec/ruby/language/numbered_parameters_spec.rb index 9d2bd7ff92467b..3a35cf146560f2 100644 --- a/spec/ruby/language/numbered_parameters_spec.rb +++ b/spec/ruby/language/numbered_parameters_spec.rb @@ -82,6 +82,19 @@ lambda { _9 }.arity.should == 9 end + it "affects block parameters" do + -> { _1 }.parameters.should == [[:req, :_1]] + -> { _2 }.parameters.should == [[:req, :_1], [:req, :_2]] + + proc { _1 }.parameters.should == [[:opt, :_1]] + proc { _2 }.parameters.should == [[:opt, :_1], [:opt, :_2]] + end + + it "affects binding local variables" do + -> { _1; binding.local_variables }.call("a").should == [:_1] + -> { _2; binding.local_variables }.call("a", "b").should == [:_1, :_2] + end + it "does not work in methods" do obj = Object.new def obj.foo; _1 end diff --git a/spec/ruby/language/singleton_class_spec.rb b/spec/ruby/language/singleton_class_spec.rb index 7512f0eb398ffa..9d037717b24cb9 100644 --- a/spec/ruby/language/singleton_class_spec.rb +++ b/spec/ruby/language/singleton_class_spec.rb @@ -307,4 +307,11 @@ def singleton_method; 1 end o.freeze klass.frozen?.should == true end + + it "will be unfrozen if the frozen object is cloned with freeze set to false" do + o = Object.new + o.freeze + o2 = o.clone(freeze: false) + o2.singleton_class.frozen?.should == false + end end diff --git a/spec/ruby/library/date/iso8601_spec.rb b/spec/ruby/library/date/iso8601_spec.rb index a29652014ec2d1..af66845a6b4f81 100644 --- a/spec/ruby/library/date/iso8601_spec.rb +++ b/spec/ruby/library/date/iso8601_spec.rb @@ -22,6 +22,18 @@ d.should == Date.civil(-4712, 1, 1) end + it "raises a Date::Error if the argument is a invalid Date" do + -> { + Date.iso8601('invalid') + }.should raise_error(Date::Error, "invalid date") + end + + it "raises a Date::Error when passed a nil" do + -> { + Date.iso8601(nil) + }.should raise_error(Date::Error, "invalid date") + end + it "raises a TypeError when passed an Object" do -> { Date.iso8601(Object.new) }.should raise_error(TypeError) end @@ -32,4 +44,13 @@ h = Date._iso8601('invalid') h.should == {} end + + it "returns an empty hash if the argument is nil" do + h = Date._iso8601(nil) + h.should == {} + end + + it "raises a TypeError when passed an Object" do + -> { Date._iso8601(Object.new) }.should raise_error(TypeError) + end end diff --git a/spec/ruby/library/date/shared/parse.rb b/spec/ruby/library/date/shared/parse.rb index 1015285e048c07..40af908386c930 100644 --- a/spec/ruby/library/date/shared/parse.rb +++ b/spec/ruby/library/date/shared/parse.rb @@ -13,7 +13,7 @@ d.day.should == 23 end - it "can parse a 'mmm DD YYYY' string into a Date object" do + it "can parse a 'DD mmm YYYY' string into a Date object" do d = Date.parse("23#{@sep}feb#{@sep}2008") d.year.should == 2008 d.month.should == 2 @@ -42,7 +42,7 @@ d.should == Date.civil(2005, 11, 5) end - it "can parse a year, day and month name into a Date object" do + it "can parse a day, month name and year into a Date object" do d = Date.parse("5th#{@sep}november#{@sep}2005") d.should == Date.civil(2005, 11, 5) end diff --git a/spec/ruby/library/date/shared/parse_eu.rb b/spec/ruby/library/date/shared/parse_eu.rb index ecb15e3c0e4f76..3819524a571ef4 100644 --- a/spec/ruby/library/date/shared/parse_eu.rb +++ b/spec/ruby/library/date/shared/parse_eu.rb @@ -7,28 +7,28 @@ d.day.should == 1 end - it "can parse a MM-DD-YYYY string into a Date object" do + it "can parse a DD-MM-YYYY string into a Date object" do d = Date.parse("10#{@sep}01#{@sep}2007") d.year.should == 2007 d.month.should == 1 d.day.should == 10 end - it "can parse a MM-DD-YY string into a Date object" do + it "can parse a YY-MM-DD string into a Date object" do d = Date.parse("10#{@sep}01#{@sep}07") d.year.should == 2010 d.month.should == 1 d.day.should == 7 end - it "can parse a MM-DD-YY string into a Date object NOT using the year digits as 20XX" do + it "can parse a YY-MM-DD string into a Date object NOT using the year digits as 20XX" do d = Date.parse("10#{@sep}01#{@sep}07", false) d.year.should == 10 d.month.should == 1 d.day.should == 7 end - it "can parse a MM-DD-YY string into a Date object using the year digits as 20XX" do + it "can parse a YY-MM-DD string into a Date object using the year digits as 20XX" do d = Date.parse("10#{@sep}01#{@sep}07", true) d.year.should == 2010 d.month.should == 1 diff --git a/spec/ruby/library/date/shared/parse_us.rb b/spec/ruby/library/date/shared/parse_us.rb index 7be62b1af1fc2e..17e2fc96c18a65 100644 --- a/spec/ruby/library/date/shared/parse_us.rb +++ b/spec/ruby/library/date/shared/parse_us.rb @@ -6,28 +6,28 @@ d.day.should == 1 end - it "parses a MM#{@sep}DD#{@sep}YYYY string into a Date object" do + it "parses a DD#{@sep}MM#{@sep}YYYY string into a Date object" do d = Date.parse("10#{@sep}01#{@sep}2007") d.year.should == 2007 d.month.should == 1 d.day.should == 10 end - it "parses a MM#{@sep}DD#{@sep}YY string into a Date object" do + it "parses a YY#{@sep}MM#{@sep}DD string into a Date object" do d = Date.parse("10#{@sep}01#{@sep}07") d.year.should == 2010 d.month.should == 1 d.day.should == 7 end - it "parses a MM#{@sep}DD#{@sep}YY string into a Date object NOT using the year digits as 20XX" do + it "parses a YY#{@sep}MM#{@sep}DD string into a Date object NOT using the year digits as 20XX" do d = Date.parse("10#{@sep}01#{@sep}07", false) d.year.should == 10 d.month.should == 1 d.day.should == 7 end - it "parses a MM#{@sep}DD#{@sep}YY string into a Date object using the year digits as 20XX" do + it "parses a YY#{@sep}MM#{@sep}DD string into a Date object using the year digits as 20XX" do d = Date.parse("10#{@sep}01#{@sep}07", true) d.year.should == 2010 d.month.should == 1 diff --git a/spec/ruby/library/digest/instance/shared/update.rb b/spec/ruby/library/digest/instance/shared/update.rb index dccc8f80dfd693..17779e54a48fe5 100644 --- a/spec/ruby/library/digest/instance/shared/update.rb +++ b/spec/ruby/library/digest/instance/shared/update.rb @@ -3,6 +3,6 @@ c = Class.new do include Digest::Instance end - -> { c.new.update("test") }.should raise_error(RuntimeError) + -> { c.new.send(@method, "test") }.should raise_error(RuntimeError) end end diff --git a/spec/ruby/library/etc/uname_spec.rb b/spec/ruby/library/etc/uname_spec.rb new file mode 100644 index 00000000000000..a42558f5932b9c --- /dev/null +++ b/spec/ruby/library/etc/uname_spec.rb @@ -0,0 +1,14 @@ +require_relative '../../spec_helper' +require 'etc' + +describe "Etc.uname" do + it "returns a Hash with the documented keys" do + uname = Etc.uname + uname.should be_kind_of(Hash) + uname.should.key?(:sysname) + uname.should.key?(:nodename) + uname.should.key?(:release) + uname.should.key?(:version) + uname.should.key?(:machine) + end +end diff --git a/spec/ruby/library/openssl/fixed_length_secure_compare_spec.rb b/spec/ruby/library/openssl/fixed_length_secure_compare_spec.rb new file mode 100644 index 00000000000000..5a2ca168b5367d --- /dev/null +++ b/spec/ruby/library/openssl/fixed_length_secure_compare_spec.rb @@ -0,0 +1,42 @@ +require_relative '../../spec_helper' +require 'openssl' + +describe "OpenSSL.fixed_length_secure_compare" do + it "returns true for two strings with the same content" do + input1 = "the quick brown fox jumps over the lazy dog" + input2 = "the quick brown fox jumps over the lazy dog" + OpenSSL.fixed_length_secure_compare(input1, input2).should be_true + end + + it "returns false for two strings of equal size with different content" do + input1 = "the quick brown fox jumps over the lazy dog" + input2 = "the lazy dog jumps over the quick brown fox" + OpenSSL.fixed_length_secure_compare(input1, input2).should be_false + end + + it "converts both arguments to strings using #to_str" do + input1 = mock("input1") + input1.should_receive(:to_str).and_return("the quick brown fox jumps over the lazy dog") + input2 = mock("input2") + input2.should_receive(:to_str).and_return("the quick brown fox jumps over the lazy dog") + OpenSSL.fixed_length_secure_compare(input1, input2).should be_true + end + + it "does not accept arguments that are not string and cannot be coerced into strings" do + -> { + OpenSSL.fixed_length_secure_compare("input1", :input2) + }.should raise_error(TypeError, 'no implicit conversion of Symbol into String') + + -> { + OpenSSL.fixed_length_secure_compare(Object.new, "input2") + }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end + + it "raises an ArgumentError for two strings of different size" do + input1 = "the quick brown fox jumps over the lazy dog" + input2 = "the quick brown fox" + -> { + OpenSSL.fixed_length_secure_compare(input1, input2) + }.should raise_error(ArgumentError, 'inputs must be of equal length') + end +end diff --git a/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb b/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb index 000aa0c1139f1b..40f85972759a5a 100644 --- a/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb +++ b/spec/ruby/library/openssl/kdf/pbkdf2_hmac_spec.rb @@ -154,23 +154,6 @@ }.should raise_error(ArgumentError, 'missing keywords: :salt, :iterations, :length, :hash') end -=begin - guard -> { OpenSSL::OPENSSL_VERSION_NUMBER < 0x30000000 } do - it "treats 0 or less iterations as a single iteration" do - salt = "\x00".b * 16 - length = 16 - hash = "sha1" - - # "Any iter less than 1 is treated as a single iteration." - key0 = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: 0) - key_negative = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: -1) - key1 = OpenSSL::KDF.pbkdf2_hmac("secret", **@defaults, iterations: 1) - key0.should == key1 - key_negative.should == key1 - end - end -=end - guard -> { OpenSSL::OPENSSL_VERSION_NUMBER >= 0x30000000 } do it "raises an OpenSSL::KDF::KDFError for 0 or less iterations" do -> { diff --git a/spec/ruby/library/openssl/secure_compare_spec.rb b/spec/ruby/library/openssl/secure_compare_spec.rb new file mode 100644 index 00000000000000..cec48e01e7b7a8 --- /dev/null +++ b/spec/ruby/library/openssl/secure_compare_spec.rb @@ -0,0 +1,38 @@ +require_relative '../../spec_helper' +require 'openssl' + +describe "OpenSSL.secure_compare" do + it "returns true for two strings with the same content" do + input1 = "the quick brown fox jumps over the lazy dog" + input2 = "the quick brown fox jumps over the lazy dog" + OpenSSL.secure_compare(input1, input2).should be_true + end + + it "returns false for two strings with different content" do + input1 = "the quick brown fox jumps over the lazy dog" + input2 = "the lazy dog jumps over the quick brown fox" + OpenSSL.secure_compare(input1, input2).should be_false + end + + it "converts both arguments to strings using #to_str, but adds equality check for the original objects" do + input1 = mock("input1") + input1.should_receive(:to_str).and_return("the quick brown fox jumps over the lazy dog") + input2 = mock("input2") + input2.should_receive(:to_str).and_return("the quick brown fox jumps over the lazy dog") + OpenSSL.secure_compare(input1, input2).should be_false + + input = mock("input") + input.should_receive(:to_str).twice.and_return("the quick brown fox jumps over the lazy dog") + OpenSSL.secure_compare(input, input).should be_true + end + + it "does not accept arguments that are not string and cannot be coerced into strings" do + -> { + OpenSSL.secure_compare("input1", :input2) + }.should raise_error(TypeError, 'no implicit conversion of Symbol into String') + + -> { + OpenSSL.secure_compare(Object.new, "input2") + }.should raise_error(TypeError, 'no implicit conversion of Object into String') + end +end diff --git a/spec/ruby/library/openssl/x509/name/verify_spec.rb b/spec/ruby/library/openssl/x509/store/verify_spec.rb similarity index 98% rename from spec/ruby/library/openssl/x509/name/verify_spec.rb rename to spec/ruby/library/openssl/x509/store/verify_spec.rb index 6dcfc994663a58..6a6a53d992b104 100644 --- a/spec/ruby/library/openssl/x509/name/verify_spec.rb +++ b/spec/ruby/library/openssl/x509/store/verify_spec.rb @@ -1,7 +1,7 @@ require_relative '../../../../spec_helper' require 'openssl' -describe "OpenSSL::X509::Name.verify" do +describe "OpenSSL::X509::Store#verify" do it "returns true for valid certificate" do key = OpenSSL::PKey::RSA.new 2048 cert = OpenSSL::X509::Certificate.new diff --git a/spec/ruby/library/yaml/dump_spec.rb b/spec/ruby/library/yaml/dump_spec.rb index 3107a8f51d32cb..ea94b2f8565b69 100644 --- a/spec/ruby/library/yaml/dump_spec.rb +++ b/spec/ruby/library/yaml/dump_spec.rb @@ -1,17 +1,21 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' -# TODO: WTF is this using a global? +require 'yaml' + describe "YAML.dump" do + before :each do + @test_file = tmp("yaml_test_file") + end + after :each do - rm_r $test_file + rm_r @test_file end it "converts an object to YAML and write result to io when io provided" do - File.open($test_file, 'w' ) do |io| + File.open(@test_file, 'w' ) do |io| YAML.dump( ['badger', 'elephant', 'tiger'], io ) end - YAML.load_file($test_file).should == ['badger', 'elephant', 'tiger'] + YAML.load_file(@test_file).should == ['badger', 'elephant', 'tiger'] end it "returns a string containing dumped YAML when no io provided" do diff --git a/spec/ruby/library/yaml/dump_stream_spec.rb b/spec/ruby/library/yaml/dump_stream_spec.rb index 9d30fef819e505..f0578fa800e87e 100644 --- a/spec/ruby/library/yaml/dump_stream_spec.rb +++ b/spec/ruby/library/yaml/dump_stream_spec.rb @@ -1,5 +1,6 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' + +require 'yaml' describe "YAML.dump_stream" do it "returns a YAML stream containing the objects passed" do diff --git a/spec/ruby/library/yaml/fixtures/common.rb b/spec/ruby/library/yaml/fixtures/common.rb deleted file mode 100644 index 895213b844f087..00000000000000 --- a/spec/ruby/library/yaml/fixtures/common.rb +++ /dev/null @@ -1,4 +0,0 @@ -require 'yaml' - -$test_file = tmp("yaml_test_file") -$test_parse_file = __dir__ + "/test_yaml.yml" diff --git a/spec/ruby/library/yaml/load_file_spec.rb b/spec/ruby/library/yaml/load_file_spec.rb index 2363c081208628..4941d0485b3c19 100644 --- a/spec/ruby/library/yaml/load_file_spec.rb +++ b/spec/ruby/library/yaml/load_file_spec.rb @@ -1,13 +1,18 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' + +require 'yaml' describe "YAML.load_file" do + before :each do + @test_file = tmp("yaml_test_file") + end + after :each do - rm_r $test_file + rm_r @test_file end it "returns a hash" do - File.open($test_file,'w' ){|io| YAML.dump( {"bar"=>2, "car"=>1}, io ) } - YAML.load_file($test_file).should == {"bar"=>2, "car"=>1} + File.open(@test_file,'w' ){|io| YAML.dump( {"bar"=>2, "car"=>1}, io ) } + YAML.load_file(@test_file).should == {"bar"=>2, "car"=>1} end end diff --git a/spec/ruby/library/yaml/load_stream_spec.rb b/spec/ruby/library/yaml/load_stream_spec.rb index 689653c8cd64ac..31bc862f5e7479 100644 --- a/spec/ruby/library/yaml/load_stream_spec.rb +++ b/spec/ruby/library/yaml/load_stream_spec.rb @@ -1,8 +1,9 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' require_relative 'fixtures/strings' require_relative 'shared/each_document' +require 'yaml' + describe "YAML.load_stream" do it_behaves_like :yaml_each_document, :load_stream end diff --git a/spec/ruby/library/yaml/parse_file_spec.rb b/spec/ruby/library/yaml/parse_file_spec.rb index 8c59a2d7ef8b4a..7bffcdc62f9bf7 100644 --- a/spec/ruby/library/yaml/parse_file_spec.rb +++ b/spec/ruby/library/yaml/parse_file_spec.rb @@ -1,8 +1,10 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' -describe "YAML#parse_file" do +require 'yaml' + +describe "YAML.parse_file" do it "returns a YAML::Syck::Map object after parsing a YAML file" do - YAML.parse_file($test_parse_file).should be_kind_of(Psych::Nodes::Document) + test_parse_file = fixture __FILE__, "test_yaml.yml" + YAML.parse_file(test_parse_file).should be_kind_of(Psych::Nodes::Document) end end diff --git a/spec/ruby/library/yaml/parse_spec.rb b/spec/ruby/library/yaml/parse_spec.rb index d5dbfdcee25e57..37e2b7fa0a5420 100644 --- a/spec/ruby/library/yaml/parse_spec.rb +++ b/spec/ruby/library/yaml/parse_spec.rb @@ -1,13 +1,14 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' -describe "YAML#parse with an empty string" do +require 'yaml' + +describe "YAML.parse with an empty string" do it "returns false" do YAML.parse('').should be_false end end -describe "YAML#parse" do +describe "YAML.parse" do before :each do @string_yaml = "foo".to_yaml end diff --git a/spec/ruby/library/yaml/shared/each_document.rb b/spec/ruby/library/yaml/shared/each_document.rb index 999123dc2a4d15..7d32c6001fc6f5 100644 --- a/spec/ruby/library/yaml/shared/each_document.rb +++ b/spec/ruby/library/yaml/shared/each_document.rb @@ -9,7 +9,8 @@ end it "works on files" do - File.open($test_parse_file, "r") do |file| + test_parse_file = fixture __FILE__, "test_yaml.yml" + File.open(test_parse_file, "r") do |file| YAML.send(@method, file) do |doc| doc.should == {"project"=>{"name"=>"RubySpec"}} end diff --git a/spec/ruby/library/yaml/shared/load.rb b/spec/ruby/library/yaml/shared/load.rb index 185a5a60cd9ddc..1ebe08be2c2c2a 100644 --- a/spec/ruby/library/yaml/shared/load.rb +++ b/spec/ruby/library/yaml/shared/load.rb @@ -1,14 +1,16 @@ -require_relative '../fixtures/common' require_relative '../fixtures/strings' +require 'yaml' + describe :yaml_load_safe, shared: true do it "returns a document from current io stream when io provided" do - File.open($test_file, 'w') do |io| + @test_file = tmp("yaml_test_file") + File.open(@test_file, 'w') do |io| YAML.dump( ['badger', 'elephant', 'tiger'], io ) end - File.open($test_file) { |yf| YAML.send(@method, yf ) }.should == ['badger', 'elephant', 'tiger'] + File.open(@test_file) { |yf| YAML.send(@method, yf ) }.should == ['badger', 'elephant', 'tiger'] ensure - rm_r $test_file + rm_r @test_file end it "loads strings" do diff --git a/spec/ruby/library/yaml/to_yaml_spec.rb b/spec/ruby/library/yaml/to_yaml_spec.rb index 8e80b02cb453ed..547009c94233c7 100644 --- a/spec/ruby/library/yaml/to_yaml_spec.rb +++ b/spec/ruby/library/yaml/to_yaml_spec.rb @@ -1,7 +1,8 @@ require_relative '../../spec_helper' -require_relative 'fixtures/common' require_relative 'fixtures/example_class' +require 'yaml' + describe "Object#to_yaml" do it "returns the YAML representation of an Array object" do @@ -12,13 +13,21 @@ { "a" => "b"}.to_yaml.should match_yaml("--- \na: b\n") end - it "returns the YAML representation of a Class object" do + it "returns the YAML representation of an object" do YAMLSpecs::Example.new("baz").to_yaml.should match_yaml("--- !ruby/object:YAMLSpecs::Example\nname: baz\n") end + it "returns the YAML representation of a Class object" do + YAMLSpecs::Example.to_yaml.should match_yaml("--- !ruby/class 'YAMLSpecs::Example'\n") + end + + it "returns the YAML representation of a Module object" do + Enumerable.to_yaml.should match_yaml("--- !ruby/module 'Enumerable'\n") + end + it "returns the YAML representation of a Date object" do require 'date' - Date.parse('1997/12/30').to_yaml.should match_yaml("--- 1997-12-30\n") + Date.new(1997, 12, 30).to_yaml.should match_yaml("--- 1997-12-30\n") end it "returns the YAML representation of a FalseClass" do @@ -58,6 +67,11 @@ Person.new("Jane", "female").to_yaml.should match_yaml("--- !ruby/struct:Person\nname: Jane\ngender: female\n") end + it "returns the YAML representation of an unnamed Struct object" do + person = Struct.new(:name, :gender) + person.new("Jane", "female").to_yaml.should match_yaml("--- !ruby/struct\nname: Jane\ngender: female\n") + end + it "returns the YAML representation of a Symbol object" do :symbol.to_yaml.should match_yaml("--- :symbol\n") end diff --git a/spec/ruby/optional/capi/array_spec.rb b/spec/ruby/optional/capi/array_spec.rb index 8e90980c6a5551..9c35017e211ed6 100644 --- a/spec/ruby/optional/capi/array_spec.rb +++ b/spec/ruby/optional/capi/array_spec.rb @@ -343,6 +343,40 @@ end end + describe "rb_iterate" do + it "calls an callback function as a block passed to an method" do + s = [1,2,3,4] + s2 = @s.rb_iterate(s) + + s2.should == s + + # Make sure they're different objects + s2.equal?(s).should be_false + end + + it "calls a function with the other function available as a block" do + h = {a: 1, b: 2} + + @s.rb_iterate_each_pair(h).sort.should == [1,2] + end + + it "calls a function which can yield into the original block" do + s2 = [] + + o = Object.new + def o.each + yield 1 + yield 2 + yield 3 + yield 4 + end + + @s.rb_iterate_then_yield(o) { |x| s2 << x } + + s2.should == [1,2,3,4] + end + end + describe "rb_block_call" do it "calls an callback function as a block passed to an method" do s = [1,2,3,4] diff --git a/spec/ruby/optional/capi/ext/array_spec.c b/spec/ruby/optional/capi/ext/array_spec.c index 9386239813ea3f..8d5005c8911484 100644 --- a/spec/ruby/optional/capi/ext/array_spec.c +++ b/spec/ruby/optional/capi/ext/array_spec.c @@ -196,6 +196,18 @@ static VALUE copy_ary(RB_BLOCK_CALL_FUNC_ARGLIST(el, new_ary)) { return rb_ary_push(new_ary, el); } +// Suppress deprecations warnings for rb_iterate(), we want to test it while it exists +RBIMPL_WARNING_PUSH() +RBIMPL_WARNING_IGNORED(-Wdeprecated-declarations) + +static VALUE array_spec_rb_iterate(VALUE self, VALUE ary) { + VALUE new_ary = rb_ary_new(); + + rb_iterate(rb_each, ary, copy_ary, new_ary); + + return new_ary; +} + static VALUE array_spec_rb_block_call(VALUE self, VALUE ary) { VALUE new_ary = rb_ary_new(); @@ -208,6 +220,18 @@ static VALUE sub_pair(RB_BLOCK_CALL_FUNC_ARGLIST(el, holder)) { return rb_ary_push(holder, rb_ary_entry(el, 1)); } +static VALUE each_pair(VALUE obj) { + return rb_funcall(obj, rb_intern("each_pair"), 0); +} + +static VALUE array_spec_rb_iterate_each_pair(VALUE self, VALUE obj) { + VALUE new_ary = rb_ary_new(); + + rb_iterate(each_pair, obj, sub_pair, new_ary); + + return new_ary; +} + static VALUE array_spec_rb_block_call_each_pair(VALUE self, VALUE obj) { VALUE new_ary = rb_ary_new(); @@ -221,11 +245,18 @@ static VALUE iter_yield(RB_BLOCK_CALL_FUNC_ARGLIST(el, ary)) { return Qnil; } +static VALUE array_spec_rb_iterate_then_yield(VALUE self, VALUE obj) { + rb_iterate(rb_each, obj, iter_yield, obj); + return Qnil; +} + static VALUE array_spec_rb_block_call_then_yield(VALUE self, VALUE obj) { rb_block_call(obj, rb_intern("each"), 0, 0, iter_yield, obj); return Qnil; } +RBIMPL_WARNING_POP() + static VALUE array_spec_rb_mem_clear(VALUE self, VALUE obj) { VALUE ary[1]; ary[0] = obj; @@ -283,6 +314,9 @@ void Init_array_spec(void) { rb_define_method(cls, "rb_ary_plus", array_spec_rb_ary_plus, 2); rb_define_method(cls, "rb_ary_unshift", array_spec_rb_ary_unshift, 2); rb_define_method(cls, "rb_assoc_new", array_spec_rb_assoc_new, 2); + rb_define_method(cls, "rb_iterate", array_spec_rb_iterate, 1); + rb_define_method(cls, "rb_iterate_each_pair", array_spec_rb_iterate_each_pair, 1); + rb_define_method(cls, "rb_iterate_then_yield", array_spec_rb_iterate_then_yield, 1); rb_define_method(cls, "rb_block_call", array_spec_rb_block_call, 1); rb_define_method(cls, "rb_block_call_each_pair", array_spec_rb_block_call_each_pair, 1); rb_define_method(cls, "rb_block_call_then_yield", array_spec_rb_block_call_then_yield, 1); diff --git a/spec/ruby/optional/capi/ext/encoding_spec.c b/spec/ruby/optional/capi/ext/encoding_spec.c index 3343848b542958..4d2ff52ef397ca 100644 --- a/spec/ruby/optional/capi/ext/encoding_spec.c +++ b/spec/ruby/optional/capi/ext/encoding_spec.c @@ -271,12 +271,15 @@ static VALUE encoding_spec_rb_enc_str_asciionly_p(VALUE self, VALUE str) { } } +RBIMPL_WARNING_PUSH() +RBIMPL_WARNING_IGNORED(-Wformat-security) static VALUE encoding_spec_rb_enc_raise(VALUE self, VALUE encoding, VALUE exception_class, VALUE format) { rb_encoding *e = rb_to_encoding(encoding); const char *f = RSTRING_PTR(format); rb_enc_raise(e, exception_class, "%s", f); } +RBIMPL_WARNING_POP() static VALUE encoding_spec_rb_uv_to_utf8(VALUE self, VALUE buf, VALUE num) { int len = rb_uv_to_utf8(RSTRING_PTR(buf), NUM2INT(num)); diff --git a/spec/ruby/optional/capi/ext/object_spec.c b/spec/ruby/optional/capi/ext/object_spec.c index fbcf6d99626b48..4c19ec12c71974 100644 --- a/spec/ruby/optional/capi/ext/object_spec.c +++ b/spec/ruby/optional/capi/ext/object_spec.c @@ -154,28 +154,14 @@ static VALUE object_specs_rb_obj_method(VALUE self, VALUE obj, VALUE method) { return rb_obj_method(obj, method); } -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#elif defined(__clang__) && defined(__has_warning) -# if __has_warning("-Wdeprecated-declarations") -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdeprecated-declarations" -# endif -#endif - #ifndef RUBY_VERSION_IS_3_2 +// Suppress deprecations warnings for rb_obj_taint(), we want to test it while it exists +RBIMPL_WARNING_PUSH() +RBIMPL_WARNING_IGNORED(-Wdeprecated-declarations) static VALUE object_spec_rb_obj_taint(VALUE self, VALUE obj) { return rb_obj_taint(obj); } -#endif - -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) -# pragma GCC diagnostic pop -#elif defined(__clang__) && defined(__has_warning) -# if __has_warning("-Wdeprecated-declarations") -# pragma clang diagnostic pop -# endif +RBIMPL_WARNING_POP() #endif static VALUE so_require(VALUE self) { diff --git a/spec/ruby/optional/capi/ext/rbasic_spec.c b/spec/ruby/optional/capi/ext/rbasic_spec.c index 9178e5f6390089..26be2fed6d37e6 100644 --- a/spec/ruby/optional/capi/ext/rbasic_spec.c +++ b/spec/ruby/optional/capi/ext/rbasic_spec.c @@ -5,6 +5,14 @@ extern "C" { #endif +#ifndef RBASIC_FLAGS +#define RBASIC_FLAGS(obj) (RBASIC(obj)->flags) +#endif + +#ifndef RBASIC_SET_FLAGS +#define RBASIC_SET_FLAGS(obj, flags_to_set) (RBASIC(obj)->flags = flags_to_set) +#endif + #ifndef FL_SHAREABLE static const VALUE VISIBLE_BITS = FL_TAINT | FL_FREEZE; static const VALUE DATA_VISIBLE_BITS = FL_TAINT | FL_FREEZE | ~(FL_USER0 - 1); @@ -34,47 +42,53 @@ VALUE rbasic_spec_freeze_flag(VALUE self) { return VALUE2NUM(RUBY_FL_FREEZE); } - static VALUE spec_get_flags(const struct RBasic *b, VALUE visible_bits) { - VALUE flags = b->flags & visible_bits; +static VALUE spec_get_flags(VALUE obj, VALUE visible_bits) { + VALUE flags = RB_FL_TEST(obj, visible_bits); return VALUE2NUM(flags); } -static VALUE spec_set_flags(struct RBasic *b, VALUE flags, VALUE visible_bits) { +static VALUE spec_set_flags(VALUE obj, VALUE flags, VALUE visible_bits) { flags &= visible_bits; - b->flags = (b->flags & ~visible_bits) | flags; + + // Could also be done like: + // RB_FL_UNSET(obj, visible_bits); + // RB_FL_SET(obj, flags); + // But that seems rather indirect + RBASIC_SET_FLAGS(obj, (RBASIC_FLAGS(obj) & ~visible_bits) | flags); + return VALUE2NUM(flags); } -VALUE rbasic_spec_get_flags(VALUE self, VALUE val) { - return spec_get_flags(RBASIC(val), VISIBLE_BITS); +static VALUE rbasic_spec_get_flags(VALUE self, VALUE obj) { + return spec_get_flags(obj, VISIBLE_BITS); } -VALUE rbasic_spec_set_flags(VALUE self, VALUE val, VALUE flags) { - return spec_set_flags(RBASIC(val), NUM2VALUE(flags), VISIBLE_BITS); +static VALUE rbasic_spec_set_flags(VALUE self, VALUE obj, VALUE flags) { + return spec_set_flags(obj, NUM2VALUE(flags), VISIBLE_BITS); } -VALUE rbasic_spec_copy_flags(VALUE self, VALUE to, VALUE from) { - return spec_set_flags(RBASIC(to), RBASIC(from)->flags, VISIBLE_BITS); +static VALUE rbasic_spec_copy_flags(VALUE self, VALUE to, VALUE from) { + return spec_set_flags(to, RBASIC_FLAGS(from), VISIBLE_BITS); } -VALUE rbasic_spec_get_klass(VALUE self, VALUE val) { - return RBASIC(val)->klass; +static VALUE rbasic_spec_get_klass(VALUE self, VALUE obj) { + return RBASIC_CLASS(obj); } -VALUE rbasic_rdata_spec_get_flags(VALUE self, VALUE structure) { - return spec_get_flags(&RDATA(structure)->basic, DATA_VISIBLE_BITS); +static VALUE rbasic_rdata_spec_get_flags(VALUE self, VALUE structure) { + return spec_get_flags(structure, DATA_VISIBLE_BITS); } -VALUE rbasic_rdata_spec_set_flags(VALUE self, VALUE structure, VALUE flags) { - return spec_set_flags(&RDATA(structure)->basic, NUM2VALUE(flags), DATA_VISIBLE_BITS); +static VALUE rbasic_rdata_spec_set_flags(VALUE self, VALUE structure, VALUE flags) { + return spec_set_flags(structure, NUM2VALUE(flags), DATA_VISIBLE_BITS); } -VALUE rbasic_rdata_spec_copy_flags(VALUE self, VALUE to, VALUE from) { - return spec_set_flags(&RDATA(to)->basic, RDATA(from)->basic.flags, DATA_VISIBLE_BITS); +static VALUE rbasic_rdata_spec_copy_flags(VALUE self, VALUE to, VALUE from) { + return spec_set_flags(to, RBASIC_FLAGS(from), DATA_VISIBLE_BITS); } -VALUE rbasic_rdata_spec_get_klass(VALUE self, VALUE structure) { - return RDATA(structure)->basic.klass; +static VALUE rbasic_rdata_spec_get_klass(VALUE self, VALUE structure) { + return RBASIC_CLASS(structure); } void Init_rbasic_spec(void) { diff --git a/spec/ruby/optional/capi/ext/string_spec.c b/spec/ruby/optional/capi/ext/string_spec.c index 24a9b4e4ca0b2c..702620b9dacb02 100644 --- a/spec/ruby/optional/capi/ext/string_spec.c +++ b/spec/ruby/optional/capi/ext/string_spec.c @@ -254,17 +254,11 @@ VALUE string_spec_rb_str_new5(VALUE self, VALUE str, VALUE ptr, VALUE len) { return rb_str_new5(str, RSTRING_PTR(ptr), FIX2INT(len)); } -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#elif defined(__clang__) && defined(__has_warning) -# if __has_warning("-Wdeprecated-declarations") -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wdeprecated-declarations" -# endif -#endif - #ifndef RUBY_VERSION_IS_3_2 +// Suppress deprecations warnings for rb_tainted_str_new(), we want to test it while it exists +RBIMPL_WARNING_PUSH() +RBIMPL_WARNING_IGNORED(-Wdeprecated-declarations) + VALUE string_spec_rb_tainted_str_new(VALUE self, VALUE str, VALUE len) { return rb_tainted_str_new(RSTRING_PTR(str), FIX2INT(len)); } @@ -272,14 +266,8 @@ VALUE string_spec_rb_tainted_str_new(VALUE self, VALUE str, VALUE len) { VALUE string_spec_rb_tainted_str_new2(VALUE self, VALUE str) { return rb_tainted_str_new2(RSTRING_PTR(str)); } -#endif -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) -# pragma GCC diagnostic pop -#elif defined(__clang__) && defined(__has_warning) -# if __has_warning("-Wdeprecated-declarations") -# pragma clang diagnostic pop -# endif +RBIMPL_WARNING_POP() #endif VALUE string_spec_rb_str_plus(VALUE self, VALUE str1, VALUE str2) { diff --git a/spec/ruby/optional/capi/ext/thread_spec.c b/spec/ruby/optional/capi/ext/thread_spec.c index 6307e5cc99e1a9..14bd2079540254 100644 --- a/spec/ruby/optional/capi/ext/thread_spec.c +++ b/spec/ruby/optional/capi/ext/thread_spec.c @@ -26,9 +26,7 @@ static VALUE thread_spec_rb_thread_alone(VALUE self) { return rb_thread_alone() ? Qtrue : Qfalse; } -#if defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" -#endif +RBIMPL_WARNING_IGNORED(-Wdeprecated-declarations) /* This is unblocked by unblock_func(). */ static void* blocking_gvl_func(void* data) { diff --git a/spec/ruby/optional/capi/ext/typed_data_spec.c b/spec/ruby/optional/capi/ext/typed_data_spec.c index eca2b667cc227d..38889ecf5c8efa 100644 --- a/spec/ruby/optional/capi/ext/typed_data_spec.c +++ b/spec/ruby/optional/capi/ext/typed_data_spec.c @@ -106,6 +106,12 @@ VALUE sws_typed_wrap_struct(VALUE self, VALUE val) { return TypedData_Wrap_Struct(rb_cObject, &sample_typed_wrapped_struct_data_type, bar); } +VALUE sws_untyped_wrap_struct(VALUE self, VALUE val) { + int* data = (int*) malloc(sizeof(int)); + *data = FIX2INT(val); + return Data_Wrap_Struct(rb_cObject, NULL, free, data); +} + VALUE sws_typed_get_struct(VALUE self, VALUE obj) { struct sample_typed_wrapped_struct* bar; TypedData_Get_Struct(obj, struct sample_typed_wrapped_struct, &sample_typed_wrapped_struct_data_type, bar); @@ -165,12 +171,17 @@ VALUE sws_typed_rb_check_typeddata_different_type(VALUE self, VALUE obj) { return rb_check_typeddata(obj, &sample_typed_wrapped_struct_other_data_type) == DATA_PTR(obj) ? Qtrue : Qfalse; } +VALUE sws_typed_RTYPEDDATA_P(VALUE self, VALUE obj) { + return RTYPEDDATA_P(obj) ? Qtrue : Qfalse; +} + void Init_typed_data_spec(void) { VALUE cls = rb_define_class("CApiAllocTypedSpecs", rb_cObject); rb_define_alloc_func(cls, sdaf_alloc_typed_func); rb_define_method(cls, "typed_wrapped_data", sdaf_typed_get_struct, 0); cls = rb_define_class("CApiWrappedTypedStructSpecs", rb_cObject); rb_define_method(cls, "typed_wrap_struct", sws_typed_wrap_struct, 1); + rb_define_method(cls, "untyped_wrap_struct", sws_untyped_wrap_struct, 1); rb_define_method(cls, "typed_get_struct", sws_typed_get_struct, 1); rb_define_method(cls, "typed_get_struct_other", sws_typed_get_struct_different_type, 1); rb_define_method(cls, "typed_get_struct_parent", sws_typed_get_struct_parent_type, 1); @@ -181,6 +192,7 @@ void Init_typed_data_spec(void) { rb_define_method(cls, "rb_check_typeddata_same_type", sws_typed_rb_check_typeddata_same_type, 1); rb_define_method(cls, "rb_check_typeddata_same_type_parent", sws_typed_rb_check_typeddata_same_type_parent, 1); rb_define_method(cls, "rb_check_typeddata_different_type", sws_typed_rb_check_typeddata_different_type, 1); + rb_define_method(cls, "RTYPEDDATA_P", sws_typed_RTYPEDDATA_P, 1); } #ifdef __cplusplus diff --git a/spec/ruby/optional/capi/rbasic_spec.rb b/spec/ruby/optional/capi/rbasic_spec.rb index 577f2060dade53..f3367e05ffcf32 100644 --- a/spec/ruby/optional/capi/rbasic_spec.rb +++ b/spec/ruby/optional/capi/rbasic_spec.rb @@ -33,6 +33,8 @@ initial = @specs.get_flags(obj1) @specs.get_flags(obj2).should == initial @specs.set_flags(obj1, 1 << 14 | 1 << 16 | initial) + @specs.get_flags(obj1).should == 1 << 14 | 1 << 16 | initial + @specs.copy_flags(obj2, obj1) @specs.get_flags(obj2).should == 1 << 14 | 1 << 16 | initial @specs.set_flags(obj1, initial) diff --git a/spec/ruby/optional/capi/shared/rbasic.rb b/spec/ruby/optional/capi/shared/rbasic.rb index 95c313714364a6..9d80a93e1d6d15 100644 --- a/spec/ruby/optional/capi/shared/rbasic.rb +++ b/spec/ruby/optional/capi/shared/rbasic.rb @@ -1,5 +1,4 @@ describe :rbasic, shared: true do - before :all do specs = CApiRBasicSpecs.new @taint = ruby_version_is(''...'3.1') ? specs.taint_flag : 0 diff --git a/spec/ruby/optional/capi/typed_data_spec.rb b/spec/ruby/optional/capi/typed_data_spec.rb index 23b7c157efb3e0..6d1398a1a0b9ab 100644 --- a/spec/ruby/optional/capi/typed_data_spec.rb +++ b/spec/ruby/optional/capi/typed_data_spec.rb @@ -85,4 +85,16 @@ -> { @s.rb_check_typeddata_different_type(a) }.should raise_error(TypeError) end end + + describe "RTYPEDDATA_P" do + it "returns true for a typed data" do + a = @s.typed_wrap_struct(1024) + @s.RTYPEDDATA_P(a).should == true + end + + it "returns false for an untyped data object" do + a = @s.untyped_wrap_struct(1024) + @s.RTYPEDDATA_P(a).should == false + end + end end diff --git a/spec/ruby/shared/sizedqueue/enque.rb b/spec/ruby/shared/sizedqueue/enque.rb index 3bc8008fa4e982..7b6b1df7302f22 100644 --- a/spec/ruby/shared/sizedqueue/enque.rb +++ b/spec/ruby/shared/sizedqueue/enque.rb @@ -120,7 +120,7 @@ q << 1 t = Thread.new { - -> { q.send(@method, 1, timeout: 0.1) }.should raise_error(ClosedQueueError, "queue closed") + -> { q.send(@method, 1, timeout: 10) }.should raise_error(ClosedQueueError, "queue closed") } Thread.pass until q.num_waiting == 1 From bd4a992f38f59d15e325966a8e57f12559f331ad Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 27 Nov 2023 12:30:46 -0500 Subject: [PATCH 60/78] [ruby/prism] Correct template.rb comment https://github.com/ruby/prism/commit/4d689fe1df --- prism/templates/template.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prism/templates/template.rb b/prism/templates/template.rb index 9d3481e75d131a..626347ee2485ed 100755 --- a/prism/templates/template.rb +++ b/prism/templates/template.rb @@ -380,7 +380,8 @@ def template(name, write_to: nil) if (extension == ".c" || extension == ".h") && !contents.ascii_only? # Enforce that we only have ASCII characters here. This is necessary - # for some locales that only allow ASCII characters in C source files. + # for non-UTF-8 locales that only allow ASCII characters in C source + # files. contents.each_line.with_index(1) do |line, line_number| raise "Non-ASCII character on line #{line_number} of #{write_to}" unless line.ascii_only? end From 8654859dbd062af5071343effe48062123f356f7 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Sun, 26 Nov 2023 17:20:35 +0900 Subject: [PATCH 61/78] [ruby/prism] Fix and reuse pm_call_node_index_p Fix https://github.com/ruby/prism/pull/1925 Fix https://github.com/ruby/prism/pull/1927 Previously pm_call_node_index_p does not check about a block argument correctly and is not used in parse_write to check an index call node. This commit fixes these problems. https://github.com/ruby/prism/commit/92bab044ff --- prism/prism.c | 15 +- test/prism/errors_test.rb | 27 ++ test/prism/fixtures/arrays.txt | 8 + test/prism/snapshots/arrays.txt | 567 ++++++++++++++++++++------------ 4 files changed, 394 insertions(+), 223 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 7aee11c93c5d5f..27617f8f704d71 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -1840,8 +1840,8 @@ pm_call_node_variable_call_p(pm_call_node_t *node) { } /** - * Returns whether or not this call is to the [] method in the index form (as - * opposed to `foo.[]`). + * Returns whether or not this call is to the [] method in the index form without a block (as + * opposed to `foo.[]` and `foo[] { }`). */ static inline bool pm_call_node_index_p(pm_call_node_t *node) { @@ -1849,7 +1849,8 @@ pm_call_node_index_p(pm_call_node_t *node) { (node->call_operator_loc.start == NULL) && (node->message_loc.start != NULL) && (node->message_loc.start[0] == '[') && - (node->message_loc.end[-1] == ']') + (node->message_loc.end[-1] == ']') && + (node->block == NULL || PM_NODE_TYPE_P(node->block, PM_BLOCK_ARGUMENT_NODE)) ); } @@ -10827,13 +10828,7 @@ parse_write(pm_parser_t *parser, pm_node_t *target, pm_token_t *operator, pm_nod // If there is no call operator and the message is "[]" then this is // an aref expression, and we can transform it into an aset // expression. - if ( - (call->call_operator_loc.start == NULL) && - (call->message_loc.start != NULL) && - (call->message_loc.start[0] == '[') && - (call->message_loc.end[-1] == ']') && - (call->block == NULL) - ) { + if (pm_call_node_index_p(call)) { if (call->arguments == NULL) { call->arguments = pm_arguments_node_create(parser); } diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 6f6df71d1408aa..82f50edd6e1a84 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -1256,6 +1256,33 @@ def test_call_with_block_operator_write ] end + def test_index_call_with_block_and_write + source = "foo[1] {} &&= 1" + assert_errors expression(source), source, [ + ["Unexpected write target", 0..9], + ["Unexpected operator after a call with arguments", 10..13], + ["Unexpected operator after a call with a block", 10..13] + ] + end + + def test_index_call_with_block_or_write + source = "foo[1] {} ||= 1" + assert_errors expression(source), source, [ + ["Unexpected write target", 0..9], + ["Unexpected operator after a call with arguments", 10..13], + ["Unexpected operator after a call with a block", 10..13] + ] + end + + def test_index_call_with_block_operator_write + source = "foo[1] {} += 1" + assert_errors expression(source), source, [ + ["Unexpected write target", 0..9], + ["Unexpected operator after a call with arguments", 10..12], + ["Unexpected operator after a call with a block", 10..12] + ] + end + def test_writing_numbered_parameter assert_errors expression("-> { _1 = 0 }"), "-> { _1 = 0 }", [ ["_1 is reserved for a numbered parameter", 5..7] diff --git a/test/prism/fixtures/arrays.txt b/test/prism/fixtures/arrays.txt index bae1793e417415..31ffc58e64e4e2 100644 --- a/test/prism/fixtures/arrays.txt +++ b/test/prism/fixtures/arrays.txt @@ -81,6 +81,14 @@ foo[bar] = baz %w[\C:] +foo[&bar] = 1 + +foo.foo[&bar] = 1 + +def foo(&) + bar[&] = 1 +end + foo[] += 1 foo[] ||= 1 diff --git a/test/prism/snapshots/arrays.txt b/test/prism/snapshots/arrays.txt index 0038ec8ec0e370..8a031e909154e3 100644 --- a/test/prism/snapshots/arrays.txt +++ b/test/prism/snapshots/arrays.txt @@ -1,8 +1,8 @@ -@ ProgramNode (location: (1,0)-(118,24)) +@ ProgramNode (location: (1,0)-(126,24)) ├── locals: [] └── statements: - @ StatementsNode (location: (1,0)-(118,24)) - └── body: (length: 48) + @ StatementsNode (location: (1,0)-(126,24)) + └── body: (length: 51) ├── @ ArrayNode (location: (1,0)-(1,4)) │ ├── elements: (length: 1) │ │ └── @ SplatNode (location: (1,1)-(1,3)) @@ -907,7 +907,7 @@ │ ├── opening_loc: (82,0)-(82,3) = "%w[" │ ├── closing_loc: (82,6)-(82,7) = "]" │ └── flags: ∅ - ├── @ IndexOperatorWriteNode (location: (84,0)-(84,10)) + ├── @ CallNode (location: (84,0)-(84,13)) │ ├── receiver: │ │ @ CallNode (location: (84,0)-(84,3)) │ │ ├── receiver: ∅ @@ -920,716 +920,857 @@ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ + │ ├── name: :[]= + │ ├── message_loc: (84,3)-(84,9) = "[&bar]" │ ├── opening_loc: (84,3)-(84,4) = "[" + │ ├── arguments: + │ │ @ ArgumentsNode (location: (84,12)-(84,13)) + │ │ ├── arguments: (length: 1) + │ │ │ └── @ IntegerNode (location: (84,12)-(84,13)) + │ │ │ └── flags: decimal + │ │ └── flags: ∅ + │ ├── closing_loc: (84,8)-(84,9) = "]" + │ ├── block: + │ │ @ BlockArgumentNode (location: (84,4)-(84,8)) + │ │ ├── expression: + │ │ │ @ CallNode (location: (84,5)-(84,8)) + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :bar + │ │ │ ├── message_loc: (84,5)-(84,8) = "bar" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ ├── block: ∅ + │ │ │ └── flags: variable_call + │ │ └── operator_loc: (84,4)-(84,5) = "&" + │ └── flags: ∅ + ├── @ CallNode (location: (86,0)-(86,17)) + │ ├── receiver: + │ │ @ CallNode (location: (86,0)-(86,7)) + │ │ ├── receiver: + │ │ │ @ CallNode (location: (86,0)-(86,3)) + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :foo + │ │ │ ├── message_loc: (86,0)-(86,3) = "foo" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ ├── block: ∅ + │ │ │ └── flags: variable_call + │ │ ├── call_operator_loc: (86,3)-(86,4) = "." + │ │ ├── name: :foo + │ │ ├── message_loc: (86,4)-(86,7) = "foo" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: ∅ + │ │ ├── closing_loc: ∅ + │ │ ├── block: ∅ + │ │ └── flags: ∅ + │ ├── call_operator_loc: ∅ + │ ├── name: :[]= + │ ├── message_loc: (86,7)-(86,13) = "[&bar]" + │ ├── opening_loc: (86,7)-(86,8) = "[" + │ ├── arguments: + │ │ @ ArgumentsNode (location: (86,16)-(86,17)) + │ │ ├── arguments: (length: 1) + │ │ │ └── @ IntegerNode (location: (86,16)-(86,17)) + │ │ │ └── flags: decimal + │ │ └── flags: ∅ + │ ├── closing_loc: (86,12)-(86,13) = "]" + │ ├── block: + │ │ @ BlockArgumentNode (location: (86,8)-(86,12)) + │ │ ├── expression: + │ │ │ @ CallNode (location: (86,9)-(86,12)) + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :bar + │ │ │ ├── message_loc: (86,9)-(86,12) = "bar" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ ├── block: ∅ + │ │ │ └── flags: variable_call + │ │ └── operator_loc: (86,8)-(86,9) = "&" + │ └── flags: ∅ + ├── @ DefNode (location: (88,0)-(90,3)) + │ ├── name: :foo + │ ├── name_loc: (88,4)-(88,7) = "foo" + │ ├── receiver: ∅ + │ ├── parameters: + │ │ @ ParametersNode (location: (88,8)-(88,9)) + │ │ ├── requireds: (length: 0) + │ │ ├── optionals: (length: 0) + │ │ ├── rest: ∅ + │ │ ├── posts: (length: 0) + │ │ ├── keywords: (length: 0) + │ │ ├── keyword_rest: ∅ + │ │ └── block: + │ │ @ BlockParameterNode (location: (88,8)-(88,9)) + │ │ ├── name: ∅ + │ │ ├── name_loc: ∅ + │ │ └── operator_loc: (88,8)-(88,9) = "&" + │ ├── body: + │ │ @ StatementsNode (location: (89,2)-(89,12)) + │ │ └── body: (length: 1) + │ │ └── @ CallNode (location: (89,2)-(89,12)) + │ │ ├── receiver: + │ │ │ @ CallNode (location: (89,2)-(89,5)) + │ │ │ ├── receiver: ∅ + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :bar + │ │ │ ├── message_loc: (89,2)-(89,5) = "bar" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ ├── block: ∅ + │ │ │ └── flags: variable_call + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :[]= + │ │ ├── message_loc: (89,5)-(89,8) = "[&]" + │ │ ├── opening_loc: (89,5)-(89,6) = "[" + │ │ ├── arguments: + │ │ │ @ ArgumentsNode (location: (89,11)-(89,12)) + │ │ │ ├── arguments: (length: 1) + │ │ │ │ └── @ IntegerNode (location: (89,11)-(89,12)) + │ │ │ │ └── flags: decimal + │ │ │ └── flags: ∅ + │ │ ├── closing_loc: (89,7)-(89,8) = "]" + │ │ ├── block: + │ │ │ @ BlockArgumentNode (location: (89,6)-(89,7)) + │ │ │ ├── expression: ∅ + │ │ │ └── operator_loc: (89,6)-(89,7) = "&" + │ │ └── flags: ∅ + │ ├── locals: [:&] + │ ├── def_keyword_loc: (88,0)-(88,3) = "def" + │ ├── operator_loc: ∅ + │ ├── lparen_loc: (88,7)-(88,8) = "(" + │ ├── rparen_loc: (88,9)-(88,10) = ")" + │ ├── equal_loc: ∅ + │ └── end_keyword_loc: (90,0)-(90,3) = "end" + ├── @ IndexOperatorWriteNode (location: (92,0)-(92,10)) + │ ├── receiver: + │ │ @ CallNode (location: (92,0)-(92,3)) + │ │ ├── receiver: ∅ + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :foo + │ │ ├── message_loc: (92,0)-(92,3) = "foo" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: ∅ + │ │ ├── closing_loc: ∅ + │ │ ├── block: ∅ + │ │ └── flags: variable_call + │ ├── call_operator_loc: ∅ + │ ├── opening_loc: (92,3)-(92,4) = "[" │ ├── arguments: ∅ - │ ├── closing_loc: (84,4)-(84,5) = "]" + │ ├── closing_loc: (92,4)-(92,5) = "]" │ ├── block: ∅ │ ├── flags: ∅ │ ├── operator: :+ - │ ├── operator_loc: (84,6)-(84,8) = "+=" + │ ├── operator_loc: (92,6)-(92,8) = "+=" │ └── value: - │ @ IntegerNode (location: (84,9)-(84,10)) + │ @ IntegerNode (location: (92,9)-(92,10)) │ └── flags: decimal - ├── @ IndexOrWriteNode (location: (86,0)-(86,11)) + ├── @ IndexOrWriteNode (location: (94,0)-(94,11)) │ ├── receiver: - │ │ @ CallNode (location: (86,0)-(86,3)) + │ │ @ CallNode (location: (94,0)-(94,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (86,0)-(86,3) = "foo" + │ │ ├── message_loc: (94,0)-(94,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (86,3)-(86,4) = "[" + │ ├── opening_loc: (94,3)-(94,4) = "[" │ ├── arguments: ∅ - │ ├── closing_loc: (86,4)-(86,5) = "]" + │ ├── closing_loc: (94,4)-(94,5) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (86,6)-(86,9) = "||=" + │ ├── operator_loc: (94,6)-(94,9) = "||=" │ └── value: - │ @ IntegerNode (location: (86,10)-(86,11)) + │ @ IntegerNode (location: (94,10)-(94,11)) │ └── flags: decimal - ├── @ IndexAndWriteNode (location: (88,0)-(88,11)) + ├── @ IndexAndWriteNode (location: (96,0)-(96,11)) │ ├── receiver: - │ │ @ CallNode (location: (88,0)-(88,3)) + │ │ @ CallNode (location: (96,0)-(96,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (88,0)-(88,3) = "foo" + │ │ ├── message_loc: (96,0)-(96,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (88,3)-(88,4) = "[" + │ ├── opening_loc: (96,3)-(96,4) = "[" │ ├── arguments: ∅ - │ ├── closing_loc: (88,4)-(88,5) = "]" + │ ├── closing_loc: (96,4)-(96,5) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (88,6)-(88,9) = "&&=" + │ ├── operator_loc: (96,6)-(96,9) = "&&=" │ └── value: - │ @ IntegerNode (location: (88,10)-(88,11)) + │ @ IntegerNode (location: (96,10)-(96,11)) │ └── flags: decimal - ├── @ IndexOperatorWriteNode (location: (90,0)-(90,14)) + ├── @ IndexOperatorWriteNode (location: (98,0)-(98,14)) │ ├── receiver: - │ │ @ CallNode (location: (90,0)-(90,7)) + │ │ @ CallNode (location: (98,0)-(98,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (90,0)-(90,3)) + │ │ │ @ CallNode (location: (98,0)-(98,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (90,0)-(90,3) = "foo" + │ │ │ ├── message_loc: (98,0)-(98,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (90,3)-(90,4) = "." + │ │ ├── call_operator_loc: (98,3)-(98,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (90,4)-(90,7) = "foo" + │ │ ├── message_loc: (98,4)-(98,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (90,7)-(90,8) = "[" + │ ├── opening_loc: (98,7)-(98,8) = "[" │ ├── arguments: ∅ - │ ├── closing_loc: (90,8)-(90,9) = "]" + │ ├── closing_loc: (98,8)-(98,9) = "]" │ ├── block: ∅ │ ├── flags: ∅ │ ├── operator: :+ - │ ├── operator_loc: (90,10)-(90,12) = "+=" + │ ├── operator_loc: (98,10)-(98,12) = "+=" │ └── value: - │ @ IntegerNode (location: (90,13)-(90,14)) + │ @ IntegerNode (location: (98,13)-(98,14)) │ └── flags: decimal - ├── @ IndexOrWriteNode (location: (92,0)-(92,15)) + ├── @ IndexOrWriteNode (location: (100,0)-(100,15)) │ ├── receiver: - │ │ @ CallNode (location: (92,0)-(92,7)) + │ │ @ CallNode (location: (100,0)-(100,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (92,0)-(92,3)) + │ │ │ @ CallNode (location: (100,0)-(100,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (92,0)-(92,3) = "foo" + │ │ │ ├── message_loc: (100,0)-(100,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (92,3)-(92,4) = "." + │ │ ├── call_operator_loc: (100,3)-(100,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (92,4)-(92,7) = "foo" + │ │ ├── message_loc: (100,4)-(100,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (92,7)-(92,8) = "[" + │ ├── opening_loc: (100,7)-(100,8) = "[" │ ├── arguments: ∅ - │ ├── closing_loc: (92,8)-(92,9) = "]" + │ ├── closing_loc: (100,8)-(100,9) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (92,10)-(92,13) = "||=" + │ ├── operator_loc: (100,10)-(100,13) = "||=" │ └── value: - │ @ IntegerNode (location: (92,14)-(92,15)) + │ @ IntegerNode (location: (100,14)-(100,15)) │ └── flags: decimal - ├── @ IndexAndWriteNode (location: (94,0)-(94,15)) + ├── @ IndexAndWriteNode (location: (102,0)-(102,15)) │ ├── receiver: - │ │ @ CallNode (location: (94,0)-(94,7)) + │ │ @ CallNode (location: (102,0)-(102,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (94,0)-(94,3)) + │ │ │ @ CallNode (location: (102,0)-(102,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (94,0)-(94,3) = "foo" + │ │ │ ├── message_loc: (102,0)-(102,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (94,3)-(94,4) = "." + │ │ ├── call_operator_loc: (102,3)-(102,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (94,4)-(94,7) = "foo" + │ │ ├── message_loc: (102,4)-(102,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (94,7)-(94,8) = "[" + │ ├── opening_loc: (102,7)-(102,8) = "[" │ ├── arguments: ∅ - │ ├── closing_loc: (94,8)-(94,9) = "]" + │ ├── closing_loc: (102,8)-(102,9) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (94,10)-(94,13) = "&&=" + │ ├── operator_loc: (102,10)-(102,13) = "&&=" │ └── value: - │ @ IntegerNode (location: (94,14)-(94,15)) + │ @ IntegerNode (location: (102,14)-(102,15)) │ └── flags: decimal - ├── @ IndexOperatorWriteNode (location: (96,0)-(96,13)) + ├── @ IndexOperatorWriteNode (location: (104,0)-(104,13)) │ ├── receiver: - │ │ @ CallNode (location: (96,0)-(96,3)) + │ │ @ CallNode (location: (104,0)-(104,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (96,0)-(96,3) = "foo" + │ │ ├── message_loc: (104,0)-(104,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (96,3)-(96,4) = "[" + │ ├── opening_loc: (104,3)-(104,4) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (96,4)-(96,7)) + │ │ @ ArgumentsNode (location: (104,4)-(104,7)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (96,4)-(96,7)) + │ │ │ └── @ CallNode (location: (104,4)-(104,7)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (96,4)-(96,7) = "bar" + │ │ │ ├── message_loc: (104,4)-(104,7) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (96,7)-(96,8) = "]" + │ ├── closing_loc: (104,7)-(104,8) = "]" │ ├── block: ∅ │ ├── flags: ∅ │ ├── operator: :+ - │ ├── operator_loc: (96,9)-(96,11) = "+=" + │ ├── operator_loc: (104,9)-(104,11) = "+=" │ └── value: - │ @ IntegerNode (location: (96,12)-(96,13)) + │ @ IntegerNode (location: (104,12)-(104,13)) │ └── flags: decimal - ├── @ IndexOrWriteNode (location: (98,0)-(98,14)) + ├── @ IndexOrWriteNode (location: (106,0)-(106,14)) │ ├── receiver: - │ │ @ CallNode (location: (98,0)-(98,3)) + │ │ @ CallNode (location: (106,0)-(106,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (98,0)-(98,3) = "foo" + │ │ ├── message_loc: (106,0)-(106,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (98,3)-(98,4) = "[" + │ ├── opening_loc: (106,3)-(106,4) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (98,4)-(98,7)) + │ │ @ ArgumentsNode (location: (106,4)-(106,7)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (98,4)-(98,7)) + │ │ │ └── @ CallNode (location: (106,4)-(106,7)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (98,4)-(98,7) = "bar" + │ │ │ ├── message_loc: (106,4)-(106,7) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (98,7)-(98,8) = "]" + │ ├── closing_loc: (106,7)-(106,8) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (98,9)-(98,12) = "||=" + │ ├── operator_loc: (106,9)-(106,12) = "||=" │ └── value: - │ @ IntegerNode (location: (98,13)-(98,14)) + │ @ IntegerNode (location: (106,13)-(106,14)) │ └── flags: decimal - ├── @ IndexAndWriteNode (location: (100,0)-(100,14)) + ├── @ IndexAndWriteNode (location: (108,0)-(108,14)) │ ├── receiver: - │ │ @ CallNode (location: (100,0)-(100,3)) + │ │ @ CallNode (location: (108,0)-(108,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (100,0)-(100,3) = "foo" + │ │ ├── message_loc: (108,0)-(108,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (100,3)-(100,4) = "[" + │ ├── opening_loc: (108,3)-(108,4) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (100,4)-(100,7)) + │ │ @ ArgumentsNode (location: (108,4)-(108,7)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (100,4)-(100,7)) + │ │ │ └── @ CallNode (location: (108,4)-(108,7)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (100,4)-(100,7) = "bar" + │ │ │ ├── message_loc: (108,4)-(108,7) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (100,7)-(100,8) = "]" + │ ├── closing_loc: (108,7)-(108,8) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (100,9)-(100,12) = "&&=" + │ ├── operator_loc: (108,9)-(108,12) = "&&=" │ └── value: - │ @ IntegerNode (location: (100,13)-(100,14)) + │ @ IntegerNode (location: (108,13)-(108,14)) │ └── flags: decimal - ├── @ IndexOperatorWriteNode (location: (102,0)-(102,17)) + ├── @ IndexOperatorWriteNode (location: (110,0)-(110,17)) │ ├── receiver: - │ │ @ CallNode (location: (102,0)-(102,7)) + │ │ @ CallNode (location: (110,0)-(110,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (102,0)-(102,3)) + │ │ │ @ CallNode (location: (110,0)-(110,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (102,0)-(102,3) = "foo" + │ │ │ ├── message_loc: (110,0)-(110,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (102,3)-(102,4) = "." + │ │ ├── call_operator_loc: (110,3)-(110,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (102,4)-(102,7) = "foo" + │ │ ├── message_loc: (110,4)-(110,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (102,7)-(102,8) = "[" + │ ├── opening_loc: (110,7)-(110,8) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (102,8)-(102,11)) + │ │ @ ArgumentsNode (location: (110,8)-(110,11)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (102,8)-(102,11)) + │ │ │ └── @ CallNode (location: (110,8)-(110,11)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (102,8)-(102,11) = "bar" + │ │ │ ├── message_loc: (110,8)-(110,11) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (102,11)-(102,12) = "]" + │ ├── closing_loc: (110,11)-(110,12) = "]" │ ├── block: ∅ │ ├── flags: ∅ │ ├── operator: :+ - │ ├── operator_loc: (102,13)-(102,15) = "+=" + │ ├── operator_loc: (110,13)-(110,15) = "+=" │ └── value: - │ @ IntegerNode (location: (102,16)-(102,17)) + │ @ IntegerNode (location: (110,16)-(110,17)) │ └── flags: decimal - ├── @ IndexOrWriteNode (location: (104,0)-(104,18)) + ├── @ IndexOrWriteNode (location: (112,0)-(112,18)) │ ├── receiver: - │ │ @ CallNode (location: (104,0)-(104,7)) + │ │ @ CallNode (location: (112,0)-(112,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (104,0)-(104,3)) + │ │ │ @ CallNode (location: (112,0)-(112,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (104,0)-(104,3) = "foo" + │ │ │ ├── message_loc: (112,0)-(112,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (104,3)-(104,4) = "." + │ │ ├── call_operator_loc: (112,3)-(112,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (104,4)-(104,7) = "foo" + │ │ ├── message_loc: (112,4)-(112,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (104,7)-(104,8) = "[" + │ ├── opening_loc: (112,7)-(112,8) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (104,8)-(104,11)) + │ │ @ ArgumentsNode (location: (112,8)-(112,11)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (104,8)-(104,11)) + │ │ │ └── @ CallNode (location: (112,8)-(112,11)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (104,8)-(104,11) = "bar" + │ │ │ ├── message_loc: (112,8)-(112,11) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (104,11)-(104,12) = "]" + │ ├── closing_loc: (112,11)-(112,12) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (104,13)-(104,16) = "||=" + │ ├── operator_loc: (112,13)-(112,16) = "||=" │ └── value: - │ @ IntegerNode (location: (104,17)-(104,18)) + │ @ IntegerNode (location: (112,17)-(112,18)) │ └── flags: decimal - ├── @ IndexAndWriteNode (location: (106,0)-(106,18)) + ├── @ IndexAndWriteNode (location: (114,0)-(114,18)) │ ├── receiver: - │ │ @ CallNode (location: (106,0)-(106,7)) + │ │ @ CallNode (location: (114,0)-(114,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (106,0)-(106,3)) + │ │ │ @ CallNode (location: (114,0)-(114,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (106,0)-(106,3) = "foo" + │ │ │ ├── message_loc: (114,0)-(114,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (106,3)-(106,4) = "." + │ │ ├── call_operator_loc: (114,3)-(114,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (106,4)-(106,7) = "foo" + │ │ ├── message_loc: (114,4)-(114,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (106,7)-(106,8) = "[" + │ ├── opening_loc: (114,7)-(114,8) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (106,8)-(106,11)) + │ │ @ ArgumentsNode (location: (114,8)-(114,11)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (106,8)-(106,11)) + │ │ │ └── @ CallNode (location: (114,8)-(114,11)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (106,8)-(106,11) = "bar" + │ │ │ ├── message_loc: (114,8)-(114,11) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (106,11)-(106,12) = "]" + │ ├── closing_loc: (114,11)-(114,12) = "]" │ ├── block: ∅ │ ├── flags: ∅ - │ ├── operator_loc: (106,13)-(106,16) = "&&=" + │ ├── operator_loc: (114,13)-(114,16) = "&&=" │ └── value: - │ @ IntegerNode (location: (106,17)-(106,18)) + │ @ IntegerNode (location: (114,17)-(114,18)) │ └── flags: decimal - ├── @ IndexOperatorWriteNode (location: (108,0)-(108,19)) + ├── @ IndexOperatorWriteNode (location: (116,0)-(116,19)) │ ├── receiver: - │ │ @ CallNode (location: (108,0)-(108,3)) + │ │ @ CallNode (location: (116,0)-(116,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (108,0)-(108,3) = "foo" + │ │ ├── message_loc: (116,0)-(116,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (108,3)-(108,4) = "[" + │ ├── opening_loc: (116,3)-(116,4) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (108,4)-(108,7)) + │ │ @ ArgumentsNode (location: (116,4)-(116,7)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (108,4)-(108,7)) + │ │ │ └── @ CallNode (location: (116,4)-(116,7)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (108,4)-(108,7) = "bar" + │ │ │ ├── message_loc: (116,4)-(116,7) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (108,13)-(108,14) = "]" + │ ├── closing_loc: (116,13)-(116,14) = "]" │ ├── block: - │ │ @ BlockArgumentNode (location: (108,9)-(108,13)) + │ │ @ BlockArgumentNode (location: (116,9)-(116,13)) │ │ ├── expression: - │ │ │ @ CallNode (location: (108,10)-(108,13)) + │ │ │ @ CallNode (location: (116,10)-(116,13)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :baz - │ │ │ ├── message_loc: (108,10)-(108,13) = "baz" + │ │ │ ├── message_loc: (116,10)-(116,13) = "baz" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ └── operator_loc: (108,9)-(108,10) = "&" + │ │ └── operator_loc: (116,9)-(116,10) = "&" │ ├── flags: ∅ │ ├── operator: :+ - │ ├── operator_loc: (108,15)-(108,17) = "+=" + │ ├── operator_loc: (116,15)-(116,17) = "+=" │ └── value: - │ @ IntegerNode (location: (108,18)-(108,19)) + │ @ IntegerNode (location: (116,18)-(116,19)) │ └── flags: decimal - ├── @ IndexOrWriteNode (location: (110,0)-(110,20)) + ├── @ IndexOrWriteNode (location: (118,0)-(118,20)) │ ├── receiver: - │ │ @ CallNode (location: (110,0)-(110,3)) + │ │ @ CallNode (location: (118,0)-(118,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (110,0)-(110,3) = "foo" + │ │ ├── message_loc: (118,0)-(118,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (110,3)-(110,4) = "[" + │ ├── opening_loc: (118,3)-(118,4) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (110,4)-(110,7)) + │ │ @ ArgumentsNode (location: (118,4)-(118,7)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (110,4)-(110,7)) + │ │ │ └── @ CallNode (location: (118,4)-(118,7)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (110,4)-(110,7) = "bar" + │ │ │ ├── message_loc: (118,4)-(118,7) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (110,13)-(110,14) = "]" + │ ├── closing_loc: (118,13)-(118,14) = "]" │ ├── block: - │ │ @ BlockArgumentNode (location: (110,9)-(110,13)) + │ │ @ BlockArgumentNode (location: (118,9)-(118,13)) │ │ ├── expression: - │ │ │ @ CallNode (location: (110,10)-(110,13)) + │ │ │ @ CallNode (location: (118,10)-(118,13)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :baz - │ │ │ ├── message_loc: (110,10)-(110,13) = "baz" + │ │ │ ├── message_loc: (118,10)-(118,13) = "baz" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ └── operator_loc: (110,9)-(110,10) = "&" + │ │ └── operator_loc: (118,9)-(118,10) = "&" │ ├── flags: ∅ - │ ├── operator_loc: (110,15)-(110,18) = "||=" + │ ├── operator_loc: (118,15)-(118,18) = "||=" │ └── value: - │ @ IntegerNode (location: (110,19)-(110,20)) + │ @ IntegerNode (location: (118,19)-(118,20)) │ └── flags: decimal - ├── @ IndexAndWriteNode (location: (112,0)-(112,20)) + ├── @ IndexAndWriteNode (location: (120,0)-(120,20)) │ ├── receiver: - │ │ @ CallNode (location: (112,0)-(112,3)) + │ │ @ CallNode (location: (120,0)-(120,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (112,0)-(112,3) = "foo" + │ │ ├── message_loc: (120,0)-(120,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (112,3)-(112,4) = "[" + │ ├── opening_loc: (120,3)-(120,4) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (112,4)-(112,7)) + │ │ @ ArgumentsNode (location: (120,4)-(120,7)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (112,4)-(112,7)) + │ │ │ └── @ CallNode (location: (120,4)-(120,7)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (112,4)-(112,7) = "bar" + │ │ │ ├── message_loc: (120,4)-(120,7) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (112,13)-(112,14) = "]" + │ ├── closing_loc: (120,13)-(120,14) = "]" │ ├── block: - │ │ @ BlockArgumentNode (location: (112,9)-(112,13)) + │ │ @ BlockArgumentNode (location: (120,9)-(120,13)) │ │ ├── expression: - │ │ │ @ CallNode (location: (112,10)-(112,13)) + │ │ │ @ CallNode (location: (120,10)-(120,13)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :baz - │ │ │ ├── message_loc: (112,10)-(112,13) = "baz" + │ │ │ ├── message_loc: (120,10)-(120,13) = "baz" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ └── operator_loc: (112,9)-(112,10) = "&" + │ │ └── operator_loc: (120,9)-(120,10) = "&" │ ├── flags: ∅ - │ ├── operator_loc: (112,15)-(112,18) = "&&=" + │ ├── operator_loc: (120,15)-(120,18) = "&&=" │ └── value: - │ @ IntegerNode (location: (112,19)-(112,20)) + │ @ IntegerNode (location: (120,19)-(120,20)) │ └── flags: decimal - ├── @ IndexOperatorWriteNode (location: (114,0)-(114,23)) + ├── @ IndexOperatorWriteNode (location: (122,0)-(122,23)) │ ├── receiver: - │ │ @ CallNode (location: (114,0)-(114,7)) + │ │ @ CallNode (location: (122,0)-(122,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (114,0)-(114,3)) + │ │ │ @ CallNode (location: (122,0)-(122,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (114,0)-(114,3) = "foo" + │ │ │ ├── message_loc: (122,0)-(122,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (114,3)-(114,4) = "." + │ │ ├── call_operator_loc: (122,3)-(122,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (114,4)-(114,7) = "foo" + │ │ ├── message_loc: (122,4)-(122,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (114,7)-(114,8) = "[" + │ ├── opening_loc: (122,7)-(122,8) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (114,8)-(114,11)) + │ │ @ ArgumentsNode (location: (122,8)-(122,11)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (114,8)-(114,11)) + │ │ │ └── @ CallNode (location: (122,8)-(122,11)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (114,8)-(114,11) = "bar" + │ │ │ ├── message_loc: (122,8)-(122,11) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (114,17)-(114,18) = "]" + │ ├── closing_loc: (122,17)-(122,18) = "]" │ ├── block: - │ │ @ BlockArgumentNode (location: (114,13)-(114,17)) + │ │ @ BlockArgumentNode (location: (122,13)-(122,17)) │ │ ├── expression: - │ │ │ @ CallNode (location: (114,14)-(114,17)) + │ │ │ @ CallNode (location: (122,14)-(122,17)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :baz - │ │ │ ├── message_loc: (114,14)-(114,17) = "baz" + │ │ │ ├── message_loc: (122,14)-(122,17) = "baz" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ └── operator_loc: (114,13)-(114,14) = "&" + │ │ └── operator_loc: (122,13)-(122,14) = "&" │ ├── flags: ∅ │ ├── operator: :+ - │ ├── operator_loc: (114,19)-(114,21) = "+=" + │ ├── operator_loc: (122,19)-(122,21) = "+=" │ └── value: - │ @ IntegerNode (location: (114,22)-(114,23)) + │ @ IntegerNode (location: (122,22)-(122,23)) │ └── flags: decimal - ├── @ IndexOrWriteNode (location: (116,0)-(116,24)) + ├── @ IndexOrWriteNode (location: (124,0)-(124,24)) │ ├── receiver: - │ │ @ CallNode (location: (116,0)-(116,7)) + │ │ @ CallNode (location: (124,0)-(124,7)) │ │ ├── receiver: - │ │ │ @ CallNode (location: (116,0)-(116,3)) + │ │ │ @ CallNode (location: (124,0)-(124,3)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :foo - │ │ │ ├── message_loc: (116,0)-(116,3) = "foo" + │ │ │ ├── message_loc: (124,0)-(124,3) = "foo" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ ├── call_operator_loc: (116,3)-(116,4) = "." + │ │ ├── call_operator_loc: (124,3)-(124,4) = "." │ │ ├── name: :foo - │ │ ├── message_loc: (116,4)-(116,7) = "foo" + │ │ ├── message_loc: (124,4)-(124,7) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: ∅ │ ├── call_operator_loc: ∅ - │ ├── opening_loc: (116,7)-(116,8) = "[" + │ ├── opening_loc: (124,7)-(124,8) = "[" │ ├── arguments: - │ │ @ ArgumentsNode (location: (116,8)-(116,11)) + │ │ @ ArgumentsNode (location: (124,8)-(124,11)) │ │ ├── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (116,8)-(116,11)) + │ │ │ └── @ CallNode (location: (124,8)-(124,11)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :bar - │ │ │ ├── message_loc: (116,8)-(116,11) = "bar" + │ │ │ ├── message_loc: (124,8)-(124,11) = "bar" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call │ │ └── flags: ∅ - │ ├── closing_loc: (116,17)-(116,18) = "]" + │ ├── closing_loc: (124,17)-(124,18) = "]" │ ├── block: - │ │ @ BlockArgumentNode (location: (116,13)-(116,17)) + │ │ @ BlockArgumentNode (location: (124,13)-(124,17)) │ │ ├── expression: - │ │ │ @ CallNode (location: (116,14)-(116,17)) + │ │ │ @ CallNode (location: (124,14)-(124,17)) │ │ │ ├── receiver: ∅ │ │ │ ├── call_operator_loc: ∅ │ │ │ ├── name: :baz - │ │ │ ├── message_loc: (116,14)-(116,17) = "baz" + │ │ │ ├── message_loc: (124,14)-(124,17) = "baz" │ │ │ ├── opening_loc: ∅ │ │ │ ├── arguments: ∅ │ │ │ ├── closing_loc: ∅ │ │ │ ├── block: ∅ │ │ │ └── flags: variable_call - │ │ └── operator_loc: (116,13)-(116,14) = "&" + │ │ └── operator_loc: (124,13)-(124,14) = "&" │ ├── flags: ∅ - │ ├── operator_loc: (116,19)-(116,22) = "||=" + │ ├── operator_loc: (124,19)-(124,22) = "||=" │ └── value: - │ @ IntegerNode (location: (116,23)-(116,24)) + │ @ IntegerNode (location: (124,23)-(124,24)) │ └── flags: decimal - └── @ IndexAndWriteNode (location: (118,0)-(118,24)) + └── @ IndexAndWriteNode (location: (126,0)-(126,24)) ├── receiver: - │ @ CallNode (location: (118,0)-(118,7)) + │ @ CallNode (location: (126,0)-(126,7)) │ ├── receiver: - │ │ @ CallNode (location: (118,0)-(118,3)) + │ │ @ CallNode (location: (126,0)-(126,3)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :foo - │ │ ├── message_loc: (118,0)-(118,3) = "foo" + │ │ ├── message_loc: (126,0)-(126,3) = "foo" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call - │ ├── call_operator_loc: (118,3)-(118,4) = "." + │ ├── call_operator_loc: (126,3)-(126,4) = "." │ ├── name: :foo - │ ├── message_loc: (118,4)-(118,7) = "foo" + │ ├── message_loc: (126,4)-(126,7) = "foo" │ ├── opening_loc: ∅ │ ├── arguments: ∅ │ ├── closing_loc: ∅ │ ├── block: ∅ │ └── flags: ∅ ├── call_operator_loc: ∅ - ├── opening_loc: (118,7)-(118,8) = "[" + ├── opening_loc: (126,7)-(126,8) = "[" ├── arguments: - │ @ ArgumentsNode (location: (118,8)-(118,11)) + │ @ ArgumentsNode (location: (126,8)-(126,11)) │ ├── arguments: (length: 1) - │ │ └── @ CallNode (location: (118,8)-(118,11)) + │ │ └── @ CallNode (location: (126,8)-(126,11)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :bar - │ │ ├── message_loc: (118,8)-(118,11) = "bar" + │ │ ├── message_loc: (126,8)-(126,11) = "bar" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call │ └── flags: ∅ - ├── closing_loc: (118,17)-(118,18) = "]" + ├── closing_loc: (126,17)-(126,18) = "]" ├── block: - │ @ BlockArgumentNode (location: (118,13)-(118,17)) + │ @ BlockArgumentNode (location: (126,13)-(126,17)) │ ├── expression: - │ │ @ CallNode (location: (118,14)-(118,17)) + │ │ @ CallNode (location: (126,14)-(126,17)) │ │ ├── receiver: ∅ │ │ ├── call_operator_loc: ∅ │ │ ├── name: :baz - │ │ ├── message_loc: (118,14)-(118,17) = "baz" + │ │ ├── message_loc: (126,14)-(126,17) = "baz" │ │ ├── opening_loc: ∅ │ │ ├── arguments: ∅ │ │ ├── closing_loc: ∅ │ │ ├── block: ∅ │ │ └── flags: variable_call - │ └── operator_loc: (118,13)-(118,14) = "&" + │ └── operator_loc: (126,13)-(126,14) = "&" ├── flags: ∅ - ├── operator_loc: (118,19)-(118,22) = "&&=" + ├── operator_loc: (126,19)-(126,22) = "&&=" └── value: - @ IntegerNode (location: (118,23)-(118,24)) + @ IntegerNode (location: (126,23)-(126,24)) └── flags: decimal From 3af56e87ca79740521c81e1336cfb5523f55ee29 Mon Sep 17 00:00:00 2001 From: TSUYUSATO Kitsune Date: Sun, 26 Nov 2023 16:47:41 +0900 Subject: [PATCH 62/78] [ruby/prism] Check void expressions for constant paths Fix https://github.com/ruby/prism/pull/1920 https://github.com/ruby/prism/commit/ee8e03bac7 --- prism/prism.c | 2 ++ test/prism/errors_test.rb | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/prism/prism.c b/prism/prism.c index 27617f8f704d71..96ed3989e2ebc6 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -2457,6 +2457,8 @@ pm_constant_path_or_write_node_create(pm_parser_t *parser, pm_constant_path_node */ static pm_constant_path_node_t * pm_constant_path_node_create(pm_parser_t *parser, pm_node_t *parent, const pm_token_t *delimiter, pm_node_t *child) { + pm_assert_value_expression(parser, parent); + pm_constant_path_node_t *node = PM_ALLOC_NODE(parser, pm_constant_path_node_t); *node = (pm_constant_path_node_t) { diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 82f50edd6e1a84..3289f67a7180a8 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -1654,6 +1654,7 @@ def test_void_value_expression_in_call (return).(1) (return)[1] (return)[1] = 2 + (return)::foo RUBY message = 'Unexpected void value expression' assert_errors expression(source), source, [ @@ -1661,6 +1662,19 @@ def test_void_value_expression_in_call [message, 14..20], [message, 27..33], [message, 39..45], + [message, 55..61], + ], compare_ripper: false # Ripper does not check 'void value expression'. + end + + def test_void_value_expression_in_constant_path + source = <<~RUBY + (return)::A + class (return)::A; end + RUBY + message = 'Unexpected void value expression' + assert_errors expression(source), source, [ + [message, 1..7], + [message, 19..25], ], compare_ripper: false # Ripper does not check 'void value expression'. end From 150ed44d87e2d5b00f38019a487e05a67a303482 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 27 Nov 2023 11:30:47 -0500 Subject: [PATCH 63/78] Fix compaction during ary_make_partial The ary_make_shared call may allocate, which can trigger a GC compaction. This can cause the array to be embedded because it has a length of 0. --- array.c | 5 ++++- test/ruby/test_array.rb | 4 ++++ tool/lib/envutil.rb | 9 +++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/array.c b/array.c index b44c0c2adead47..a9f4990462dabd 100644 --- a/array.c +++ b/array.c @@ -1204,7 +1204,10 @@ ary_make_partial(VALUE ary, VALUE klass, long offset, long len) else { VALUE shared = ary_make_shared(ary); - assert(!ARY_EMBED_P(result)); + /* The ary_make_shared call may allocate, which can trigger a GC + * compaction. This can cause the array to be embedded because it has + * a length of 0. */ + FL_UNSET_EMBED(result); ARY_SET_PTR(result, RARRAY_CONST_PTR(ary)); ARY_SET_LEN(result, RARRAY_LEN(ary)); diff --git a/test/ruby/test_array.rb b/test/ruby/test_array.rb index 6c0db0832b93f1..838ef15b91d2ed 100644 --- a/test/ruby/test_array.rb +++ b/test/ruby/test_array.rb @@ -1693,6 +1693,10 @@ def test_slice_out_of_range assert_equal([100], a.slice(-1, 1_000_000_000)) end + def test_slice_gc_compact_stress + EnvUtil.under_gc_compact_stress { assert_equal([1, 2, 3, 4, 5], (0..10).to_a[1, 5]) } + end + def test_slice! a = @cls[1, 2, 3, 4, 5] assert_equal(3, a.slice!(2)) diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 1ca76d17a71a40..e47523a24b9a0c 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -245,6 +245,15 @@ def under_gc_stress(stress = true) end module_function :under_gc_stress + def under_gc_compact_stress(&block) + auto_compact = GC.auto_compact + GC.auto_compact = true + under_gc_stress(&block) + ensure + GC.auto_compact = auto_compact + end + module_function :under_gc_compact_stress + def with_default_external(enc) suppress_warning { Encoding.default_external = enc } yield From 6d447fa35f877edae96e4a7f98c9f5e70219314b Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 20 Nov 2023 16:16:22 -0500 Subject: [PATCH 64/78] [PRISM] Don't pop several args related nodes --- prism_compile.c | 12 ++++++------ test/ruby/test_compile_prism.rb | 9 +++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 6850c84c04fd9c..b66c2e0ed04828 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -1354,8 +1354,8 @@ pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinf case PM_ASSOC_NODE: { pm_assoc_node_t *assoc = (pm_assoc_node_t *)cur_node; - PM_COMPILE(assoc->key); - PM_COMPILE(assoc->value); + PM_COMPILE_NOT_POPPED(assoc->key); + PM_COMPILE_NOT_POPPED(assoc->value); cur_hash_size++; // If we're at the last keyword arg, or the last assoc node of this "set", @@ -1386,7 +1386,7 @@ pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinf } pm_assoc_splat_node_t *assoc_splat = (pm_assoc_splat_node_t *)cur_node; - PM_COMPILE(assoc_splat->value); + PM_COMPILE_NOT_POPPED(assoc_splat->value); *flags |= VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT; @@ -1411,14 +1411,14 @@ pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinf } else { *kw_arg = rb_xmalloc_mul_add(len, sizeof(VALUE), sizeof(struct rb_callinfo_kwarg)); - *flags |= VM_CALL_KWARG; - (*kw_arg)->keyword_len += len; + *flags = VM_CALL_KWARG; + (*kw_arg)->keyword_len = (int) len; // TODO: Method callers like `foo(a => b)` for (size_t i = 0; i < len; i++) { pm_assoc_node_t *assoc = (pm_assoc_node_t *)keyword_arg->elements.nodes[i]; (*kw_arg)->keywords[i] = pm_static_literal_value(assoc->key, scope_node, parser); - PM_COMPILE(assoc->value); + PM_COMPILE_NOT_POPPED(assoc->value); } } break; diff --git a/test/ruby/test_compile_prism.rb b/test/ruby/test_compile_prism.rb index aa140085f08989..a79db57bb59c69 100644 --- a/test/ruby/test_compile_prism.rb +++ b/test/ruby/test_compile_prism.rb @@ -804,6 +804,15 @@ def test_CallNode # with arguments and popped assert_prism_eval("eval '1'; 1") + + # With different types of calling arguments + assert_prism_eval(<<-CODE) + def self.prism_test_call_node(**); end + prism_test_call_node(b: 1, **{}) + CODE + assert_prism_eval(<<-CODE) + prism_test_call_node(:b => 1) + CODE end def test_CallAndWriteNode From 95064bb88db7ff0d71bf9c921b0b87fe1ae712d7 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Wed, 22 Nov 2023 09:49:26 -0500 Subject: [PATCH 65/78] [PRISM] Fix compilation for SplatNodes within ArrayNodes SplatNodes within ArrayNodes (e.g. [*1..2, 3]) need to be special cased in the compiler because they use a combination of concatarray and newarray instructions to treat each sequence of splat or non-splat elements as independent arrays which get concatenated. This commit implements those cases. --- prism_compile.c | 73 ++++++++++++++++++++++++++++++--- test/ruby/test_compile_prism.rb | 7 ++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index b66c2e0ed04828..fa5a7aafca3be6 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -1575,15 +1575,78 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_array_node_t *cast = (pm_array_node_t *) node; pm_node_list_t *elements = &cast->elements; - for (size_t index = 0; index < elements->size; index++) { - PM_COMPILE(elements->nodes[index]); + // In the case that there is a splat node within the array, + // the array gets compiled slightly differently. + if (node->flags & PM_ARRAY_NODE_FLAGS_CONTAINS_SPLAT) { + if (elements->size == 1) { + // If the only nodes is a SplatNode, we never + // need to emit the newarray or concatarray + // instructions + PM_COMPILE_NOT_POPPED(elements->nodes[0]); + } + else { + // We treat all sequences of non-splat elements as their + // own arrays, followed by a newarray, and then continually + // concat the arrays with the SplatNodes + int new_array_size = 0; + bool need_to_concat_array = false; + for (size_t index = 0; index < elements->size; index++) { + pm_node_t *array_element = elements->nodes[index]; + if (PM_NODE_TYPE_P(array_element, PM_SPLAT_NODE)) { + pm_splat_node_t *splat_element = (pm_splat_node_t *)array_element; + + // If we already have non-splat elements, we need to emit a newarray + // instruction + if (new_array_size) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(new_array_size)); + + // We don't want to emit a concat array in the case where + // we're seeing our first splat, and already have elements + if (need_to_concat_array) { + ADD_INSN(ret, &dummy_line_node, concatarray); + } + + new_array_size = 0; + } + + PM_COMPILE_NOT_POPPED(splat_element->expression); + + if (index > 0) { + ADD_INSN(ret, &dummy_line_node, concatarray); + } + else { + // If this is the first element, we need to splatarray + ADD_INSN1(ret, &dummy_line_node, splatarray, Qtrue); + } + + need_to_concat_array = true; + } + else { + new_array_size++; + PM_COMPILE_NOT_POPPED(array_element); + } + } + + if (new_array_size) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(new_array_size)); + if (need_to_concat_array) { + ADD_INSN(ret, &dummy_line_node, concatarray); + } + } + } + + PM_POP_IF_POPPED; } + else { + for (size_t index = 0; index < elements->size; index++) { + PM_COMPILE(elements->nodes[index]); + } - if (!popped) { - ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(elements->size)); + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(elements->size)); + } } } - return; } case PM_ASSOC_NODE: { diff --git a/test/ruby/test_compile_prism.rb b/test/ruby/test_compile_prism.rb index a79db57bb59c69..a01666ee54cbb6 100644 --- a/test/ruby/test_compile_prism.rb +++ b/test/ruby/test_compile_prism.rb @@ -488,6 +488,13 @@ def test_ArrayNode assert_prism_eval("[1, 2, 3]") assert_prism_eval("%i[foo bar baz]") assert_prism_eval("%w[foo bar baz]") + assert_prism_eval("[*1..2]") + assert_prism_eval("[*1..2, 3, 4, *5..6, 7, 8]") + assert_prism_eval("[*1..2, 3, 4, *5..6, 7, 8, *9..11]") + assert_prism_eval("[0, *1..2, 3, 4, *5..6, 7, 8, *9..11]") + assert_prism_eval("[-1, true, 0, *1..2, 3, 4, *5..6, 7, 8, *9..11]") + assert_prism_eval("a = [1,2]; [0, *a, 3, 4, *5..6, 7, 8, *9..11]") + assert_prism_eval("[[*1..2], 3, *4..5]") end def test_AssocNode From 872922b03d9be2a4d861c5eb8190be24498e3e9d Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 27 Nov 2023 14:04:56 -0500 Subject: [PATCH 66/78] Fix indentation in comment in shape.c --- shape.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shape.c b/shape.c index 7406e74dbf5a48..7fe21b1c587f02 100644 --- a/shape.c +++ b/shape.c @@ -806,7 +806,7 @@ shape_cache_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) } /* Verify the cache is correct by checking that this instance variable - * does not exist in the shape tree either. */ + * does not exist in the shape tree either. */ RUBY_ASSERT(!shape_get_iv_index(shape, id, value)); } From 4d71f70fd1cd0e11d9698e52411cd55045044c34 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 27 Nov 2023 14:05:25 -0500 Subject: [PATCH 67/78] Add assertions to check created red-black tree --- shape.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/shape.c b/shape.c index 7fe21b1c587f02..c969b78da5b5d1 100644 --- a/shape.c +++ b/shape.c @@ -409,6 +409,14 @@ redblack_cache_ancestors(rb_shape_t * shape) if (shape->type == SHAPE_IVAR) { shape->ancestor_index = redblack_insert(parent_index, shape->edge_name, shape); + +#if RUBY_DEBUG + if (shape->ancestor_index) { + redblack_node_t *inserted_node = redblack_find(shape->ancestor_index, shape->edge_name); + RUBY_ASSERT(inserted_node); + RUBY_ASSERT(redblack_value(inserted_node) == shape); + } +#endif } else { shape->ancestor_index = parent_index; From 13b7cddc2b7fb224065677b9c0030898ebbeb21e Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 27 Nov 2023 08:04:24 -0500 Subject: [PATCH 68/78] [PRISM] Compile IndexOrWriteNode --- prism_compile.c | 69 +++++++++++++++++++++++++++++++++ test/ruby/test_compile_prism.rb | 21 ++++++++++ 2 files changed, 90 insertions(+) diff --git a/prism_compile.c b/prism_compile.c index fa5a7aafca3be6..b1e30f741610c0 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -2652,6 +2652,75 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE(cast->value); return; } + case PM_INDEX_OR_WRITE_NODE: { + pm_index_or_write_node_t *index_or_write_node = (pm_index_or_write_node_t *)node; + + PM_PUTNIL_UNLESS_POPPED; + + PM_COMPILE_NOT_POPPED(index_or_write_node->receiver); + + int flag = 0; + struct rb_callinfo_kwarg *keywords = NULL; + int argc_int = 0; + + if (index_or_write_node->arguments) { + argc_int = pm_setup_args(index_or_write_node->arguments, &flag, &keywords, iseq, ret, src, popped, scope_node, dummy_line_node, parser); + } + + VALUE argc = INT2FIX(argc_int); + int boff = 0; + + if (index_or_write_node->block) { + PM_COMPILE_NOT_POPPED(index_or_write_node->block); + flag |= VM_CALL_ARGS_BLOCKARG; + boff = 1; + } + + ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + boff)); + + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idAREF, argc, INT2FIX(flag)); + + LABEL *label = NEW_LABEL(lineno); + LABEL *lfin = NEW_LABEL(lineno); + + PM_DUP; + ADD_INSNL(ret, &dummy_line_node, branchif, label); + PM_POP; + + PM_COMPILE_NOT_POPPED(index_or_write_node->value); + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + } + if (flag & VM_CALL_ARGS_SPLAT) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(1)); + if (boff > 0) { + ADD_INSN1(ret, &dummy_line_node, dupn, INT2FIX(3)); + ADD_INSN(ret, &dummy_line_node, swap); + ADD_INSN(ret, &dummy_line_node, pop); + } + ADD_INSN(ret, &dummy_line_node, concatarray); + if (boff > 0) { + ADD_INSN1(ret, &dummy_line_node, setn, INT2FIX(3)); + PM_POP; + } + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, argc, INT2FIX(flag)); + } + else { + if (boff > 0) + ADD_INSN(ret, &dummy_line_node, swap); + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, FIXNUM_INC(argc, 1), INT2FIX(flag)); + } + PM_POP; + ADD_INSNL(ret, &dummy_line_node, jump, lfin); + ADD_LABEL(ret, label); + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + } + ADD_INSN1(ret, &dummy_line_node, adjuststack, FIXNUM_INC(argc, 2 + boff)); + ADD_LABEL(ret, lfin); + return; + } case PM_INSTANCE_VARIABLE_AND_WRITE_NODE: { pm_instance_variable_and_write_node_t *instance_variable_and_write_node = (pm_instance_variable_and_write_node_t*) node; diff --git a/test/ruby/test_compile_prism.rb b/test/ruby/test_compile_prism.rb index a01666ee54cbb6..96aad1ab5bd1e6 100644 --- a/test/ruby/test_compile_prism.rb +++ b/test/ruby/test_compile_prism.rb @@ -238,6 +238,27 @@ def test_GlobalVariableWriteNode assert_prism_eval("$pit = 1") end + def test_IndexOrWriteNode + assert_prism_eval("[0][0] ||= 1") + assert_prism_eval("[nil][0] ||= 1") + + # Testing `[]` with a block passed in + assert_prism_eval(<<-CODE) + class CustomHash < Hash + def []=(key, value, &block) + super(block.call(key), value) + end + end + + hash = CustomHash.new + + # Call the custom method with a block that modifies + # the key before assignment + hash["key", &(Proc.new { _1.upcase })] ||= "value" + hash + CODE + end + def test_InstanceVariableAndWriteNode assert_prism_eval("@pit = 0; @pit &&= 1") end From ec5eddf695e9bc3414c8628a6311ffb856fa7374 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 27 Nov 2023 11:12:24 -0500 Subject: [PATCH 69/78] [PRISM] Compile IndexAndWriteNode --- prism_compile.c | 470 +++++++++++++++++--------------- test/ruby/test_compile_prism.rb | 22 ++ 2 files changed, 268 insertions(+), 224 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index b1e30f741610c0..9f9296900b8953 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -836,6 +836,245 @@ pm_compile_call_and_or_write_node(bool and_node, pm_node_t *receiver, pm_node_t return; } +static int +pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinfo_kwarg **kw_arg, rb_iseq_t *iseq, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, pm_scope_node_t *scope_node, NODE dummy_line_node, pm_parser_t *parser) +{ + int orig_argc = 0; + if (arguments_node == NULL) { + if (*flags & VM_CALL_FCALL) { + *flags |= VM_CALL_VCALL; + } + } + else { + pm_node_list_t arguments_node_list = arguments_node->arguments; + + bool has_keyword_splat = (arguments_node->base.flags & PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT); + bool has_splat = false; + + // We count the number of elements post the splat node that are not keyword elements to + // eventually pass as an argument to newarray + int post_splat_counter = 0; + + for (size_t index = 0; index < arguments_node_list.size; index++) { + pm_node_t *argument = arguments_node_list.nodes[index]; + + switch (PM_NODE_TYPE(argument)) { + // A keyword hash node contains all keyword arguments as AssocNodes and AssocSplatNodes + case PM_KEYWORD_HASH_NODE: { + pm_keyword_hash_node_t *keyword_arg = (pm_keyword_hash_node_t *)argument; + size_t len = keyword_arg->elements.size; + + if (has_keyword_splat) { + int cur_hash_size = 0; + orig_argc++; + + bool new_hash_emitted = false; + for (size_t i = 0; i < len; i++) { + pm_node_t *cur_node = keyword_arg->elements.nodes[i]; + + pm_node_type_t cur_type = PM_NODE_TYPE(cur_node); + + switch (PM_NODE_TYPE(cur_node)) { + case PM_ASSOC_NODE: { + pm_assoc_node_t *assoc = (pm_assoc_node_t *)cur_node; + + PM_COMPILE_NOT_POPPED(assoc->key); + PM_COMPILE_NOT_POPPED(assoc->value); + cur_hash_size++; + + // If we're at the last keyword arg, or the last assoc node of this "set", + // then we want to either construct a newhash or merge onto previous hashes + if (i == (len - 1) || !PM_NODE_TYPE_P(keyword_arg->elements.nodes[i + 1], cur_type)) { + if (new_hash_emitted) { + ADD_SEND(ret, &dummy_line_node, id_core_hash_merge_ptr, INT2FIX(cur_hash_size * 2 + 1)); + } + else { + ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(cur_hash_size * 2)); + cur_hash_size = 0; + new_hash_emitted = true; + } + } + + break; + } + case PM_ASSOC_SPLAT_NODE: { + if (len > 1) { + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + if (i == 0) { + ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(0)); + new_hash_emitted = true; + } + else { + PM_SWAP; + } + } + + pm_assoc_splat_node_t *assoc_splat = (pm_assoc_splat_node_t *)cur_node; + PM_COMPILE_NOT_POPPED(assoc_splat->value); + + *flags |= VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT; + + if (len > 1) { + ADD_SEND(ret, &dummy_line_node, id_core_hash_merge_kwd, INT2FIX(2)); + } + + if ((i < len - 1) && !PM_NODE_TYPE_P(keyword_arg->elements.nodes[i + 1], cur_type)) { + ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); + PM_SWAP; + } + + cur_hash_size = 0; + break; + } + default: { + rb_bug("Unknown type"); + } + } + } + break; + } + else { + *kw_arg = rb_xmalloc_mul_add(len, sizeof(VALUE), sizeof(struct rb_callinfo_kwarg)); + *flags = VM_CALL_KWARG; + (*kw_arg)->keyword_len = (int) len; + + // TODO: Method callers like `foo(a => b)` + for (size_t i = 0; i < len; i++) { + pm_assoc_node_t *assoc = (pm_assoc_node_t *)keyword_arg->elements.nodes[i]; + (*kw_arg)->keywords[i] = pm_static_literal_value(assoc->key, scope_node, parser); + PM_COMPILE_NOT_POPPED(assoc->value); + } + } + break; + } + case PM_SPLAT_NODE: { + *flags |= VM_CALL_ARGS_SPLAT; + pm_splat_node_t *splat_node = (pm_splat_node_t *)argument; + if (splat_node->expression) { + orig_argc++; + PM_COMPILE_NOT_POPPED(splat_node->expression); + } + + ADD_INSN1(ret, &dummy_line_node, splatarray, popped ? Qfalse : Qtrue); + + has_splat = true; + post_splat_counter = 0; + + break; + } + case PM_FORWARDING_ARGUMENTS_NODE: { + orig_argc++; + *flags |= VM_CALL_ARGS_BLOCKARG | VM_CALL_ARGS_SPLAT; + ADD_GETLOCAL(ret, &dummy_line_node, 3, 0); + ADD_INSN1(ret, &dummy_line_node, splatarray, RBOOL(arguments_node_list.size > 1)); + ADD_INSN2(ret, &dummy_line_node, getblockparamproxy, INT2FIX(4), INT2FIX(0)); + break; + } + default: { + orig_argc++; + post_splat_counter++; + PM_COMPILE_NOT_POPPED(argument); + + if (has_splat) { + // If the next node starts the keyword section of parameters + if ((index < arguments_node_list.size - 1) && PM_NODE_TYPE_P(arguments_node_list.nodes[index + 1], PM_KEYWORD_HASH_NODE)) { + + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(post_splat_counter)); + ADD_INSN1(ret, &dummy_line_node, splatarray, Qfalse); + ADD_INSN(ret, &dummy_line_node, concatarray); + } + // If it's the final node + else if (index == arguments_node_list.size - 1) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(post_splat_counter)); + ADD_INSN(ret, &dummy_line_node, concatarray); + } + } + } + } + } + } + return orig_argc; +} + +static void +pm_compile_index_and_or_write_node(bool and_node, pm_node_t *receiver, pm_node_t *value, pm_arguments_node_t *arguments, pm_node_t *block, LINK_ANCHOR *const ret, rb_iseq_t *iseq, int lineno, const uint8_t * src, bool popped, pm_scope_node_t *scope_node, pm_parser_t *parser) +{ + NODE dummy_line_node = generate_dummy_line_node(lineno, lineno); + PM_PUTNIL_UNLESS_POPPED; + + PM_COMPILE_NOT_POPPED(receiver); + + int flag = 0; + struct rb_callinfo_kwarg *keywords = NULL; + int argc_int = 0; + + if (arguments) { + argc_int = pm_setup_args(arguments, &flag, &keywords, iseq, ret, src, popped, scope_node, dummy_line_node, parser); + } + + VALUE argc = INT2FIX(argc_int); + int boff = 0; + + if (block) { + PM_COMPILE_NOT_POPPED(block); + flag |= VM_CALL_ARGS_BLOCKARG; + boff = 1; + } + + ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + boff)); + + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idAREF, argc, INT2FIX(flag)); + + LABEL *label = NEW_LABEL(lineno); + LABEL *lfin = NEW_LABEL(lineno); + + PM_DUP; + + if (and_node) { + ADD_INSNL(ret, &dummy_line_node, branchunless, label); + } + else { + // ornode + ADD_INSNL(ret, &dummy_line_node, branchif, label); + } + + PM_POP; + + PM_COMPILE_NOT_POPPED(value); + + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + } + if (flag & VM_CALL_ARGS_SPLAT) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(1)); + if (boff > 0) { + ADD_INSN1(ret, &dummy_line_node, dupn, INT2FIX(3)); + ADD_INSN(ret, &dummy_line_node, swap); + ADD_INSN(ret, &dummy_line_node, pop); + } + ADD_INSN(ret, &dummy_line_node, concatarray); + if (boff > 0) { + ADD_INSN1(ret, &dummy_line_node, setn, INT2FIX(3)); + PM_POP; + } + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, argc, INT2FIX(flag)); + } + else { + if (boff > 0) + ADD_INSN(ret, &dummy_line_node, swap); + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, FIXNUM_INC(argc, 1), INT2FIX(flag)); + } + PM_POP; + ADD_INSNL(ret, &dummy_line_node, jump, lfin); + ADD_LABEL(ret, label); + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + } + ADD_INSN1(ret, &dummy_line_node, adjuststack, FIXNUM_INC(argc, 2 + boff)); + ADD_LABEL(ret, lfin); + return; +} + /** * In order to properly compile multiple-assignment, some preprocessing needs to * be performed in the case of call or constant path targets. This is when they @@ -1312,166 +1551,6 @@ pm_compile_defined_expr(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *con ADD_LABEL(ret, lfinish[0]); } -static int -pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinfo_kwarg **kw_arg, rb_iseq_t *iseq, LINK_ANCHOR *const ret, const uint8_t *src, bool popped, pm_scope_node_t *scope_node, NODE dummy_line_node, pm_parser_t *parser) -{ - int orig_argc = 0; - if (arguments_node == NULL) { - if (*flags & VM_CALL_FCALL) { - *flags |= VM_CALL_VCALL; - } - } - else { - pm_node_list_t arguments_node_list = arguments_node->arguments; - - bool has_keyword_splat = (arguments_node->base.flags & PM_ARGUMENTS_NODE_FLAGS_CONTAINS_KEYWORD_SPLAT); - bool has_splat = false; - - // We count the number of elements post the splat node that are not keyword elements to - // eventually pass as an argument to newarray - int post_splat_counter = 0; - - for (size_t index = 0; index < arguments_node_list.size; index++) { - pm_node_t *argument = arguments_node_list.nodes[index]; - - switch (PM_NODE_TYPE(argument)) { - // A keyword hash node contains all keyword arguments as AssocNodes and AssocSplatNodes - case PM_KEYWORD_HASH_NODE: { - pm_keyword_hash_node_t *keyword_arg = (pm_keyword_hash_node_t *)argument; - size_t len = keyword_arg->elements.size; - - if (has_keyword_splat) { - int cur_hash_size = 0; - orig_argc++; - - bool new_hash_emitted = false; - for (size_t i = 0; i < len; i++) { - pm_node_t *cur_node = keyword_arg->elements.nodes[i]; - - pm_node_type_t cur_type = PM_NODE_TYPE(cur_node); - - switch (PM_NODE_TYPE(cur_node)) { - case PM_ASSOC_NODE: { - pm_assoc_node_t *assoc = (pm_assoc_node_t *)cur_node; - - PM_COMPILE_NOT_POPPED(assoc->key); - PM_COMPILE_NOT_POPPED(assoc->value); - cur_hash_size++; - - // If we're at the last keyword arg, or the last assoc node of this "set", - // then we want to either construct a newhash or merge onto previous hashes - if (i == (len - 1) || !PM_NODE_TYPE_P(keyword_arg->elements.nodes[i + 1], cur_type)) { - if (new_hash_emitted) { - ADD_SEND(ret, &dummy_line_node, id_core_hash_merge_ptr, INT2FIX(cur_hash_size * 2 + 1)); - } - else { - ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(cur_hash_size * 2)); - cur_hash_size = 0; - new_hash_emitted = true; - } - } - - break; - } - case PM_ASSOC_SPLAT_NODE: { - if (len > 1) { - ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); - if (i == 0) { - ADD_INSN1(ret, &dummy_line_node, newhash, INT2FIX(0)); - new_hash_emitted = true; - } - else { - PM_SWAP; - } - } - - pm_assoc_splat_node_t *assoc_splat = (pm_assoc_splat_node_t *)cur_node; - PM_COMPILE_NOT_POPPED(assoc_splat->value); - - *flags |= VM_CALL_KW_SPLAT | VM_CALL_KW_SPLAT_MUT; - - if (len > 1) { - ADD_SEND(ret, &dummy_line_node, id_core_hash_merge_kwd, INT2FIX(2)); - } - - if ((i < len - 1) && !PM_NODE_TYPE_P(keyword_arg->elements.nodes[i + 1], cur_type)) { - ADD_INSN1(ret, &dummy_line_node, putspecialobject, INT2FIX(VM_SPECIAL_OBJECT_VMCORE)); - PM_SWAP; - } - - cur_hash_size = 0; - break; - } - default: { - rb_bug("Unknown type"); - } - } - } - break; - } - else { - *kw_arg = rb_xmalloc_mul_add(len, sizeof(VALUE), sizeof(struct rb_callinfo_kwarg)); - *flags = VM_CALL_KWARG; - (*kw_arg)->keyword_len = (int) len; - - // TODO: Method callers like `foo(a => b)` - for (size_t i = 0; i < len; i++) { - pm_assoc_node_t *assoc = (pm_assoc_node_t *)keyword_arg->elements.nodes[i]; - (*kw_arg)->keywords[i] = pm_static_literal_value(assoc->key, scope_node, parser); - PM_COMPILE_NOT_POPPED(assoc->value); - } - } - break; - } - case PM_SPLAT_NODE: { - *flags |= VM_CALL_ARGS_SPLAT; - pm_splat_node_t *splat_node = (pm_splat_node_t *)argument; - if (splat_node->expression) { - orig_argc++; - PM_COMPILE_NOT_POPPED(splat_node->expression); - } - - ADD_INSN1(ret, &dummy_line_node, splatarray, popped ? Qfalse : Qtrue); - - has_splat = true; - post_splat_counter = 0; - - break; - } - case PM_FORWARDING_ARGUMENTS_NODE: { - orig_argc++; - *flags |= VM_CALL_ARGS_BLOCKARG | VM_CALL_ARGS_SPLAT; - ADD_GETLOCAL(ret, &dummy_line_node, 3, 0); - ADD_INSN1(ret, &dummy_line_node, splatarray, RBOOL(arguments_node_list.size > 1)); - ADD_INSN2(ret, &dummy_line_node, getblockparamproxy, INT2FIX(4), INT2FIX(0)); - break; - } - default: { - orig_argc++; - post_splat_counter++; - PM_COMPILE_NOT_POPPED(argument); - - if (has_splat) { - // If the next node starts the keyword section of parameters - if ((index < arguments_node_list.size - 1) && PM_NODE_TYPE_P(arguments_node_list.nodes[index + 1], PM_KEYWORD_HASH_NODE)) { - - ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(post_splat_counter)); - ADD_INSN1(ret, &dummy_line_node, splatarray, Qfalse); - ADD_INSN(ret, &dummy_line_node, concatarray); - } - // If it's the final node - else if (index == arguments_node_list.size - 1) { - ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(post_splat_counter)); - ADD_INSN(ret, &dummy_line_node, concatarray); - } - } - } - } - } - } - return orig_argc; -} - /* * Compiles a prism node into instruction sequences * @@ -2652,73 +2731,16 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, PM_COMPILE(cast->value); return; } + case PM_INDEX_AND_WRITE_NODE: { + pm_index_and_write_node_t *index_and_write_node = (pm_index_and_write_node_t *)node; + + pm_compile_index_and_or_write_node(true, index_and_write_node->receiver, index_and_write_node->value, index_and_write_node->arguments, index_and_write_node->block, ret, iseq, lineno, src, popped, scope_node, parser); + return; + } case PM_INDEX_OR_WRITE_NODE: { pm_index_or_write_node_t *index_or_write_node = (pm_index_or_write_node_t *)node; - PM_PUTNIL_UNLESS_POPPED; - - PM_COMPILE_NOT_POPPED(index_or_write_node->receiver); - - int flag = 0; - struct rb_callinfo_kwarg *keywords = NULL; - int argc_int = 0; - - if (index_or_write_node->arguments) { - argc_int = pm_setup_args(index_or_write_node->arguments, &flag, &keywords, iseq, ret, src, popped, scope_node, dummy_line_node, parser); - } - - VALUE argc = INT2FIX(argc_int); - int boff = 0; - - if (index_or_write_node->block) { - PM_COMPILE_NOT_POPPED(index_or_write_node->block); - flag |= VM_CALL_ARGS_BLOCKARG; - boff = 1; - } - - ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + boff)); - - ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idAREF, argc, INT2FIX(flag)); - - LABEL *label = NEW_LABEL(lineno); - LABEL *lfin = NEW_LABEL(lineno); - - PM_DUP; - ADD_INSNL(ret, &dummy_line_node, branchif, label); - PM_POP; - - PM_COMPILE_NOT_POPPED(index_or_write_node->value); - - if (!popped) { - ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); - } - if (flag & VM_CALL_ARGS_SPLAT) { - ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(1)); - if (boff > 0) { - ADD_INSN1(ret, &dummy_line_node, dupn, INT2FIX(3)); - ADD_INSN(ret, &dummy_line_node, swap); - ADD_INSN(ret, &dummy_line_node, pop); - } - ADD_INSN(ret, &dummy_line_node, concatarray); - if (boff > 0) { - ADD_INSN1(ret, &dummy_line_node, setn, INT2FIX(3)); - PM_POP; - } - ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, argc, INT2FIX(flag)); - } - else { - if (boff > 0) - ADD_INSN(ret, &dummy_line_node, swap); - ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, FIXNUM_INC(argc, 1), INT2FIX(flag)); - } - PM_POP; - ADD_INSNL(ret, &dummy_line_node, jump, lfin); - ADD_LABEL(ret, label); - if (!popped) { - ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); - } - ADD_INSN1(ret, &dummy_line_node, adjuststack, FIXNUM_INC(argc, 2 + boff)); - ADD_LABEL(ret, lfin); + pm_compile_index_and_or_write_node(false, index_or_write_node->receiver, index_or_write_node->value, index_or_write_node->arguments, index_or_write_node->block, ret, iseq, lineno, src, popped, scope_node, parser); return; } case PM_INSTANCE_VARIABLE_AND_WRITE_NODE: { diff --git a/test/ruby/test_compile_prism.rb b/test/ruby/test_compile_prism.rb index 96aad1ab5bd1e6..2ee078673ff132 100644 --- a/test/ruby/test_compile_prism.rb +++ b/test/ruby/test_compile_prism.rb @@ -238,6 +238,28 @@ def test_GlobalVariableWriteNode assert_prism_eval("$pit = 1") end + def test_IndexAndWriteNode + assert_prism_eval("[0][0] &&= 1") + assert_prism_eval("[nil][0] &&= 1") + + # Testing `[]` with a block passed in + assert_prism_eval(<<-CODE) + class CustomHash < Hash + def []=(key, value, &block) + block ? super(block.call(key), value) : super(key, value) + end + end + + hash = CustomHash.new + + # Call the custom method with a block that modifies + # the key before assignment + hash["KEY"] = "test" + hash["key", &(Proc.new { _1.upcase })] &&= "value" + hash + CODE + end + def test_IndexOrWriteNode assert_prism_eval("[0][0] ||= 1") assert_prism_eval("[nil][0] ||= 1") From ba26c6eae0c51dc07beb5b5c7c5651ef4589f351 Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 27 Nov 2023 12:48:51 -0500 Subject: [PATCH 70/78] [PRISM] Compile IndexOperatorWriteNode --- prism_compile.c | 93 +++++++++++++++++++++++++-------- test/ruby/test_compile_prism.rb | 25 +++++++++ 2 files changed, 95 insertions(+), 23 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 9f9296900b8953..5dd4d983210190 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -996,6 +996,37 @@ pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinf return orig_argc; } +static void +pm_compile_index_write_nodes_add_send(bool popped, LINK_ANCHOR *const ret, rb_iseq_t *iseq, NODE dummy_line_node, VALUE argc, int flag, int boff) +{ + if (!popped) { + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + } + + if (flag & VM_CALL_ARGS_SPLAT) { + ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(1)); + if (boff > 0) { + ADD_INSN1(ret, &dummy_line_node, dupn, INT2FIX(3)); + ADD_INSN(ret, &dummy_line_node, swap); + ADD_INSN(ret, &dummy_line_node, pop); + } + ADD_INSN(ret, &dummy_line_node, concatarray); + if (boff > 0) { + ADD_INSN1(ret, &dummy_line_node, setn, INT2FIX(3)); + PM_POP; + } + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, argc, INT2FIX(flag)); + } + else { + if (boff > 0) + ADD_INSN(ret, &dummy_line_node, swap); + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, FIXNUM_INC(argc, 1), INT2FIX(flag)); + } + + PM_POP; + return; +} + static void pm_compile_index_and_or_write_node(bool and_node, pm_node_t *receiver, pm_node_t *value, pm_arguments_node_t *arguments, pm_node_t *block, LINK_ANCHOR *const ret, rb_iseq_t *iseq, int lineno, const uint8_t * src, bool popped, pm_scope_node_t *scope_node, pm_parser_t *parser) { @@ -1042,29 +1073,8 @@ pm_compile_index_and_or_write_node(bool and_node, pm_node_t *receiver, pm_node_t PM_COMPILE_NOT_POPPED(value); - if (!popped) { - ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); - } - if (flag & VM_CALL_ARGS_SPLAT) { - ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(1)); - if (boff > 0) { - ADD_INSN1(ret, &dummy_line_node, dupn, INT2FIX(3)); - ADD_INSN(ret, &dummy_line_node, swap); - ADD_INSN(ret, &dummy_line_node, pop); - } - ADD_INSN(ret, &dummy_line_node, concatarray); - if (boff > 0) { - ADD_INSN1(ret, &dummy_line_node, setn, INT2FIX(3)); - PM_POP; - } - ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, argc, INT2FIX(flag)); - } - else { - if (boff > 0) - ADD_INSN(ret, &dummy_line_node, swap); - ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, FIXNUM_INC(argc, 1), INT2FIX(flag)); - } - PM_POP; + pm_compile_index_write_nodes_add_send(popped, ret, iseq, dummy_line_node, argc, flag, boff); + ADD_INSNL(ret, &dummy_line_node, jump, lfin); ADD_LABEL(ret, label); if (!popped) { @@ -2743,6 +2753,43 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, pm_compile_index_and_or_write_node(false, index_or_write_node->receiver, index_or_write_node->value, index_or_write_node->arguments, index_or_write_node->block, ret, iseq, lineno, src, popped, scope_node, parser); return; } + case PM_INDEX_OPERATOR_WRITE_NODE: { + pm_index_operator_write_node_t *index_operator_write_node = (pm_index_operator_write_node_t *)node; + + PM_PUTNIL_UNLESS_POPPED; + + PM_COMPILE_NOT_POPPED(index_operator_write_node->receiver); + + int flag = 0; + struct rb_callinfo_kwarg *keywords = NULL; + int argc_int = 0; + + if (index_operator_write_node->arguments) { + argc_int = pm_setup_args(index_operator_write_node->arguments, &flag, &keywords, iseq, ret, src, popped, scope_node, dummy_line_node, parser); + } + + VALUE argc = INT2FIX(argc_int); + int boff = 0; + + if (index_operator_write_node->block) { + PM_COMPILE_NOT_POPPED(index_operator_write_node->block); + flag |= VM_CALL_ARGS_BLOCKARG; + boff = 1; + } + + ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + boff)); + + ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idAREF, argc, INT2FIX(flag)); + + PM_COMPILE_NOT_POPPED(index_operator_write_node->value); + + ID method_id = pm_constant_id_lookup(scope_node, index_operator_write_node->operator); + ADD_SEND(ret, &dummy_line_node, method_id, INT2FIX(1)); + + pm_compile_index_write_nodes_add_send(popped, ret, iseq, dummy_line_node, argc, flag, boff); + + return; + } case PM_INSTANCE_VARIABLE_AND_WRITE_NODE: { pm_instance_variable_and_write_node_t *instance_variable_and_write_node = (pm_instance_variable_and_write_node_t*) node; diff --git a/test/ruby/test_compile_prism.rb b/test/ruby/test_compile_prism.rb index 2ee078673ff132..78c62c3b67b4ab 100644 --- a/test/ruby/test_compile_prism.rb +++ b/test/ruby/test_compile_prism.rb @@ -281,6 +281,31 @@ def []=(key, value, &block) CODE end + def test_IndexOperatorWriteNode + assert_prism_eval("[0][0] += 1") + + # Testing `[]` with a block passed in + assert_prism_eval(<<-CODE) + class CustomHash < Hash + def [](key, &block) + block ? super(block.call(key)) : super(key) + end + + def []=(key, value, &block) + block ? super(block.call(key), value) : super(key, value) + end + end + + hash = CustomHash.new + + # Call the custom method with a block that modifies + # the key before assignment + hash["KEY"] = "test" + hash["key", &(Proc.new { _1.upcase })] &&= "value" + hash + CODE + end + def test_InstanceVariableAndWriteNode assert_prism_eval("@pit = 0; @pit &&= 1") end From e4dd8f6c9c49526f92b5b70c46a067e97e64616e Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 27 Nov 2023 14:07:27 -0500 Subject: [PATCH 71/78] [PRISM] Renamed some variables, added comments --- prism_compile.c | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 5dd4d983210190..1339906ad144d2 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -997,29 +997,30 @@ pm_setup_args(pm_arguments_node_t *arguments_node, int *flags, struct rb_callinf } static void -pm_compile_index_write_nodes_add_send(bool popped, LINK_ANCHOR *const ret, rb_iseq_t *iseq, NODE dummy_line_node, VALUE argc, int flag, int boff) +pm_compile_index_write_nodes_add_send(bool popped, LINK_ANCHOR *const ret, rb_iseq_t *iseq, NODE dummy_line_node, VALUE argc, int flag, int block_offset) { if (!popped) { - ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + block_offset)); } if (flag & VM_CALL_ARGS_SPLAT) { ADD_INSN1(ret, &dummy_line_node, newarray, INT2FIX(1)); - if (boff > 0) { + if (block_offset > 0) { ADD_INSN1(ret, &dummy_line_node, dupn, INT2FIX(3)); - ADD_INSN(ret, &dummy_line_node, swap); - ADD_INSN(ret, &dummy_line_node, pop); + PM_SWAP; + PM_POP; } ADD_INSN(ret, &dummy_line_node, concatarray); - if (boff > 0) { + if (block_offset > 0) { ADD_INSN1(ret, &dummy_line_node, setn, INT2FIX(3)); PM_POP; } ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, argc, INT2FIX(flag)); } else { - if (boff > 0) - ADD_INSN(ret, &dummy_line_node, swap); + if (block_offset > 0) { + PM_SWAP; + } ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idASET, FIXNUM_INC(argc, 1), INT2FIX(flag)); } @@ -1036,23 +1037,23 @@ pm_compile_index_and_or_write_node(bool and_node, pm_node_t *receiver, pm_node_t PM_COMPILE_NOT_POPPED(receiver); int flag = 0; - struct rb_callinfo_kwarg *keywords = NULL; int argc_int = 0; if (arguments) { - argc_int = pm_setup_args(arguments, &flag, &keywords, iseq, ret, src, popped, scope_node, dummy_line_node, parser); + // Get any arguments, and set the appropriate values for flag + argc_int = pm_setup_args(arguments, &flag, NULL, iseq, ret, src, popped, scope_node, dummy_line_node, parser); } VALUE argc = INT2FIX(argc_int); - int boff = 0; + int block_offset = 0; if (block) { PM_COMPILE_NOT_POPPED(block); flag |= VM_CALL_ARGS_BLOCKARG; - boff = 1; + block_offset = 1; } - ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + boff)); + ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + block_offset)); ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idAREF, argc, INT2FIX(flag)); @@ -1073,15 +1074,16 @@ pm_compile_index_and_or_write_node(bool and_node, pm_node_t *receiver, pm_node_t PM_COMPILE_NOT_POPPED(value); - pm_compile_index_write_nodes_add_send(popped, ret, iseq, dummy_line_node, argc, flag, boff); + pm_compile_index_write_nodes_add_send(popped, ret, iseq, dummy_line_node, argc, flag, block_offset); ADD_INSNL(ret, &dummy_line_node, jump, lfin); ADD_LABEL(ret, label); if (!popped) { - ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + boff)); + ADD_INSN1(ret, &dummy_line_node, setn, FIXNUM_INC(argc, 2 + block_offset)); } - ADD_INSN1(ret, &dummy_line_node, adjuststack, FIXNUM_INC(argc, 2 + boff)); + ADD_INSN1(ret, &dummy_line_node, adjuststack, FIXNUM_INC(argc, 2 + block_offset)); ADD_LABEL(ret, lfin); + return; } @@ -2769,15 +2771,16 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } VALUE argc = INT2FIX(argc_int); - int boff = 0; + + int block_offset = 0; if (index_operator_write_node->block) { PM_COMPILE_NOT_POPPED(index_operator_write_node->block); flag |= VM_CALL_ARGS_BLOCKARG; - boff = 1; + block_offset = 1; } - ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + boff)); + ADD_INSN1(ret, &dummy_line_node, dupn, FIXNUM_INC(argc, 1 + block_offset)); ADD_SEND_WITH_FLAG(ret, &dummy_line_node, idAREF, argc, INT2FIX(flag)); @@ -2786,7 +2789,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, ID method_id = pm_constant_id_lookup(scope_node, index_operator_write_node->operator); ADD_SEND(ret, &dummy_line_node, method_id, INT2FIX(1)); - pm_compile_index_write_nodes_add_send(popped, ret, iseq, dummy_line_node, argc, flag, boff); + pm_compile_index_write_nodes_add_send(popped, ret, iseq, dummy_line_node, argc, flag, block_offset); return; } From 1acea5010023deb0b4747e6c7309c4d69bdaf47d Mon Sep 17 00:00:00 2001 From: Jemma Issroff Date: Mon, 13 Nov 2023 11:32:39 -0800 Subject: [PATCH 72/78] [PRISM] Small fixes to parameters ordering and methods --- prism_compile.c | 16 +++++++++++----- test/ruby/test_compile_prism.rb | 9 +++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/prism_compile.c b/prism_compile.c index 1339906ad144d2..7951f25e62b142 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -3656,10 +3656,14 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, for (size_t i = 0; i < keywords_list->size; i++) { pm_node_t *keyword_parameter_node = keywords_list->nodes[i]; + pm_constant_id_t name; switch PM_NODE_TYPE(keyword_parameter_node) { case PM_OPTIONAL_KEYWORD_PARAMETER_NODE: { - pm_node_t *value = ((pm_optional_keyword_parameter_node_t *)keyword_parameter_node)->value; + pm_optional_keyword_parameter_node_t *cast = ((pm_optional_keyword_parameter_node_t *)keyword_parameter_node); + + pm_node_t *value = cast->value; + name = cast->name; if (pm_static_literal_p(value)) { rb_ary_push(default_values, pm_static_literal_value(value, scope_node, parser)); @@ -3672,8 +3676,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, break; } case PM_REQUIRED_KEYWORD_PARAMETER_NODE: { - pm_required_keyword_parameter_node_t *cast = (pm_required_keyword_parameter_node_t *)keyword_parameter_node; - ids[keyword->required_num] = pm_constant_id_lookup(scope_node, cast->name); + name = ((pm_required_keyword_parameter_node_t *)keyword_parameter_node)->name; keyword->required_num++; break; } @@ -3681,8 +3684,12 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, rb_bug("Unexpected keyword parameter node type"); } } + + ids[i] = pm_constant_id_lookup(scope_node, name); } + keyword->table = ids; + VALUE *dvs = ALLOC_N(VALUE, RARRAY_LEN(default_values)); for (int i = 0; i < RARRAY_LEN(default_values); i++) { @@ -3695,7 +3702,6 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } keyword->default_values = dvs; - keyword->table = ids; } if (parameters_node) { @@ -3712,7 +3718,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, if (!body->param.flags.has_kw) { body->param.keyword = keyword = ZALLOC_N(struct rb_iseq_param_keyword, 1); } - keyword->rest_start = (int) locals_size; + keyword->rest_start = (int) locals_size - (parameters_node->block ? 2 : 1); body->param.flags.has_kwrest = true; } } diff --git a/test/ruby/test_compile_prism.rb b/test/ruby/test_compile_prism.rb index 78c62c3b67b4ab..c1a32b84fd114f 100644 --- a/test/ruby/test_compile_prism.rb +++ b/test/ruby/test_compile_prism.rb @@ -787,6 +787,15 @@ def self.prism_test_def_node2() yield 1 end ) end + def test_method_parameters + assert_prism_eval(<<-CODE) + def self.prism_test_method_parameters(a, b=1, *c, d:, e: 2, **f, &g) + end + + method(:prism_test_method_parameters).parameters + CODE + end + def test_LambdaNode assert_prism_eval("-> { to_s }.call") end From 94015e0dce38d238d428b60b46dcb9f3caef445f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 27 Nov 2023 14:45:47 -0500 Subject: [PATCH 73/78] Guard match from GC when scanning string We need to guard match from GC because otherwise it could end up being reclaimed or moved in compaction. --- string.c | 28 +++++++++++++++++----------- test/ruby/test_string.rb | 4 ++++ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/string.c b/string.c index 2f69ab1adb626d..13ca9b33d5e12e 100644 --- a/string.c +++ b/string.c @@ -9973,11 +9973,11 @@ rb_str_strip(VALUE str) static VALUE scan_once(VALUE str, VALUE pat, long *start, int set_backref_str) { - VALUE result, match; - struct re_registers *regs; - int i; + VALUE result = Qnil; long end, pos = rb_pat_search(pat, str, *start, set_backref_str); if (pos >= 0) { + VALUE match; + struct re_registers *regs; if (BUILTIN_TYPE(pat) == T_STRING) { regs = NULL; end = pos + RSTRING_LEN(pat); @@ -9988,6 +9988,7 @@ scan_once(VALUE str, VALUE pat, long *start, int set_backref_str) pos = BEG(0); end = END(0); } + if (pos == end) { rb_encoding *enc = STR_ENC_GET(str); /* @@ -10002,22 +10003,27 @@ scan_once(VALUE str, VALUE pat, long *start, int set_backref_str) else { *start = end; } + if (!regs || regs->num_regs == 1) { result = rb_str_subseq(str, pos, end - pos); return result; } - result = rb_ary_new2(regs->num_regs); - for (i=1; i < regs->num_regs; i++) { - VALUE s = Qnil; - if (BEG(i) >= 0) { - s = rb_str_subseq(str, BEG(i), END(i)-BEG(i)); + else { + result = rb_ary_new2(regs->num_regs); + for (int i = 1; i < regs->num_regs; i++) { + VALUE s = Qnil; + if (BEG(i) >= 0) { + s = rb_str_subseq(str, BEG(i), END(i)-BEG(i)); + } + + rb_ary_push(result, s); } - rb_ary_push(result, s); } - return result; + RB_GC_GUARD(match); } - return Qnil; + + return result; } diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 22bec09855f8c4..fdf23ad90b4d98 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -1622,6 +1622,10 @@ def test_scan assert_equal(%w[1 2 3], S("a1 a2 a3").scan(/a\K./)) end + def test_scan_gc_compact_stress + EnvUtil.under_gc_compact_stress { assert_equal([["1a"], ["2b"], ["3c"]], S("1a2b3c").scan(/(\d.)/)) } + end + def test_scan_segv bug19159 = '[Bug #19159]' assert_nothing_raised(Exception, bug19159) do From 7f50c705742dd92509ae9fc3003eb7561baa7e8a Mon Sep 17 00:00:00 2001 From: Maxime Chevalier-Boisvert Date: Mon, 27 Nov 2023 17:49:53 -0500 Subject: [PATCH 74/78] YJIT: add top C function call counts to `--yjit-stats` (#9047) * YJIT: gather call counts for individual cfuncs Co-authored by Takashi Kokubun --- yjit.rb | 29 ++++++++++++++++++ yjit/src/codegen.rs | 27 +++++++++++++++++ yjit/src/stats.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/yjit.rb b/yjit.rb index 8691a43cd6ee8e..7100242aff3f1d 100644 --- a/yjit.rb +++ b/yjit.rb @@ -369,6 +369,35 @@ def _print_stats(out: $stderr) # :nodoc: out.puts "avg_len_in_yjit: " + ("%13.1f" % stats[:avg_len_in_yjit]) print_sorted_exit_counts(stats, out: out, prefix: "exit_") + + print_sorted_cfunc_calls(stats, out:out) + end + + def print_sorted_cfunc_calls(stats, out:, how_many: 20, left_pad: 4) # :nodoc: + calls = stats[:cfunc_calls] + #puts calls + + # Total number of cfunc calls + num_send_cfunc = stats[:num_send_cfunc] + + # Sort calls by decreasing frequency and keep the top N + pairs = calls.map { |k,v| [k, v] } + pairs.sort_by! {|pair| pair[1] } + pairs.reverse! + pairs = pairs[0...how_many] + + top_n_total = pairs.sum { |name, count| count } + top_n_pct = 100.0 * top_n_total / num_send_cfunc + longest_name_len = pairs.max_by { |name, count| name.length }.first.length + + out.puts "Top-#{pairs.size} most frequent C calls (#{"%.1f" % top_n_pct}% of C calls):" + + pairs.each do |name, count| + padding = longest_name_len + left_pad + padded_name = "%#{padding}s" % name + padded_count = format_number_pct(10, count, num_send_cfunc) + out.puts("#{padded_name}: #{padded_count}") + end end def print_sorted_exit_counts(stats, out:, prefix:, how_many: 20, left_pad: 4) # :nodoc: diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 7846635e718399..51a0ba3a314750 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -5398,6 +5398,7 @@ fn gen_send_cfunc( return None; } + // Increment total cfunc send count gen_counter_incr(asm, Counter::num_send_cfunc); // Delegate to codegen for C methods if we have it. @@ -5416,6 +5417,32 @@ fn gen_send_cfunc( } } + // Log the name of the method we're calling to, + // note that we intentionally don't do this for inlined cfuncs + if get_option!(gen_stats) { + // TODO: extract code to get method name string into its own function + + // Assemble the method name string + let mid = unsafe { vm_ci_mid(ci) }; + let class_name = if recv_known_klass != ptr::null() { + unsafe { cstr_to_rust_string(rb_class2name(*recv_known_klass)) }.unwrap() + } else { + "Unknown".to_string() + }; + let method_name = if mid != 0 { + unsafe { cstr_to_rust_string(rb_id2name(mid)) }.unwrap() + } else { + "Unknown".to_string() + }; + let name_str = format!("{}#{}", class_name, method_name); + + // Get an index for this cfunc name + let cfunc_idx = get_cfunc_idx(&name_str); + + // Increment the counter for this cfunc + asm.ccall(incr_cfunc_counter as *const u8, vec![cfunc_idx.into()]); + } + // Check for interrupts gen_check_ints(asm, Counter::guard_send_interrupted); diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index e3b610329943ff..7a52fc0e2310e0 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -6,6 +6,7 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; +use std::collections::HashMap; use crate::codegen::CodegenGlobals; use crate::core::Context; @@ -52,6 +53,58 @@ unsafe impl GlobalAlloc for StatsAlloc { } } +/// Mapping of C function name to integer indices +/// This is accessed at compilation time only (protected by a lock) +static mut CFUNC_NAME_TO_IDX: Option> = None; + +/// Vector of call counts for each C function index +/// This is modified (but not resized) by JITted code +static mut CFUNC_CALL_COUNT: Option> = None; + +/// Assign an index to a given cfunc name string +pub fn get_cfunc_idx(name: &str) -> usize +{ + //println!("{}", name); + + unsafe { + if CFUNC_NAME_TO_IDX.is_none() { + CFUNC_NAME_TO_IDX = Some(HashMap::default()); + } + + if CFUNC_CALL_COUNT.is_none() { + CFUNC_CALL_COUNT = Some(Vec::default()); + } + + let name_to_idx = CFUNC_NAME_TO_IDX.as_mut().unwrap(); + + match name_to_idx.get(name) { + Some(idx) => *idx, + None => { + let idx = name_to_idx.len(); + name_to_idx.insert(name.to_string(), idx); + + // Resize the call count vector + let cfunc_call_count = CFUNC_CALL_COUNT.as_mut().unwrap(); + if idx >= cfunc_call_count.len() { + cfunc_call_count.resize(idx + 1, 0); + } + + idx + } + } + } +} + +// Increment the counter for a C function +pub extern "C" fn incr_cfunc_counter(idx: usize) +{ + unsafe { + let cfunc_call_count = CFUNC_CALL_COUNT.as_mut().unwrap(); + assert!(idx < cfunc_call_count.len()); + cfunc_call_count[idx] += 1; + } +} + // YJIT exit counts for each instruction type const VM_INSTRUCTION_SIZE_USIZE: usize = VM_INSTRUCTION_SIZE as usize; static mut EXIT_OP_COUNT: [u64; VM_INSTRUCTION_SIZE_USIZE] = [0; VM_INSTRUCTION_SIZE_USIZE]; @@ -663,7 +716,6 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { return hash; } - // If the stats feature is enabled unsafe { // Indicate that the complete set of stats is available rb_hash_aset(hash, rust_str_to_sym("all_stats"), Qtrue); @@ -689,6 +741,23 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { let value = VALUE::fixnum_from_usize(EXIT_OP_COUNT[op_idx] as usize); rb_hash_aset(hash, key, value); } + + // Create a hash for the cfunc call counts + if let Some(cfunc_name_to_idx) = CFUNC_NAME_TO_IDX.as_mut() { + let call_counts = CFUNC_CALL_COUNT.as_mut().unwrap(); + let calls_hash = rb_hash_new(); + + for (name, idx) in cfunc_name_to_idx { + let count = call_counts[*idx]; + println!("{}: {}", name, count); + + let key = rust_str_to_sym(name); + let value = VALUE::fixnum_from_usize(count as usize); + rb_hash_aset(calls_hash, key, value); + } + + rb_hash_aset(hash, rust_str_to_sym("cfunc_calls"), calls_hash); + } } hash From def416899d2b72d2299ddfa97f1f94ae2594d67b Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Tue, 28 Nov 2023 10:38:15 +0900 Subject: [PATCH 75/78] [ruby/stringio] Development of 3.1.1 started. https://github.com/ruby/stringio/commit/75da93d48f --- ext/stringio/stringio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index 7eade5bcba6d37..74e2b95c99ca46 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -13,7 +13,7 @@ **********************************************************************/ static const char *const -STRINGIO_VERSION = "3.1.0"; +STRINGIO_VERSION = "3.1.1"; #include "ruby.h" #include "ruby/io.h" From 031e81c8f388abb856d2b63ead5d3603e4e3dfe6 Mon Sep 17 00:00:00 2001 From: git Date: Tue, 28 Nov 2023 01:39:40 +0000 Subject: [PATCH 76/78] Update default gems list at def416899d2b72d2299ddfa97f1f94 [ci skip] --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index f0eded8381fe60..bcefa9c776fb77 100644 --- a/NEWS.md +++ b/NEWS.md @@ -161,7 +161,7 @@ The following default gems are updated. * securerandom 0.3.0 * shellwords 0.2.0 * singleton 0.2.0 -* stringio 3.1.0 +* stringio 3.1.1 * strscan 3.0.8 * syntax_suggest 1.1.0 * tempfile 0.2.0 From 32b5f5be7cd0140c5f919d81d6ebf826efd03bb8 Mon Sep 17 00:00:00 2001 From: Haldun Bayhantopcu Date: Mon, 27 Nov 2023 22:57:46 +0100 Subject: [PATCH 77/78] [ruby/prism] Introduce char_is_identifier_utf8 https://github.com/ruby/prism/commit/5f43e57b0f --- prism/prism.c | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/prism/prism.c b/prism/prism.c index 96ed3989e2ebc6..1751857e1e3bc3 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -5907,6 +5907,19 @@ char_is_identifier_start(pm_parser_t *parser, const uint8_t *b) { } } +/** + * Similar to char_is_identifier but this function assumes that the encoding + * has not been changed. + */ +static inline size_t +char_is_identifier_utf8(const uint8_t *b, const uint8_t *end) { + if (*b < 0x80) { + return (*b == '_') || (pm_encoding_unicode_table[*b] & PRISM_ENCODING_ALPHANUMERIC_BIT ? 1 : 0); + } else { + return (size_t) (pm_encoding_utf_8_alnum_char(b, end - b) || 1u); + } +} + /** * Like the above, this function is also used extremely frequently to lex all of * the identifiers in a source file once the first character has been found. So @@ -5925,11 +5938,8 @@ char_is_identifier(pm_parser_t *parser, const uint8_t *b) { } else { return 0; } - } else if (*b < 0x80) { - return (pm_encoding_unicode_table[*b] & PRISM_ENCODING_ALPHANUMERIC_BIT ? 1 : 0) || (*b == '_'); - } else { - return (size_t) (pm_encoding_utf_8_alnum_char(b, parser->end - b) || 1u); } + return char_is_identifier_utf8(b, parser->end); } // Here we're defining a perfect hash for the characters that are allowed in @@ -7003,9 +7013,16 @@ lex_identifier(pm_parser_t *parser, bool previous_command_start) { const uint8_t *end = parser->end; const uint8_t *current_start = parser->current.start; const uint8_t *current_end = parser->current.end; + bool encoding_changed = parser->encoding_changed; - while (current_end < end && (width = char_is_identifier(parser, current_end)) > 0) { - current_end += width; + if (encoding_changed) { + while (current_end < end && (width = char_is_identifier(parser, current_end)) > 0) { + current_end += width; + } + } else { + while (current_end < end && (width = char_is_identifier_utf8(current_end, end)) > 0) { + current_end += width; + } } parser->current.end = current_end; @@ -7123,7 +7140,7 @@ lex_identifier(pm_parser_t *parser, bool previous_command_start) { } } - if (parser->encoding_changed) { + if (encoding_changed) { return parser->encoding.isupper_char(current_start, end - current_start) ? PM_TOKEN_CONSTANT : PM_TOKEN_IDENTIFIER; } return pm_encoding_utf_8_isupper_char(current_start, end - current_start) ? PM_TOKEN_CONSTANT : PM_TOKEN_IDENTIFIER; From 0164da68c170c7f1e36dd70965b67c8c63523391 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Mon, 27 Nov 2023 15:05:06 -0500 Subject: [PATCH 78/78] [ruby/prism] Use un-capitalized error messages I don't prefer this style, but it appears that a plurality of syntax error messages between with un-capitalized messages in CRuby, so we'll go with that for consistency, for now. https://github.com/ruby/prism/commit/b02df68954 --- prism/diagnostic.c | 442 +++++++++++++++++++------------------- test/prism/errors_test.rb | 394 ++++++++++++++++----------------- 2 files changed, 418 insertions(+), 418 deletions(-) diff --git a/prism/diagnostic.c b/prism/diagnostic.c index 1161dec6be1f8e..84278b5e9cf913 100644 --- a/prism/diagnostic.c +++ b/prism/diagnostic.c @@ -5,32 +5,33 @@ * * When composing an error message, use sentence fragments. * - * Try describing the property of the code that caused the error, rather than the rule that is being - * violated. It may help to use a fragment that completes a sentence beginning, "The parser - * encountered (a) ...". If appropriate, add a description of the rule violation (or other helpful - * context) after a semicolon. + * Try describing the property of the code that caused the error, rather than + * the rule that is being violated. It may help to use a fragment that completes + * a sentence beginning, "the parser encountered (a) ...". If appropriate, add a + * description of the rule violation (or other helpful context) after a + * semicolon. * - * For example:, instead of "Control escape sequence cannot be doubled", prefer: + * For example:, instead of "control escape sequence cannot be doubled", prefer: * - * > "Invalid control escape sequence; control cannot be repeated" + * > "invalid control escape sequence; control cannot be repeated" * - * In some cases, where the failure is more general or syntax expectations are violated, it may make - * more sense to use a fragment that completes a sentence beginning, "The parser ...". + * In some cases, where the failure is more general or syntax expectations are + * violated, it may make more sense to use a fragment that completes a sentence + * beginning, "the parser ...". * * For example: * - * > "Expected an expression after `(`" - * > "Cannot parse the expression" - * + * > "expected an expression after `(`" + * > "cannot parse the expression" * * ## Message style guide * * - Use articles like "a", "an", and "the" when appropriate. - * - e.g., prefer "Cannot parse the expression" to "Cannot parse expression". + * - e.g., prefer "cannot parse the expression" to "cannot parse expression". * - Use the common name for tokens and nodes. * - e.g., prefer "keyword splat" to "assoc splat" * - e.g., prefer "embedded document" to "embdoc" - * - Capitalize the initial word of the message. + * - Do not capitalize the initial word of the message. * - Use back ticks around token literals * - e.g., "Expected a `=>` between the hash key and value" * - Do not use `.` or other punctuation at the end of the message. @@ -38,7 +39,6 @@ * - For tokens that can have multiple meanings, reference the token and its meaning. * - e.g., "`*` splat argument" is clearer and more complete than "splat argument" or "`*` argument" * - * * ## Error names (PM_ERR_*) * * - When appropriate, prefer node name to token name. @@ -51,215 +51,215 @@ * sorted. See PM_ERR_ARGUMENT_NO_FORWARDING_* for an example. */ static const char* const diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = { - [PM_ERR_ALIAS_ARGUMENT] = "Invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", - [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = "Unexpected `&&=` in a multiple assignment", - [PM_ERR_ARGUMENT_AFTER_BLOCK] = "Unexpected argument after a block argument", - [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = "Unexpected argument after `...`", - [PM_ERR_ARGUMENT_BARE_HASH] = "Unexpected bare hash argument", - [PM_ERR_ARGUMENT_BLOCK_MULTI] = "Multiple block arguments; only one block is allowed", - [PM_ERR_ARGUMENT_FORMAL_CLASS] = "Invalid formal argument; formal argument cannot be a class variable", - [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = "Invalid formal argument; formal argument cannot be a constant", - [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = "Invalid formal argument; formal argument cannot be a global variable", - [PM_ERR_ARGUMENT_FORMAL_IVAR] = "Invalid formal argument; formal argument cannot be an instance variable", - [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = "Unexpected `...` in an non-parenthesized call", - [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = "Unexpected `&` when the parent method is not forwarding", - [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = "Unexpected `...` when the parent method is not forwarding", - [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = "Unexpected `*` when the parent method is not forwarding", - [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = "Unexpected `*` splat argument after a `**` keyword splat argument", - [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = "Unexpected `*` splat argument after a `*` splat argument", - [PM_ERR_ARGUMENT_TERM_PAREN] = "Expected a `)` to close the arguments", - [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = "Unexpected `{` after a method call without parenthesis", - [PM_ERR_ARRAY_ELEMENT] = "Expected an element for the array", - [PM_ERR_ARRAY_EXPRESSION] = "Expected an expression for the array element", - [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = "Expected an expression after `*` in the array", - [PM_ERR_ARRAY_SEPARATOR] = "Expected a `,` separator for the array elements", - [PM_ERR_ARRAY_TERM] = "Expected a `]` to close the array", - [PM_ERR_BEGIN_LONELY_ELSE] = "Unexpected `else` in `begin` block; a `rescue` clause must precede `else`", - [PM_ERR_BEGIN_TERM] = "Expected an `end` to close the `begin` statement", - [PM_ERR_BEGIN_UPCASE_BRACE] = "Expected a `{` after `BEGIN`", - [PM_ERR_BEGIN_UPCASE_TERM] = "Expected a `}` to close the `BEGIN` statement", + [PM_ERR_ALIAS_ARGUMENT] = "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", + [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = "unexpected `&&=` in a multiple assignment", + [PM_ERR_ARGUMENT_AFTER_BLOCK] = "unexpected argument after a block argument", + [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = "unexpected argument after `...`", + [PM_ERR_ARGUMENT_BARE_HASH] = "unexpected bare hash argument", + [PM_ERR_ARGUMENT_BLOCK_MULTI] = "multiple block arguments; only one block is allowed", + [PM_ERR_ARGUMENT_FORMAL_CLASS] = "invalid formal argument; formal argument cannot be a class variable", + [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = "invalid formal argument; formal argument cannot be a constant", + [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = "invalid formal argument; formal argument cannot be a global variable", + [PM_ERR_ARGUMENT_FORMAL_IVAR] = "invalid formal argument; formal argument cannot be an instance variable", + [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = "unexpected `...` in an non-parenthesized call", + [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = "unexpected `&` when the parent method is not forwarding", + [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = "unexpected `...` when the parent method is not forwarding", + [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = "unexpected `*` when the parent method is not forwarding", + [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = "unexpected `*` splat argument after a `**` keyword splat argument", + [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = "unexpected `*` splat argument after a `*` splat argument", + [PM_ERR_ARGUMENT_TERM_PAREN] = "expected a `)` to close the arguments", + [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = "unexpected `{` after a method call without parenthesis", + [PM_ERR_ARRAY_ELEMENT] = "expected an element for the array", + [PM_ERR_ARRAY_EXPRESSION] = "expected an expression for the array element", + [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = "expected an expression after `*` in the array", + [PM_ERR_ARRAY_SEPARATOR] = "expected a `,` separator for the array elements", + [PM_ERR_ARRAY_TERM] = "expected a `]` to close the array", + [PM_ERR_BEGIN_LONELY_ELSE] = "unexpected `else` in `begin` block; a `rescue` clause must precede `else`", + [PM_ERR_BEGIN_TERM] = "expected an `end` to close the `begin` statement", + [PM_ERR_BEGIN_UPCASE_BRACE] = "expected a `{` after `BEGIN`", + [PM_ERR_BEGIN_UPCASE_TERM] = "expected a `}` to close the `BEGIN` statement", [PM_ERR_BEGIN_UPCASE_TOPLEVEL] = "BEGIN is permitted only at toplevel", - [PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE] = "Expected a local variable name in the block parameters", - [PM_ERR_BLOCK_PARAM_PIPE_TERM] = "Expected the block parameters to end with `|`", - [PM_ERR_BLOCK_TERM_BRACE] = "Expected a block beginning with `{` to end with `}`", - [PM_ERR_BLOCK_TERM_END] = "Expected a block beginning with `do` to end with `end`", - [PM_ERR_CANNOT_PARSE_EXPRESSION] = "Cannot parse the expression", - [PM_ERR_CANNOT_PARSE_STRING_PART] = "Cannot parse the string part", - [PM_ERR_CASE_EXPRESSION_AFTER_CASE] = "Expected an expression after `case`", - [PM_ERR_CASE_EXPRESSION_AFTER_WHEN] = "Expected an expression after `when`", - [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = "Expected a predicate for a case matching statement", - [PM_ERR_CASE_MISSING_CONDITIONS] = "Expected a `when` or `in` clause after `case`", - [PM_ERR_CASE_TERM] = "Expected an `end` to close the `case` statement", - [PM_ERR_CLASS_IN_METHOD] = "Unexpected class definition in a method body", - [PM_ERR_CLASS_NAME] = "Expected a constant name after `class`", - [PM_ERR_CLASS_SUPERCLASS] = "Expected a superclass after `<`", - [PM_ERR_CLASS_TERM] = "Expected an `end` to close the `class` statement", - [PM_ERR_CLASS_UNEXPECTED_END] = "Unexpected `end`, expecting ';' or '\n'", - [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = "Expected a predicate expression for the `elsif` statement", - [PM_ERR_CONDITIONAL_IF_PREDICATE] = "Expected a predicate expression for the `if` statement", - [PM_ERR_CONDITIONAL_PREDICATE_TERM] = "Expected `then` or `;` or '\n'", - [PM_ERR_CONDITIONAL_TERM] = "Expected an `end` to close the conditional clause", - [PM_ERR_CONDITIONAL_TERM_ELSE] = "Expected an `end` to close the `else` clause", - [PM_ERR_CONDITIONAL_UNLESS_PREDICATE] = "Expected a predicate expression for the `unless` statement", - [PM_ERR_CONDITIONAL_UNTIL_PREDICATE] = "Expected a predicate expression for the `until` statement", - [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = "Expected a predicate expression for the `while` statement", - [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = "Expected a constant after the `::` operator", - [PM_ERR_DEF_ENDLESS] = "Could not parse the endless method body", - [PM_ERR_DEF_ENDLESS_SETTER] = "Invalid method name; a setter method cannot be defined in an endless method definition", - [PM_ERR_DEF_NAME] = "Expected a method name", - [PM_ERR_DEF_NAME_AFTER_RECEIVER] = "Expected a method name after the receiver", - [PM_ERR_DEF_PARAMS_TERM] = "Expected a delimiter to close the parameters", - [PM_ERR_DEF_PARAMS_TERM_PAREN] = "Expected a `)` to close the parameters", - [PM_ERR_DEF_RECEIVER] = "Expected a receiver for the method definition", - [PM_ERR_DEF_RECEIVER_TERM] = "Expected a `.` or `::` after the receiver in a method definition", - [PM_ERR_DEF_TERM] = "Expected an `end` to close the `def` statement", - [PM_ERR_DEFINED_EXPRESSION] = "Expected an expression after `defined?`", - [PM_ERR_EMBDOC_TERM] = "Could not find a terminator for the embedded document", - [PM_ERR_EMBEXPR_END] = "Expected a `}` to close the embedded expression", - [PM_ERR_EMBVAR_INVALID] = "Invalid embedded variable", - [PM_ERR_END_UPCASE_BRACE] = "Expected a `{` after `END`", - [PM_ERR_END_UPCASE_TERM] = "Expected a `}` to close the `END` statement", - [PM_ERR_ESCAPE_INVALID_CONTROL] = "Invalid control escape sequence", - [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = "Invalid control escape sequence; control cannot be repeated", - [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = "Invalid hexadecimal escape sequence", - [PM_ERR_ESCAPE_INVALID_META] = "Invalid meta escape sequence", - [PM_ERR_ESCAPE_INVALID_META_REPEAT] = "Invalid meta escape sequence; meta cannot be repeated", - [PM_ERR_ESCAPE_INVALID_UNICODE] = "Invalid Unicode escape sequence", - [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = "Invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", - [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = "Invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", - [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = "Invalid Unicode escape sequence; maximum length is 6 digits", - [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = "Invalid Unicode escape sequence; needs closing `}`", - [PM_ERR_EXPECT_ARGUMENT] = "Expected an argument", - [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = "Expected a newline or semicolon after the statement", - [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = "Expected an expression after `&&=`", - [PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = "Expected an expression after `||=`", - [PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA] = "Expected an expression after `,`", - [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = "Expected an expression after `=`", - [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = "Expected an expression after `<<`", - [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = "Expected an expression after `(`", - [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = "Expected an expression after the operator", - [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = "Expected an expression after `*` splat in an argument", - [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = "Expected an expression after `**` in a hash", - [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = "Expected an expression after `*`", - [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = "Expected an identifier for the required parameter", - [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = "Expected a `(` to start a required parameter", - [PM_ERR_EXPECT_RBRACKET] = "Expected a matching `]`", - [PM_ERR_EXPECT_RPAREN] = "Expected a matching `)`", - [PM_ERR_EXPECT_RPAREN_AFTER_MULTI] = "Expected a `)` after multiple assignment", - [PM_ERR_EXPECT_RPAREN_REQ_PARAMETER] = "Expected a `)` to end a required parameter", - [PM_ERR_EXPECT_STRING_CONTENT] = "Expected string content after opening string delimiter", - [PM_ERR_EXPECT_WHEN_DELIMITER] = "Expected a delimiter after the predicates of a `when` clause", - [PM_ERR_EXPRESSION_BARE_HASH] = "Unexpected bare hash in expression", - [PM_ERR_FOR_COLLECTION] = "Expected a collection after the `in` in a `for` statement", - [PM_ERR_FOR_INDEX] = "Expected an index after `for`", - [PM_ERR_FOR_IN] = "Expected an `in` after the index in a `for` statement", - [PM_ERR_FOR_TERM] = "Expected an `end` to close the `for` loop", - [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = "Expected an expression after the label in a hash", - [PM_ERR_HASH_KEY] = "Expected a key in the hash literal", - [PM_ERR_HASH_ROCKET] = "Expected a `=>` between the hash key and value", - [PM_ERR_HASH_TERM] = "Expected a `}` to close the hash literal", - [PM_ERR_HASH_VALUE] = "Expected a value in the hash literal", - [PM_ERR_HEREDOC_TERM] = "Could not find a terminator for the heredoc", - [PM_ERR_INCOMPLETE_QUESTION_MARK] = "Incomplete expression at `?`", - [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = "Incomplete class variable", - [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = "Incomplete instance variable", - [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT] = "Unknown or invalid encoding in the magic comment", - [PM_ERR_INVALID_FLOAT_EXPONENT] = "Invalid exponent", - [PM_ERR_INVALID_NUMBER_BINARY] = "Invalid binary number", - [PM_ERR_INVALID_NUMBER_DECIMAL] = "Invalid decimal number", - [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = "Invalid hexadecimal number", - [PM_ERR_INVALID_NUMBER_OCTAL] = "Invalid octal number", - [PM_ERR_INVALID_NUMBER_UNDERSCORE] = "Invalid underscore placement in number", - [PM_ERR_INVALID_PERCENT] = "Invalid `%` token", // TODO WHAT? - [PM_ERR_INVALID_TOKEN] = "Invalid token", // TODO WHAT? - [PM_ERR_INVALID_VARIABLE_GLOBAL] = "Invalid global variable", - [PM_ERR_LAMBDA_OPEN] = "Expected a `do` keyword or a `{` to open the lambda block", - [PM_ERR_LAMBDA_TERM_BRACE] = "Expected a lambda block beginning with `{` to end with `}`", - [PM_ERR_LAMBDA_TERM_END] = "Expected a lambda block beginning with `do` to end with `end`", - [PM_ERR_LIST_I_LOWER_ELEMENT] = "Expected a symbol in a `%i` list", - [PM_ERR_LIST_I_LOWER_TERM] = "Expected a closing delimiter for the `%i` list", - [PM_ERR_LIST_I_UPPER_ELEMENT] = "Expected a symbol in a `%I` list", - [PM_ERR_LIST_I_UPPER_TERM] = "Expected a closing delimiter for the `%I` list", - [PM_ERR_LIST_W_LOWER_ELEMENT] = "Expected a string in a `%w` list", - [PM_ERR_LIST_W_LOWER_TERM] = "Expected a closing delimiter for the `%w` list", - [PM_ERR_LIST_W_UPPER_ELEMENT] = "Expected a string in a `%W` list", - [PM_ERR_LIST_W_UPPER_TERM] = "Expected a closing delimiter for the `%W` list", - [PM_ERR_MALLOC_FAILED] = "Failed to allocate memory", - [PM_ERR_MODULE_IN_METHOD] = "Unexpected module definition in a method body", - [PM_ERR_MODULE_NAME] = "Expected a constant name after `module`", - [PM_ERR_MODULE_TERM] = "Expected an `end` to close the `module` statement", - [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = "Multiple splats in multiple assignment", - [PM_ERR_NOT_EXPRESSION] = "Expected an expression after `not`", - [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = "Number literal ending with a `_`", - [PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED] = "Numbered parameters are not allowed alongside explicit parameters", - [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = "Numbered parameter is already used in outer scope", - [PM_ERR_OPERATOR_MULTI_ASSIGN] = "Unexpected operator for a multiple assignment", - [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = "Unexpected operator after a call with arguments", - [PM_ERR_OPERATOR_WRITE_BLOCK] = "Unexpected operator after a call with a block", - [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = "Unexpected multiple `**` splat parameters", - [PM_ERR_PARAMETER_BLOCK_MULTI] = "Multiple block parameters; only one block is allowed", - [PM_ERR_PARAMETER_METHOD_NAME] = "Unexpected name for a parameter", - [PM_ERR_PARAMETER_NAME_REPEAT] = "Repeated parameter name", - [PM_ERR_PARAMETER_NO_DEFAULT] = "Expected a default value for the parameter", - [PM_ERR_PARAMETER_NO_DEFAULT_KW] = "Expected a default value for the keyword parameter", + [PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE] = "expected a local variable name in the block parameters", + [PM_ERR_BLOCK_PARAM_PIPE_TERM] = "expected the block parameters to end with `|`", + [PM_ERR_BLOCK_TERM_BRACE] = "expected a block beginning with `{` to end with `}`", + [PM_ERR_BLOCK_TERM_END] = "expected a block beginning with `do` to end with `end`", + [PM_ERR_CANNOT_PARSE_EXPRESSION] = "cannot parse the expression", + [PM_ERR_CANNOT_PARSE_STRING_PART] = "cannot parse the string part", + [PM_ERR_CASE_EXPRESSION_AFTER_CASE] = "expected an expression after `case`", + [PM_ERR_CASE_EXPRESSION_AFTER_WHEN] = "expected an expression after `when`", + [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = "expected a predicate for a case matching statement", + [PM_ERR_CASE_MISSING_CONDITIONS] = "expected a `when` or `in` clause after `case`", + [PM_ERR_CASE_TERM] = "expected an `end` to close the `case` statement", + [PM_ERR_CLASS_IN_METHOD] = "unexpected class definition in a method body", + [PM_ERR_CLASS_NAME] = "expected a constant name after `class`", + [PM_ERR_CLASS_SUPERCLASS] = "expected a superclass after `<`", + [PM_ERR_CLASS_TERM] = "expected an `end` to close the `class` statement", + [PM_ERR_CLASS_UNEXPECTED_END] = "unexpected `end`, expecting ';' or '\n'", + [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = "expected a predicate expression for the `elsif` statement", + [PM_ERR_CONDITIONAL_IF_PREDICATE] = "expected a predicate expression for the `if` statement", + [PM_ERR_CONDITIONAL_PREDICATE_TERM] = "expected `then` or `;` or '\n'", + [PM_ERR_CONDITIONAL_TERM] = "expected an `end` to close the conditional clause", + [PM_ERR_CONDITIONAL_TERM_ELSE] = "expected an `end` to close the `else` clause", + [PM_ERR_CONDITIONAL_UNLESS_PREDICATE] = "expected a predicate expression for the `unless` statement", + [PM_ERR_CONDITIONAL_UNTIL_PREDICATE] = "expected a predicate expression for the `until` statement", + [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = "expected a predicate expression for the `while` statement", + [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = "expected a constant after the `::` operator", + [PM_ERR_DEF_ENDLESS] = "could not parse the endless method body", + [PM_ERR_DEF_ENDLESS_SETTER] = "invalid method name; a setter method cannot be defined in an endless method definition", + [PM_ERR_DEF_NAME] = "expected a method name", + [PM_ERR_DEF_NAME_AFTER_RECEIVER] = "expected a method name after the receiver", + [PM_ERR_DEF_PARAMS_TERM] = "expected a delimiter to close the parameters", + [PM_ERR_DEF_PARAMS_TERM_PAREN] = "expected a `)` to close the parameters", + [PM_ERR_DEF_RECEIVER] = "expected a receiver for the method definition", + [PM_ERR_DEF_RECEIVER_TERM] = "expected a `.` or `::` after the receiver in a method definition", + [PM_ERR_DEF_TERM] = "expected an `end` to close the `def` statement", + [PM_ERR_DEFINED_EXPRESSION] = "expected an expression after `defined?`", + [PM_ERR_EMBDOC_TERM] = "could not find a terminator for the embedded document", + [PM_ERR_EMBEXPR_END] = "expected a `}` to close the embedded expression", + [PM_ERR_EMBVAR_INVALID] = "invalid embedded variable", + [PM_ERR_END_UPCASE_BRACE] = "expected a `{` after `END`", + [PM_ERR_END_UPCASE_TERM] = "expected a `}` to close the `END` statement", + [PM_ERR_ESCAPE_INVALID_CONTROL] = "invalid control escape sequence", + [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = "invalid control escape sequence; control cannot be repeated", + [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = "invalid hexadecimal escape sequence", + [PM_ERR_ESCAPE_INVALID_META] = "invalid meta escape sequence", + [PM_ERR_ESCAPE_INVALID_META_REPEAT] = "invalid meta escape sequence; meta cannot be repeated", + [PM_ERR_ESCAPE_INVALID_UNICODE] = "invalid Unicode escape sequence", + [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", + [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = "invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", + [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = "invalid Unicode escape sequence; maximum length is 6 digits", + [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = "invalid Unicode escape sequence; needs closing `}`", + [PM_ERR_EXPECT_ARGUMENT] = "expected an argument", + [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = "expected a newline or semicolon after the statement", + [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = "expected an expression after `&&=`", + [PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = "expected an expression after `||=`", + [PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA] = "expected an expression after `,`", + [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = "expected an expression after `=`", + [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = "expected an expression after `<<`", + [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = "expected an expression after `(`", + [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = "expected an expression after the operator", + [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = "expected an expression after `*` splat in an argument", + [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = "expected an expression after `**` in a hash", + [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = "expected an expression after `*`", + [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = "expected an identifier for the required parameter", + [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = "expected a `(` to start a required parameter", + [PM_ERR_EXPECT_RBRACKET] = "expected a matching `]`", + [PM_ERR_EXPECT_RPAREN] = "expected a matching `)`", + [PM_ERR_EXPECT_RPAREN_AFTER_MULTI] = "expected a `)` after multiple assignment", + [PM_ERR_EXPECT_RPAREN_REQ_PARAMETER] = "expected a `)` to end a required parameter", + [PM_ERR_EXPECT_STRING_CONTENT] = "expected string content after opening string delimiter", + [PM_ERR_EXPECT_WHEN_DELIMITER] = "expected a delimiter after the predicates of a `when` clause", + [PM_ERR_EXPRESSION_BARE_HASH] = "unexpected bare hash in expression", + [PM_ERR_FOR_COLLECTION] = "expected a collection after the `in` in a `for` statement", + [PM_ERR_FOR_INDEX] = "expected an index after `for`", + [PM_ERR_FOR_IN] = "expected an `in` after the index in a `for` statement", + [PM_ERR_FOR_TERM] = "expected an `end` to close the `for` loop", + [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = "expected an expression after the label in a hash", + [PM_ERR_HASH_KEY] = "expected a key in the hash literal", + [PM_ERR_HASH_ROCKET] = "expected a `=>` between the hash key and value", + [PM_ERR_HASH_TERM] = "expected a `}` to close the hash literal", + [PM_ERR_HASH_VALUE] = "expected a value in the hash literal", + [PM_ERR_HEREDOC_TERM] = "could not find a terminator for the heredoc", + [PM_ERR_INCOMPLETE_QUESTION_MARK] = "incomplete expression at `?`", + [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = "incomplete class variable", + [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = "incomplete instance variable", + [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT] = "unknown or invalid encoding in the magic comment", + [PM_ERR_INVALID_FLOAT_EXPONENT] = "invalid exponent", + [PM_ERR_INVALID_NUMBER_BINARY] = "invalid binary number", + [PM_ERR_INVALID_NUMBER_DECIMAL] = "invalid decimal number", + [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = "invalid hexadecimal number", + [PM_ERR_INVALID_NUMBER_OCTAL] = "invalid octal number", + [PM_ERR_INVALID_NUMBER_UNDERSCORE] = "invalid underscore placement in number", + [PM_ERR_INVALID_PERCENT] = "invalid `%` token", // TODO WHAT? + [PM_ERR_INVALID_TOKEN] = "invalid token", // TODO WHAT? + [PM_ERR_INVALID_VARIABLE_GLOBAL] = "invalid global variable", + [PM_ERR_LAMBDA_OPEN] = "expected a `do` keyword or a `{` to open the lambda block", + [PM_ERR_LAMBDA_TERM_BRACE] = "expected a lambda block beginning with `{` to end with `}`", + [PM_ERR_LAMBDA_TERM_END] = "expected a lambda block beginning with `do` to end with `end`", + [PM_ERR_LIST_I_LOWER_ELEMENT] = "expected a symbol in a `%i` list", + [PM_ERR_LIST_I_LOWER_TERM] = "expected a closing delimiter for the `%i` list", + [PM_ERR_LIST_I_UPPER_ELEMENT] = "expected a symbol in a `%I` list", + [PM_ERR_LIST_I_UPPER_TERM] = "expected a closing delimiter for the `%I` list", + [PM_ERR_LIST_W_LOWER_ELEMENT] = "expected a string in a `%w` list", + [PM_ERR_LIST_W_LOWER_TERM] = "expected a closing delimiter for the `%w` list", + [PM_ERR_LIST_W_UPPER_ELEMENT] = "expected a string in a `%W` list", + [PM_ERR_LIST_W_UPPER_TERM] = "expected a closing delimiter for the `%W` list", + [PM_ERR_MALLOC_FAILED] = "failed to allocate memory", + [PM_ERR_MODULE_IN_METHOD] = "unexpected module definition in a method body", + [PM_ERR_MODULE_NAME] = "expected a constant name after `module`", + [PM_ERR_MODULE_TERM] = "expected an `end` to close the `module` statement", + [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = "multiple splats in multiple assignment", + [PM_ERR_NOT_EXPRESSION] = "expected an expression after `not`", + [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = "number literal ending with a `_`", + [PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED] = "numbered parameters are not allowed alongside explicit parameters", + [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = "numbered parameter is already used in outer scope", + [PM_ERR_OPERATOR_MULTI_ASSIGN] = "unexpected operator for a multiple assignment", + [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = "unexpected operator after a call with arguments", + [PM_ERR_OPERATOR_WRITE_BLOCK] = "unexpected operator after a call with a block", + [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = "unexpected multiple `**` splat parameters", + [PM_ERR_PARAMETER_BLOCK_MULTI] = "multiple block parameters; only one block is allowed", + [PM_ERR_PARAMETER_METHOD_NAME] = "unexpected name for a parameter", + [PM_ERR_PARAMETER_NAME_REPEAT] = "repeated parameter name", + [PM_ERR_PARAMETER_NO_DEFAULT] = "expected a default value for the parameter", + [PM_ERR_PARAMETER_NO_DEFAULT_KW] = "expected a default value for the keyword parameter", [PM_ERR_PARAMETER_NUMBERED_RESERVED] = "%.2s is reserved for a numbered parameter", - [PM_ERR_PARAMETER_ORDER] = "Unexpected parameter order", - [PM_ERR_PARAMETER_SPLAT_MULTI] = "Unexpected multiple `*` splat parameters", - [PM_ERR_PARAMETER_STAR] = "Unexpected parameter `*`", - [PM_ERR_PARAMETER_UNEXPECTED_FWD] = "Unexpected `...` in parameters", - [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = "Unexpected `,` in parameters", - [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = "Expected a pattern expression after the `[` operator", - [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = "Expected a pattern expression after `,`", - [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = "Expected a pattern expression after `=>`", - [PM_ERR_PATTERN_EXPRESSION_AFTER_IN] = "Expected a pattern expression after the `in` keyword", - [PM_ERR_PATTERN_EXPRESSION_AFTER_KEY] = "Expected a pattern expression after the key", - [PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN] = "Expected a pattern expression after the `(` operator", - [PM_ERR_PATTERN_EXPRESSION_AFTER_PIN] = "Expected a pattern expression after the `^` pin operator", - [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = "Expected a pattern expression after the `|` operator", - [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = "Expected a pattern expression after the range operator", - [PM_ERR_PATTERN_HASH_KEY] = "Expected a key in the hash pattern", - [PM_ERR_PATTERN_HASH_KEY_LABEL] = "Expected a label as the key in the hash pattern", // TODO // THIS // AND // ABOVE // IS WEIRD - [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = "Expected an identifier after the `=>` operator", - [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = "Expected a label after the `,` in the hash pattern", - [PM_ERR_PATTERN_REST] = "Unexpected rest pattern", - [PM_ERR_PATTERN_TERM_BRACE] = "Expected a `}` to close the pattern expression", - [PM_ERR_PATTERN_TERM_BRACKET] = "Expected a `]` to close the pattern expression", - [PM_ERR_PATTERN_TERM_PAREN] = "Expected a `)` to close the pattern expression", - [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = "Unexpected `||=` in a multiple assignment", - [PM_ERR_REGEXP_TERM] = "Expected a closing delimiter for the regular expression", - [PM_ERR_RESCUE_EXPRESSION] = "Expected a rescued expression", - [PM_ERR_RESCUE_MODIFIER_VALUE] = "Expected a value after the `rescue` modifier", - [PM_ERR_RESCUE_TERM] = "Expected a closing delimiter for the `rescue` clause", - [PM_ERR_RESCUE_VARIABLE] = "Expected an exception variable after `=>` in a rescue statement", - [PM_ERR_RETURN_INVALID] = "Invalid `return` in a class or module body", - [PM_ERR_STATEMENT_ALIAS] = "Unexpected an `alias` at a non-statement position", - [PM_ERR_STATEMENT_POSTEXE_END] = "Unexpected an `END` at a non-statement position", - [PM_ERR_STATEMENT_PREEXE_BEGIN] = "Unexpected a `BEGIN` at a non-statement position", - [PM_ERR_STATEMENT_UNDEF] = "Unexpected an `undef` at a non-statement position", - [PM_ERR_STRING_CONCATENATION] = "Expected a string for concatenation", - [PM_ERR_STRING_INTERPOLATED_TERM] = "Expected a closing delimiter for the interpolated string", - [PM_ERR_STRING_LITERAL_TERM] = "Expected a closing delimiter for the string literal", - [PM_ERR_SYMBOL_INVALID] = "Invalid symbol", // TODO expected symbol? prism.c ~9719 - [PM_ERR_SYMBOL_TERM_DYNAMIC] = "Expected a closing delimiter for the dynamic symbol", - [PM_ERR_SYMBOL_TERM_INTERPOLATED] = "Expected a closing delimiter for the interpolated symbol", - [PM_ERR_TERNARY_COLON] = "Expected a `:` after the true expression of a ternary operator", - [PM_ERR_TERNARY_EXPRESSION_FALSE] = "Expected an expression after `:` in the ternary operator", - [PM_ERR_TERNARY_EXPRESSION_TRUE] = "Expected an expression after `?` in the ternary operator", - [PM_ERR_UNDEF_ARGUMENT] = "Invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", - [PM_ERR_UNARY_RECEIVER_BANG] = "Expected a receiver for unary `!`", - [PM_ERR_UNARY_RECEIVER_MINUS] = "Expected a receiver for unary `-`", - [PM_ERR_UNARY_RECEIVER_PLUS] = "Expected a receiver for unary `+`", - [PM_ERR_UNARY_RECEIVER_TILDE] = "Expected a receiver for unary `~`", - [PM_ERR_UNTIL_TERM] = "Expected an `end` to close the `until` statement", - [PM_ERR_VOID_EXPRESSION] = "Unexpected void value expression", - [PM_ERR_WHILE_TERM] = "Expected an `end` to close the `while` statement", - [PM_ERR_WRITE_TARGET_READONLY] = "Immutable variable as a write target", - [PM_ERR_WRITE_TARGET_UNEXPECTED] = "Unexpected write target", - [PM_ERR_XSTRING_TERM] = "Expected a closing delimiter for the `%x` or backtick string", - [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = "Ambiguous first argument; put parentheses or a space even after `-` operator", - [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS] = "Ambiguous first argument; put parentheses or a space even after `+` operator", - [PM_WARN_AMBIGUOUS_PREFIX_STAR] = "Ambiguous `*` has been interpreted as an argument prefix", - [PM_WARN_AMBIGUOUS_SLASH] = "Ambiguous `/`; wrap regexp in parentheses or add a space after `/` operator", + [PM_ERR_PARAMETER_ORDER] = "unexpected parameter order", + [PM_ERR_PARAMETER_SPLAT_MULTI] = "unexpected multiple `*` splat parameters", + [PM_ERR_PARAMETER_STAR] = "unexpected parameter `*`", + [PM_ERR_PARAMETER_UNEXPECTED_FWD] = "unexpected `...` in parameters", + [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = "unexpected `,` in parameters", + [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = "expected a pattern expression after the `[` operator", + [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = "expected a pattern expression after `,`", + [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = "expected a pattern expression after `=>`", + [PM_ERR_PATTERN_EXPRESSION_AFTER_IN] = "expected a pattern expression after the `in` keyword", + [PM_ERR_PATTERN_EXPRESSION_AFTER_KEY] = "expected a pattern expression after the key", + [PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN] = "expected a pattern expression after the `(` operator", + [PM_ERR_PATTERN_EXPRESSION_AFTER_PIN] = "expected a pattern expression after the `^` pin operator", + [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = "expected a pattern expression after the `|` operator", + [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = "expected a pattern expression after the range operator", + [PM_ERR_PATTERN_HASH_KEY] = "expected a key in the hash pattern", + [PM_ERR_PATTERN_HASH_KEY_LABEL] = "expected a label as the key in the hash pattern", // TODO // THIS // AND // ABOVE // IS WEIRD + [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = "expected an identifier after the `=>` operator", + [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = "expected a label after the `,` in the hash pattern", + [PM_ERR_PATTERN_REST] = "unexpected rest pattern", + [PM_ERR_PATTERN_TERM_BRACE] = "expected a `}` to close the pattern expression", + [PM_ERR_PATTERN_TERM_BRACKET] = "expected a `]` to close the pattern expression", + [PM_ERR_PATTERN_TERM_PAREN] = "expected a `)` to close the pattern expression", + [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = "unexpected `||=` in a multiple assignment", + [PM_ERR_REGEXP_TERM] = "expected a closing delimiter for the regular expression", + [PM_ERR_RESCUE_EXPRESSION] = "expected a rescued expression", + [PM_ERR_RESCUE_MODIFIER_VALUE] = "expected a value after the `rescue` modifier", + [PM_ERR_RESCUE_TERM] = "expected a closing delimiter for the `rescue` clause", + [PM_ERR_RESCUE_VARIABLE] = "expected an exception variable after `=>` in a rescue statement", + [PM_ERR_RETURN_INVALID] = "invalid `return` in a class or module body", + [PM_ERR_STATEMENT_ALIAS] = "unexpected an `alias` at a non-statement position", + [PM_ERR_STATEMENT_POSTEXE_END] = "unexpected an `END` at a non-statement position", + [PM_ERR_STATEMENT_PREEXE_BEGIN] = "unexpected a `BEGIN` at a non-statement position", + [PM_ERR_STATEMENT_UNDEF] = "unexpected an `undef` at a non-statement position", + [PM_ERR_STRING_CONCATENATION] = "expected a string for concatenation", + [PM_ERR_STRING_INTERPOLATED_TERM] = "expected a closing delimiter for the interpolated string", + [PM_ERR_STRING_LITERAL_TERM] = "expected a closing delimiter for the string literal", + [PM_ERR_SYMBOL_INVALID] = "invalid symbol", // TODO expected symbol? prism.c ~9719 + [PM_ERR_SYMBOL_TERM_DYNAMIC] = "expected a closing delimiter for the dynamic symbol", + [PM_ERR_SYMBOL_TERM_INTERPOLATED] = "expected a closing delimiter for the interpolated symbol", + [PM_ERR_TERNARY_COLON] = "expected a `:` after the true expression of a ternary operator", + [PM_ERR_TERNARY_EXPRESSION_FALSE] = "expected an expression after `:` in the ternary operator", + [PM_ERR_TERNARY_EXPRESSION_TRUE] = "expected an expression after `?` in the ternary operator", + [PM_ERR_UNDEF_ARGUMENT] = "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", + [PM_ERR_UNARY_RECEIVER_BANG] = "expected a receiver for unary `!`", + [PM_ERR_UNARY_RECEIVER_MINUS] = "expected a receiver for unary `-`", + [PM_ERR_UNARY_RECEIVER_PLUS] = "expected a receiver for unary `+`", + [PM_ERR_UNARY_RECEIVER_TILDE] = "expected a receiver for unary `~`", + [PM_ERR_UNTIL_TERM] = "expected an `end` to close the `until` statement", + [PM_ERR_VOID_EXPRESSION] = "unexpected void value expression", + [PM_ERR_WHILE_TERM] = "expected an `end` to close the `while` statement", + [PM_ERR_WRITE_TARGET_READONLY] = "immutable variable as a write target", + [PM_ERR_WRITE_TARGET_UNEXPECTED] = "unexpected write target", + [PM_ERR_XSTRING_TERM] = "expected a closing delimiter for the `%x` or backtick string", + [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = "ambiguous first argument; put parentheses or a space even after `-` operator", + [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS] = "ambiguous first argument; put parentheses or a space even after `+` operator", + [PM_WARN_AMBIGUOUS_PREFIX_STAR] = "ambiguous `*` has been interpreted as an argument prefix", + [PM_WARN_AMBIGUOUS_SLASH] = "ambiguous `/`; wrap regexp in parentheses or add a space after `/` operator", [PM_WARN_END_IN_METHOD] = "END in method; use at_exit", }; diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 3289f67a7180a8..bcc96296225de4 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -8,8 +8,8 @@ class ErrorsTest < TestCase def test_constant_path_with_invalid_token_after assert_error_messages "A::$b", [ - "Expected a constant after the `::` operator", - "Expected a newline or semicolon after the statement" + "expected a constant after the `::` operator", + "expected a newline or semicolon after the statement" ] end @@ -26,7 +26,7 @@ def test_module_name_recoverable ) assert_errors expected, "module Parent module end", [ - ["Expected a constant name after `module`", 20..20] + ["expected a constant name after `module`", 20..20] ] end @@ -42,7 +42,7 @@ def test_for_loops_index_missing ) assert_errors expected, "for in 1..10\ni\nend", [ - ["Expected an index after `for`", 0..3] + ["expected an index after `for`", 0..3] ] end @@ -58,9 +58,9 @@ def test_for_loops_only_end ) assert_errors expected, "for end", [ - ["Expected an index after `for`", 0..3], - ["Expected an `in` after the index in a `for` statement", 3..3], - ["Expected a collection after the `in` in a `for` statement", 3..3] + ["expected an index after `for`", 0..3], + ["expected an `in` after the index in a `for` statement", 3..3], + ["expected a collection after the `in` in a `for` statement", 3..3] ] end @@ -73,7 +73,7 @@ def test_pre_execution_missing_brace ) assert_errors expected, "BEGIN 1 }", [ - ["Expected a `{` after `BEGIN`", 5..5] + ["expected a `{` after `BEGIN`", 5..5] ] end @@ -98,37 +98,37 @@ def test_pre_execution_context ) assert_errors expected, "BEGIN { 1 + }", [ - ["Expected an expression after the operator", 11..11] + ["expected an expression after the operator", 11..11] ] end def test_unterminated_embdoc assert_errors expression("1"), "1\n=begin\n", [ - ["Could not find a terminator for the embedded document", 2..9] + ["could not find a terminator for the embedded document", 2..9] ] end def test_unterminated_i_list assert_errors expression("%i["), "%i[", [ - ["Expected a closing delimiter for the `%i` list", 3..3] + ["expected a closing delimiter for the `%i` list", 3..3] ] end def test_unterminated_w_list assert_errors expression("%w["), "%w[", [ - ["Expected a closing delimiter for the `%w` list", 3..3] + ["expected a closing delimiter for the `%w` list", 3..3] ] end def test_unterminated_W_list assert_errors expression("%W["), "%W[", [ - ["Expected a closing delimiter for the `%W` list", 3..3] + ["expected a closing delimiter for the `%W` list", 3..3] ] end def test_unterminated_regular_expression assert_errors expression("/hello"), "/hello", [ - ["Expected a closing delimiter for the regular expression", 1..1] + ["expected a closing delimiter for the regular expression", 1..1] ] end @@ -136,204 +136,204 @@ def test_unterminated_regular_expression_with_heredoc source = "<<-END + /b\nEND\n" assert_errors expression(source), source, [ - ["Expected a closing delimiter for the regular expression", 16..16] + ["expected a closing delimiter for the regular expression", 16..16] ] end def test_unterminated_xstring assert_errors expression("`hello"), "`hello", [ - ["Expected a closing delimiter for the `%x` or backtick string", 1..1] + ["expected a closing delimiter for the `%x` or backtick string", 1..1] ] end def test_unterminated_string assert_errors expression('"hello'), '"hello', [ - ["Expected a closing delimiter for the interpolated string", 1..1] + ["expected a closing delimiter for the interpolated string", 1..1] ] end def test_incomplete_instance_var_string assert_errors expression('%@#@@#'), '%@#@@#', [ - ["Incomplete instance variable", 4..5], - ["Expected a newline or semicolon after the statement", 4..4] + ["incomplete instance variable", 4..5], + ["expected a newline or semicolon after the statement", 4..4] ] end def test_unterminated_s_symbol assert_errors expression("%s[abc"), "%s[abc", [ - ["Expected a closing delimiter for the dynamic symbol", 3..3] + ["expected a closing delimiter for the dynamic symbol", 3..3] ] end def test_unterminated_parenthesized_expression assert_errors expression('(1 + 2'), '(1 + 2', [ - ["Expected a newline or semicolon after the statement", 6..6], - ["Cannot parse the expression", 6..6], - ["Expected a matching `)`", 6..6] + ["expected a newline or semicolon after the statement", 6..6], + ["cannot parse the expression", 6..6], + ["expected a matching `)`", 6..6] ] end def test_missing_terminator_in_parentheses assert_error_messages "(0 0)", [ - "Expected a newline or semicolon after the statement" + "expected a newline or semicolon after the statement" ] end def test_unterminated_argument_expression assert_errors expression('a %'), 'a %', [ - ["Invalid `%` token", 2..3], - ["Expected an expression after the operator", 3..3], + ["invalid `%` token", 2..3], + ["expected an expression after the operator", 3..3], ] end def test_unterminated_interpolated_symbol assert_error_messages ":\"#", [ - "Expected a closing delimiter for the interpolated symbol" + "expected a closing delimiter for the interpolated symbol" ] end def test_cr_without_lf_in_percent_expression assert_errors expression("%\r"), "%\r", [ - ["Invalid `%` token", 0..2], + ["invalid `%` token", 0..2], ] end def test_1_2_3 assert_errors expression("(1, 2, 3)"), "(1, 2, 3)", [ - ["Expected a newline or semicolon after the statement", 2..2], - ["Cannot parse the expression", 2..2], - ["Expected a matching `)`", 2..2], - ["Expected a newline or semicolon after the statement", 2..2], - ["Cannot parse the expression", 2..2], - ["Expected a newline or semicolon after the statement", 5..5], - ["Cannot parse the expression", 5..5], - ["Expected a newline or semicolon after the statement", 8..8], - ["Cannot parse the expression", 8..8] + ["expected a newline or semicolon after the statement", 2..2], + ["cannot parse the expression", 2..2], + ["expected a matching `)`", 2..2], + ["expected a newline or semicolon after the statement", 2..2], + ["cannot parse the expression", 2..2], + ["expected a newline or semicolon after the statement", 5..5], + ["cannot parse the expression", 5..5], + ["expected a newline or semicolon after the statement", 8..8], + ["cannot parse the expression", 8..8] ] end def test_return_1_2_3 assert_error_messages "return(1, 2, 3)", [ - "Expected a newline or semicolon after the statement", - "Cannot parse the expression", - "Expected a matching `)`", - "Expected a newline or semicolon after the statement", - "Cannot parse the expression" + "expected a newline or semicolon after the statement", + "cannot parse the expression", + "expected a matching `)`", + "expected a newline or semicolon after the statement", + "cannot parse the expression" ] end def test_return_1 assert_errors expression("return 1,;"), "return 1,;", [ - ["Expected an argument", 9..9] + ["expected an argument", 9..9] ] end def test_next_1_2_3 assert_errors expression("next(1, 2, 3)"), "next(1, 2, 3)", [ - ["Expected a newline or semicolon after the statement", 6..6], - ["Cannot parse the expression", 6..6], - ["Expected a matching `)`", 6..6], - ["Expected a newline or semicolon after the statement", 12..12], - ["Cannot parse the expression", 12..12] + ["expected a newline or semicolon after the statement", 6..6], + ["cannot parse the expression", 6..6], + ["expected a matching `)`", 6..6], + ["expected a newline or semicolon after the statement", 12..12], + ["cannot parse the expression", 12..12] ] end def test_next_1 assert_errors expression("next 1,;"), "next 1,;", [ - ["Expected an argument", 7..7] + ["expected an argument", 7..7] ] end def test_break_1_2_3 assert_errors expression("break(1, 2, 3)"), "break(1, 2, 3)", [ - ["Expected a newline or semicolon after the statement", 7..7], - ["Cannot parse the expression", 7..7], - ["Expected a matching `)`", 7..7], - ["Expected a newline or semicolon after the statement", 13..13], - ["Cannot parse the expression", 13..13] + ["expected a newline or semicolon after the statement", 7..7], + ["cannot parse the expression", 7..7], + ["expected a matching `)`", 7..7], + ["expected a newline or semicolon after the statement", 13..13], + ["cannot parse the expression", 13..13] ] end def test_break_1 assert_errors expression("break 1,;"), "break 1,;", [ - ["Expected an argument", 8..8] + ["expected an argument", 8..8] ] end def test_argument_forwarding_when_parent_is_not_forwarding assert_errors expression('def a(x, y, z); b(...); end'), 'def a(x, y, z); b(...); end', [ - ["Unexpected `...` when the parent method is not forwarding", 18..21] + ["unexpected `...` when the parent method is not forwarding", 18..21] ] end def test_argument_forwarding_only_effects_its_own_internals assert_errors expression('def a(...); b(...); end; def c(x, y, z); b(...); end'), 'def a(...); b(...); end; def c(x, y, z); b(...); end', [ - ["Unexpected `...` when the parent method is not forwarding", 43..46] + ["unexpected `...` when the parent method is not forwarding", 43..46] ] end def test_top_level_constant_with_downcased_identifier assert_error_messages "::foo", [ - "Expected a constant after the `::` operator", - "Expected a newline or semicolon after the statement" + "expected a constant after the `::` operator", + "expected a newline or semicolon after the statement" ] end def test_top_level_constant_starting_with_downcased_identifier assert_error_messages "::foo::A", [ - "Expected a constant after the `::` operator", - "Expected a newline or semicolon after the statement" + "expected a constant after the `::` operator", + "expected a newline or semicolon after the statement" ] end def test_aliasing_global_variable_with_non_global_variable assert_errors expression("alias $a b"), "alias $a b", [ - ["Invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 9..10] + ["invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 9..10] ] end def test_aliasing_non_global_variable_with_global_variable assert_errors expression("alias a $b"), "alias a $b", [ - ["Invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 8..10] + ["invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 8..10] ] end def test_aliasing_global_variable_with_global_number_variable assert_errors expression("alias $a $1"), "alias $a $1", [ - ["Invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 9..11] + ["invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 9..11] ] end def test_def_with_expression_receiver_and_no_identifier assert_errors expression("def (a); end"), "def (a); end", [ - ["Expected a `.` or `::` after the receiver in a method definition", 7..7], - ["Expected a method name", 7..7] + ["expected a `.` or `::` after the receiver in a method definition", 7..7], + ["expected a method name", 7..7] ] end def test_def_with_multiple_statements_receiver assert_errors expression("def (\na\nb\n).c; end"), "def (\na\nb\n).c; end", [ - ["Expected a matching `)`", 7..7], - ["Expected a `.` or `::` after the receiver in a method definition", 7..7], - ["Expected a method name", 7..7], - ["Cannot parse the expression", 10..10], - ["Cannot parse the expression", 11..11] + ["expected a matching `)`", 7..7], + ["expected a `.` or `::` after the receiver in a method definition", 7..7], + ["expected a method name", 7..7], + ["cannot parse the expression", 10..10], + ["cannot parse the expression", 11..11] ] end def test_def_with_empty_expression_receiver assert_errors expression("def ().a; end"), "def ().a; end", [ - ["Expected a receiver for the method definition", 5..5] + ["expected a receiver for the method definition", 5..5] ] end def test_block_beginning_with_brace_and_ending_with_end assert_error_messages "x.each { x end", [ - "Expected a newline or semicolon after the statement", - "Cannot parse the expression", - "Cannot parse the expression", - "Expected a block beginning with `{` to end with `}`" + "expected a newline or semicolon after the statement", + "cannot parse the expression", + "cannot parse the expression", + "expected a block beginning with `{` to end with `}`" ] end @@ -354,7 +354,7 @@ def test_double_splat_followed_by_splat_argument ) assert_errors expected, "a(**kwargs, *args)", [ - ["Unexpected `*` splat argument after a `**` keyword splat argument", 12..17] + ["unexpected `*` splat argument after a `**` keyword splat argument", 12..17] ] end @@ -372,15 +372,15 @@ def test_arguments_after_block ) assert_errors expected, "a(&block, foo)", [ - ["Unexpected argument after a block argument", 10..13] + ["unexpected argument after a block argument", 10..13] ] end def test_arguments_binding_power_for_and assert_error_messages "foo(*bar and baz)", [ - "Expected a `)` to close the arguments", - "Expected a newline or semicolon after the statement", - "Cannot parse the expression" + "expected a `)` to close the arguments", + "expected a newline or semicolon after the statement", + "cannot parse the expression" ] end @@ -407,7 +407,7 @@ def test_splat_argument_after_keyword_argument ) assert_errors expected, "a(foo: bar, *args)", [ - ["Unexpected `*` splat argument after a `**` keyword splat argument", 12..17] + ["unexpected `*` splat argument after a `**` keyword splat argument", 12..17] ] end @@ -428,7 +428,7 @@ def test_module_definition_in_method_body ) assert_errors expected, "def foo;module A;end;end", [ - ["Unexpected module definition in a method body", 8..14] + ["unexpected module definition in a method body", 8..14] ] end @@ -466,7 +466,7 @@ def test_module_definition_in_method_body_within_block Location() ) - assert_errors expected, <<~RUBY, [["Unexpected module definition in a method body", 21..27]] + assert_errors expected, <<~RUBY, [["unexpected module definition in a method body", 21..27]] def foo bar do module Foo;end @@ -503,7 +503,7 @@ def test_class_definition_in_method_body ) assert_errors expected, "def foo;class A;end;end", [ - ["Unexpected class definition in a method body", 8..13] + ["unexpected class definition in a method body", 8..13] ] end @@ -529,10 +529,10 @@ def test_bad_arguments ) assert_errors expected, "def foo(A, @a, $A, @@a);end", [ - ["Invalid formal argument; formal argument cannot be a constant", 8..9], - ["Invalid formal argument; formal argument cannot be an instance variable", 11..13], - ["Invalid formal argument; formal argument cannot be a global variable", 15..17], - ["Invalid formal argument; formal argument cannot be a class variable", 19..22], + ["invalid formal argument; formal argument cannot be a constant", 8..9], + ["invalid formal argument; formal argument cannot be an instance variable", 11..13], + ["invalid formal argument; formal argument cannot be a global variable", 15..17], + ["invalid formal argument; formal argument cannot be a class variable", 19..22], ] end @@ -600,7 +600,7 @@ def test_do_not_allow_trailing_commas_in_method_parameters ) assert_errors expected, "def foo(a,b,c,);end", [ - ["Unexpected `,` in parameters", 13..14] + ["unexpected `,` in parameters", 13..14] ] end @@ -619,7 +619,7 @@ def test_do_not_allow_trailing_commas_in_lambda_parameters nil ) assert_errors expected, "-> (a, b, ) {}", [ - ["Unexpected `,` in parameters", 8..9] + ["unexpected `,` in parameters", 8..9] ] end @@ -627,13 +627,13 @@ def test_do_not_allow_multiple_codepoints_in_a_single_character_literal expected = StringNode(0, Location(), Location(), nil, "\u0001\u0002") assert_errors expected, '?\u{0001 0002}', [ - ["Invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", 9..12] + ["invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", 9..12] ] end def test_invalid_hex_escape assert_errors expression('"\\xx"'), '"\\xx"', [ - ["Invalid hexadecimal escape sequence", 1..3], + ["invalid hexadecimal escape sequence", 1..3], ] end @@ -641,7 +641,7 @@ def test_do_not_allow_more_than_6_hexadecimal_digits_in_u_Unicode_character_nota expected = StringNode(0, Location(), Location(), Location(), "\u0001") assert_errors expected, '"\u{0000001}"', [ - ["Invalid Unicode escape sequence; maximum length is 6 digits", 4..11], + ["invalid Unicode escape sequence; maximum length is 6 digits", 4..11], ] end @@ -649,13 +649,13 @@ def test_do_not_allow_characters_other_than_0_9_a_f_and_A_F_in_u_Unicode_charact expected = StringNode(0, Location(), Location(), Location(), "\u0000z}") assert_errors expected, '"\u{000z}"', [ - ["Invalid Unicode escape sequence", 7..7], + ["invalid Unicode escape sequence", 7..7], ] end def test_unterminated_unicode_brackets_should_be_a_syntax_error assert_errors expression('?\\u{3'), '?\\u{3', [ - ["Invalid Unicode escape sequence; needs closing `}`", 1..5], + ["invalid Unicode escape sequence; needs closing `}`", 1..5], ] end @@ -683,7 +683,7 @@ def test_method_parameters_after_block Location() ) assert_errors expected, "def foo(&block, a)\nend", [ - ["Unexpected parameter order", 16..17] + ["unexpected parameter order", 16..17] ] end @@ -704,7 +704,7 @@ def test_method_with_arguments_after_anonymous_block ) assert_errors expected, "def foo(&, a)\nend", [ - ["Unexpected parameter order", 11..12] + ["unexpected parameter order", 11..12] ] end @@ -732,7 +732,7 @@ def test_method_parameters_after_arguments_forwarding Location() ) assert_errors expected, "def foo(..., a)\nend", [ - ["Unexpected parameter order", 13..14] + ["unexpected parameter order", 13..14] ] end @@ -760,7 +760,7 @@ def test_keywords_parameters_before_required_parameters Location() ) assert_errors expected, "def foo(b:, a)\nend", [ - ["Unexpected parameter order", 12..13] + ["unexpected parameter order", 12..13] ] end @@ -789,7 +789,7 @@ def test_rest_keywords_parameters_before_required_parameters ) assert_errors expected, "def foo(**rest, b:)\nend", [ - ["Unexpected parameter order", 16..18] + ["unexpected parameter order", 16..18] ] end @@ -810,7 +810,7 @@ def test_double_arguments_forwarding ) assert_errors expected, "def foo(..., ...)\nend", [ - ["Unexpected parameter order", 13..16] + ["unexpected parameter order", 13..16] ] end @@ -839,8 +839,8 @@ def test_multiple_error_in_parameters_order ) assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] + ["unexpected parameter order", 16..17], + ["unexpected parameter order", 19..21] ] end @@ -869,8 +869,8 @@ def test_switching_to_optional_arguments_twice ) assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] + ["unexpected parameter order", 16..17], + ["unexpected parameter order", 19..21] ] end @@ -899,8 +899,8 @@ def test_switching_to_named_arguments_twice ) assert_errors expected, "def foo(**args, a, b:)\nend", [ - ["Unexpected parameter order", 16..17], - ["Unexpected parameter order", 19..21] + ["unexpected parameter order", 16..17], + ["unexpected parameter order", 19..21] ] end @@ -932,7 +932,7 @@ def test_returning_to_optional_parameters_multiple_times ) assert_errors expected, "def foo(a, b = 1, c, d = 2, e)\nend", [ - ["Unexpected parameter order", 23..24] + ["unexpected parameter order", 23..24] ] end @@ -946,7 +946,7 @@ def test_case_without_when_clauses_errors_on_else_clause ) assert_errors expected, "case :a\nelse\nend", [ - ["Expected a `when` or `in` clause after `case`", 0..4] + ["expected a `when` or `in` clause after `case`", 0..4] ] end @@ -960,7 +960,7 @@ def test_case_without_clauses ) assert_errors expected, "case :a\nend", [ - ["Expected a `when` or `in` clause after `case`", 0..4] + ["expected a `when` or `in` clause after `case`", 0..4] ] end @@ -981,7 +981,7 @@ def test_setter_method_cannot_be_defined_in_an_endless_method_definition ) assert_errors expected, "def a=() = 42", [ - ["Invalid method name; a setter method cannot be defined in an endless method definition", 4..6] + ["invalid method name; a setter method cannot be defined in an endless method definition", 4..6] ] end @@ -996,7 +996,7 @@ def test_do_not_allow_forward_arguments_in_lambda_literals ) assert_errors expected, "->(...) {}", [ - ["Unexpected `...` when the parent method is not forwarding", 3..6] + ["unexpected `...` when the parent method is not forwarding", 3..6] ] end @@ -1020,7 +1020,7 @@ def test_do_not_allow_forward_arguments_in_blocks ) assert_errors expected, "a {|...|}", [ - ["Unexpected `...` when the parent method is not forwarding", 4..7] + ["unexpected `...` when the parent method is not forwarding", 4..7] ] end @@ -1037,7 +1037,7 @@ def test_dont_allow_return_inside_class_body ) assert_errors expected, "class A; return; end", [ - ["Invalid `return` in a class or module body", 15..16] + ["invalid `return` in a class or module body", 15..16] ] end @@ -1052,7 +1052,7 @@ def test_dont_allow_return_inside_module_body ) assert_errors expected, "module A; return; end", [ - ["Invalid `return` in a class or module body", 16..17] + ["invalid `return` in a class or module body", 16..17] ] end @@ -1070,8 +1070,8 @@ def test_dont_allow_setting_to_back_and_nth_reference ) assert_errors expected, "begin\n$+ = nil\n$1466 = nil\nend", [ - ["Immutable variable as a write target", 6..8], - ["Immutable variable as a write target", 15..20] + ["immutable variable as a write target", 6..8], + ["immutable variable as a write target", 15..20] ] end @@ -1095,7 +1095,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,a);end", [ - ["Repeated parameter name", 12..13] + ["repeated parameter name", 12..13] ] end @@ -1115,7 +1115,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,*a);end", [ - ["Repeated parameter name", 13..14] + ["repeated parameter name", 13..14] ] expected = DefNode( @@ -1134,7 +1134,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,**a);end", [ - ["Repeated parameter name", 14..15] + ["repeated parameter name", 14..15] ] expected = DefNode( @@ -1153,7 +1153,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,&a);end", [ - ["Repeated parameter name", 13..14] + ["repeated parameter name", 13..14] ] expected = DefNode( @@ -1171,7 +1171,7 @@ def test_duplicated_parameter_names Location() ) - assert_errors expected, "def foo(a = 1,b,*c);end", [["Unexpected parameter `*`", 16..17]] + assert_errors expected, "def foo(a = 1,b,*c);end", [["unexpected parameter `*`", 16..17]] end def test_invalid_message_name @@ -1182,33 +1182,33 @@ def test_invalid_message_name def test_invalid_operator_write_fcall source = "foo! += 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..4] + ["unexpected write target", 0..4] ] end def test_invalid_operator_write_dot source = "foo.+= 1" assert_errors expression(source), source, [ - ["Unexpected write target", 5..6] + ["unexpected write target", 5..6] ] end def test_unterminated_global_variable assert_errors expression("$"), "$", [ - ["Invalid global variable", 0..1] + ["invalid global variable", 0..1] ] end def test_invalid_global_variable_write assert_errors expression("$',"), "$',", [ - ["Immutable variable as a write target", 0..2], - ["Unexpected write target", 0..2] + ["immutable variable as a write target", 0..2], + ["unexpected write target", 0..2] ] end def test_invalid_multi_target - error_messages = ["Unexpected write target"] - immutable = "Immutable variable as a write target" + error_messages = ["unexpected write target"] + immutable = "immutable variable as a write target" assert_error_messages "foo,", error_messages assert_error_messages "foo = 1; foo,", error_messages @@ -1235,51 +1235,51 @@ def test_invalid_multi_target def test_call_with_block_and_write source = "foo {} &&= 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..6], - ["Unexpected operator after a call with a block", 7..10] + ["unexpected write target", 0..6], + ["unexpected operator after a call with a block", 7..10] ] end def test_call_with_block_or_write source = "foo {} ||= 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..6], - ["Unexpected operator after a call with a block", 7..10] + ["unexpected write target", 0..6], + ["unexpected operator after a call with a block", 7..10] ] end def test_call_with_block_operator_write source = "foo {} += 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..6], - ["Unexpected operator after a call with a block", 7..9] + ["unexpected write target", 0..6], + ["unexpected operator after a call with a block", 7..9] ] end def test_index_call_with_block_and_write source = "foo[1] {} &&= 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..9], - ["Unexpected operator after a call with arguments", 10..13], - ["Unexpected operator after a call with a block", 10..13] + ["unexpected write target", 0..9], + ["unexpected operator after a call with arguments", 10..13], + ["unexpected operator after a call with a block", 10..13] ] end def test_index_call_with_block_or_write source = "foo[1] {} ||= 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..9], - ["Unexpected operator after a call with arguments", 10..13], - ["Unexpected operator after a call with a block", 10..13] + ["unexpected write target", 0..9], + ["unexpected operator after a call with arguments", 10..13], + ["unexpected operator after a call with a block", 10..13] ] end def test_index_call_with_block_operator_write source = "foo[1] {} += 1" assert_errors expression(source), source, [ - ["Unexpected write target", 0..9], - ["Unexpected operator after a call with arguments", 10..12], - ["Unexpected operator after a call with a block", 10..12] + ["unexpected write target", 0..9], + ["unexpected operator after a call with arguments", 10..12], + ["unexpected operator after a call with a block", 10..12] ] end @@ -1304,13 +1304,13 @@ def test_defining_numbered_parameter def test_double_scope_numbered_parameters source = "-> { _1 + -> { _2 } }" - errors = [["Numbered parameter is already used in outer scope", 15..17]] + errors = [["numbered parameter is already used in outer scope", 15..17]] assert_errors expression(source), source, errors, compare_ripper: false end def test_invalid_number_underscores - error_messages = ["Invalid underscore placement in number"] + error_messages = ["invalid underscore placement in number"] assert_error_messages "1__1", error_messages assert_error_messages "0b1__1", error_messages @@ -1328,7 +1328,7 @@ def test_invalid_number_underscores end def test_alnum_delimiters - error_messages = ["Invalid `%` token"] + error_messages = ["invalid `%` token"] assert_error_messages "%qXfooX", error_messages assert_error_messages "%QXfooX", error_messages @@ -1358,17 +1358,17 @@ def test_numbered_parameters_in_block_arguments def test_conditional_predicate_closed source = "if 0 0; elsif 0 0; end\nunless 0 0; end" assert_errors expression(source), source, [ - ["Expected `then` or `;` or '\n" + "'", 5..6], - ["Expected `then` or `;` or '\n" + "'", 16..17], - ["Expected `then` or `;` or '\n" + "'", 32..33], + ["expected `then` or `;` or '\n" + "'", 5..6], + ["expected `then` or `;` or '\n" + "'", 16..17], + ["expected `then` or `;` or '\n" + "'", 32..33], ] end def test_parameter_name_ending_with_bang_or_question_mark source = "def foo(x!,y?); end" errors = [ - ["Unexpected name for a parameter", 8..10], - ["Unexpected name for a parameter", 11..13] + ["unexpected name for a parameter", 8..10], + ["unexpected name for a parameter", 11..13] ] assert_errors expression(source), source, errors, compare_ripper: false end @@ -1376,46 +1376,46 @@ def test_parameter_name_ending_with_bang_or_question_mark def test_class_name source = "class 0.X end" assert_errors expression(source), source, [ - ["Expected a constant name after `class`", 6..9], + ["expected a constant name after `class`", 6..9], ] end def test_loop_conditional_is_closed source = "while 0 0; foo; end; until 0 0; foo; end" assert_errors expression(source), source, [ - ["Expected a predicate expression for the `while` statement", 7..7], - ["Expected a predicate expression for the `until` statement", 28..28], + ["expected a predicate expression for the `while` statement", 7..7], + ["expected a predicate expression for the `until` statement", 28..28], ] end def test_forwarding_arg_after_keyword_rest source = "def f(**,...);end" assert_errors expression(source), source, [ - ["Unexpected `...` in parameters", 9..12], + ["unexpected `...` in parameters", 9..12], ] end def test_semicolon_after_inheritance_operator source = "class Foo < Bar end" assert_errors expression(source), source, [ - ["Unexpected `end`, expecting ';' or '\n'", 15..15], + ["unexpected `end`, expecting ';' or '\n'", 15..15], ] end def test_shadow_args_in_lambda source = "->a;b{}" assert_errors expression(source), source, [ - ["Expected a `do` keyword or a `{` to open the lambda block", 3..3], - ["Expected a newline or semicolon after the statement", 7..7], - ["Cannot parse the expression", 7..7], - ["Expected a lambda block beginning with `do` to end with `end`", 7..7], + ["expected a `do` keyword or a `{` to open the lambda block", 3..3], + ["expected a newline or semicolon after the statement", 7..7], + ["cannot parse the expression", 7..7], + ["expected a lambda block beginning with `do` to end with `end`", 7..7], ] end def test_shadow_args_in_block source = "tap{|a;a|}" assert_errors expression(source), source, [ - ["Repeated parameter name", 7..8], + ["repeated parameter name", 7..8], ] end @@ -1424,7 +1424,7 @@ def test_repeated_parameter_name_in_destructured_params # In Ruby 3.0.x, `Ripper.sexp_raw` does not return `nil` for this case. compare_ripper = RUBY_ENGINE == "ruby" && (RUBY_VERSION.split('.').map { |x| x.to_i } <=> [3, 1]) >= 1 assert_errors expression(source), source, [ - ["Repeated parameter name", 14..15], + ["repeated parameter name", 14..15], ], compare_ripper: compare_ripper end @@ -1450,23 +1450,23 @@ def test_assign_to_numbered_parameter def test_symbol_in_keyword_parameter source = "def foo(x:'y':); end" assert_errors expression(source), source, [ - ["Expected a closing delimiter for the string literal", 14..14], + ["expected a closing delimiter for the string literal", 14..14], ] end def test_symbol_in_hash source = "{x:'y':}" assert_errors expression(source), source, [ - ["Expected a closing delimiter for the string literal", 7..7], + ["expected a closing delimiter for the string literal", 7..7], ] end def test_while_endless_method source = "while def f = g do end" assert_errors expression(source), source, [ - ['Expected a predicate expression for the `while` statement', 22..22], - ['Cannot parse the expression', 22..22], - ['Expected an `end` to close the `while` statement', 22..22] + ['expected a predicate expression for the `while` statement', 22..22], + ['cannot parse the expression', 22..22], + ['expected an `end` to close the `while` statement', 22..22] ] end @@ -1475,8 +1475,8 @@ def test_match_plus a in b + c a => b + c RUBY - message1 = 'Expected a newline or semicolon after the statement' - message2 = 'Cannot parse the expression' + message1 = 'expected a newline or semicolon after the statement' + message2 = 'cannot parse the expression' assert_errors expression(source), source, [ [message1, 6..6], [message2, 6..6], @@ -1487,7 +1487,7 @@ def test_match_plus def test_rational_number_with_exponential_portion source = '1e1r; 1e1ri' - message = 'Expected a newline or semicolon after the statement' + message = 'expected a newline or semicolon after the statement' assert_errors expression(source), source, [ [message, 3..3], [message, 9..9] @@ -1507,7 +1507,7 @@ def test_check_value_expression 1 => ^(if 1; (return) else (return) end) 1 => ^(unless 1; (return) else (return) end) RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 7..13], [message, 35..40], @@ -1540,7 +1540,7 @@ class << (return) for x in (return) end RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], [message, 24..30], @@ -1562,7 +1562,7 @@ def x(a = return) def x(a: return) end RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 5..11], [message, 29..35], @@ -1577,7 +1577,7 @@ def test_void_value_expression_in_assignment a, b = return, 1 a, b = 1, *return RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], [message, 18..24], @@ -1595,7 +1595,7 @@ def test_void_value_expression_in_modifier (return) => a (return) in a RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 6..12], [message, 24..30], @@ -1618,7 +1618,7 @@ def test_void_value_expression_in_expression ((return)..) ((return)...) RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], [message, 18..24], @@ -1639,7 +1639,7 @@ def test_void_value_expression_in_hash { a: return } { **return } RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 2..8], [message, 23..29], @@ -1656,7 +1656,7 @@ def test_void_value_expression_in_call (return)[1] = 2 (return)::foo RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], [message, 14..20], @@ -1671,7 +1671,7 @@ def test_void_value_expression_in_constant_path (return)::A class (return)::A; end RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], [message, 19..25], @@ -1689,7 +1689,7 @@ def test_void_value_expression_in_arguments foo(:a => return) foo(a: return) RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], [message, 19..25], @@ -1707,7 +1707,7 @@ def test_void_value_expression_in_unary_call +(return) not return RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 2..8], [message, 14..20], @@ -1723,7 +1723,7 @@ def test_void_value_expression_in_binary_call 1 or (return) (return) or 1 RUBY - message = 'Unexpected void value expression' + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 5..11], [message, 14..20], @@ -1734,29 +1734,29 @@ def test_void_value_expression_in_binary_call def test_trailing_comma_in_calls assert_errors expression("foo 1,"), "foo 1,", [ - ["Expected an argument", 5..6] + ["expected an argument", 5..6] ] end def test_argument_after_ellipsis source = 'def foo(...); foo(..., 1); end' assert_errors expression(source), source, [ - ['Unexpected argument after `...`', 23..24] + ['unexpected argument after `...`', 23..24] ] end def test_ellipsis_in_no_paren_call source = 'def foo(...); foo 1, ...; end' assert_errors expression(source), source, [ - ['Unexpected `...` in an non-parenthesized call', 21..24] + ['unexpected `...` in an non-parenthesized call', 21..24] ] end def test_non_assoc_range source = '1....2' assert_errors expression(source), source, [ - ['Expected a newline or semicolon after the statement', 4..4], - ['Cannot parse the expression', 4..4], + ['expected a newline or semicolon after the statement', 4..4], + ['cannot parse the expression', 4..4], ] end @@ -1777,8 +1777,8 @@ def test_statement_operators undef x + 1 undef x.z RUBY - message1 = 'Expected a newline or semicolon after the statement' - message2 = 'Cannot parse the expression' + message1 = 'expected a newline or semicolon after the statement' + message2 = 'cannot parse the expression' assert_errors expression(source), source, [ [message1, 9..9], [message2, 9..9], @@ -1807,10 +1807,10 @@ def test_statement_at_non_statement foo(undef x) RUBY assert_errors expression(source), source, [ - ['Unexpected an `alias` at a non-statement position', 4..9], - ['Unexpected a `BEGIN` at a non-statement position', 19..24], - ['Unexpected an `END` at a non-statement position', 38..41], - ['Unexpected an `undef` at a non-statement position', 55..60], + ['unexpected an `alias` at a non-statement position', 4..9], + ['unexpected a `BEGIN` at a non-statement position', 19..24], + ['unexpected an `END` at a non-statement position', 38..41], + ['unexpected an `undef` at a non-statement position', 55..60], ] end @@ -1819,8 +1819,8 @@ def test_binary_range_with_left_unary_range ..1.. ...1.. RUBY - message1 = 'Expected a newline or semicolon after the statement' - message2 = 'Cannot parse the expression' + message1 = 'expected a newline or semicolon after the statement' + message2 = 'cannot parse the expression' assert_errors expression(source), source, [ [message1, 3..3], [message2, 3..3],