diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index 9a7855b78475b5..4b58d9d10a1cb6 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -27,7 +27,7 @@ concurrency: # environment variables (plus the "echo $GITHUB_ENV" hack) is to reroute that # restriction. env: - default_cc: clang-17 + default_cc: clang-18 append_cc: '' # -O1 is faster than -O3 in our tests... Majority of time are consumed trying @@ -81,6 +81,7 @@ jobs: optflags: '-O2' shared: disable # check: true + - { name: clang-19, env: { default_cc: clang-19 } } - { name: clang-18, env: { default_cc: clang-18 } } - { name: clang-17, env: { default_cc: clang-17 } } - { name: clang-16, env: { default_cc: clang-16 } } @@ -213,7 +214,7 @@ jobs: runs-on: ubuntu-latest container: - image: ghcr.io/ruby/ruby-ci-image:${{ matrix.entry.container || matrix.entry.env.default_cc || 'clang-17' }} + image: ghcr.io/ruby/ruby-ci-image:${{ matrix.entry.container || matrix.entry.env.default_cc || 'clang-18' }} options: --user root if: >- diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4bb41de22e799f..c89635ea822430 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -88,7 +88,7 @@ jobs: - name: Set up Launchable uses: ./.github/actions/launchable/setup with: - os: ${{ matrix.os }} + os: ${{ matrix.os || (github.repository == 'ruby/ruby' && 'macos-arm-oss' || 'macos-14')}} test-opts: ${{ matrix.test_opts }} launchable-token: ${{ secrets.LAUNCHABLE_TOKEN }} builddir: build diff --git a/.github/workflows/prism.yml b/.github/workflows/prism.yml index 8b295f7f815349..7fc10e2ab12775 100644 --- a/.github/workflows/prism.yml +++ b/.github/workflows/prism.yml @@ -92,7 +92,7 @@ jobs: timeout-minutes: 40 env: GNUMAKEFLAGS: '' - RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb"' + RUBY_TESTOPTS: '-q --tty=no --excludes-dir="../src/test/.excludes-prism" --exclude="test_ast.rb" --exclude="error_highlight/test_error_highlight.rb" --exclude="prism/encoding_test.rb" --exclude="prism/locals_test.rb" --exclude="prism/newline_test.rb"' RUN_OPTS: ${{ matrix.run_opts }} - name: make test-prism-spec diff --git a/LEGAL b/LEGAL index e352c55ee50005..c931291c8ad9b6 100644 --- a/LEGAL +++ b/LEGAL @@ -58,12 +58,12 @@ mentioned below. [ccan/list/list.h] - This file is licensed under the {MIT License}[rdoc-label:label-MIT+License]. + This file is licensed under the {MIT License}[rdoc-ref:@MIT+License]. [coroutine] Unless otherwise specified, these files are licensed under the - {MIT License}[rdoc-label:label-MIT+License]. + {MIT License}[rdoc-ref:@MIT+License]. [include/ruby/onigmo.h] [include/ruby/oniguruma.h] @@ -546,7 +546,7 @@ mentioned below. [vsnprintf.c] - This file is under the {old-style BSD license}[rdoc-label:label-Old-style+BSD+license]. + This file is under the {old-style BSD license}[rdoc-ref:@Old-style+BSD+license]. >>> Copyright (c) 1990, 1993:: @@ -577,7 +577,7 @@ mentioned below. [missing/crypt.c] - This file is under the {old-style BSD license}[rdoc-label:label-Old-style+BSD+license]. + This file is under the {old-style BSD license}[rdoc-ref:@Old-style+BSD+license]. >>> Copyright (c) 1989, 1993:: @@ -588,7 +588,7 @@ mentioned below. [missing/setproctitle.c] - This file is under the {old-style BSD license}[rdoc-label:label-Old-style+BSD+license]. + This file is under the {old-style BSD license}[rdoc-ref:@Old-style+BSD+license]. >>> Copyright 2003:: Damien Miller @@ -879,7 +879,7 @@ mentioned below. >>> RubyGems is copyrighted free software by Chad Fowler, Rich Kilmer, Jim Weirich and others. You can redistribute it and/or modify it under - either the terms of the {MIT license}[rdoc-label:label-MIT+License], or the conditions + either the terms of the {MIT license}[rdoc-ref:@MIT+License], or the conditions below: 1. You may make and give away verbatim copies of the source form of the @@ -941,7 +941,7 @@ mentioned below. Portions copyright (c) 2010:: Andre Arko Portions copyright (c) 2009:: Engine Yard - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/thor] @@ -950,16 +950,16 @@ mentioned below. >>> Copyright (c) 2008 Yehuda Katz, Eric Hodel, et al. - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] -[lib/rubygems/resolver/molinillo] +[lib/rubygems/vendor/molinillo] molinillo is under the following license. >>> Copyright (c) 2014 Samuel E. Giddins segiddins@segiddins.me - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/pub_grub] @@ -968,7 +968,7 @@ mentioned below. >>> Copyright (c) 2018 John Hawthorn - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/connection_pool] @@ -977,7 +977,7 @@ mentioned below. >>> Copyright (c) 2011 Mike Perham - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/bundler/vendor/net-http-persistent] @@ -986,7 +986,7 @@ mentioned below. >>> Copyright (c) Eric Hodel, Aaron Patterson - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/did_you_mean] [lib/did_you_mean.rb] @@ -997,7 +997,7 @@ mentioned below. >>> Copyright (c) 2014-2016 Yuki Nishijima - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [lib/error_highlight] [lib/error_highlight.rb] @@ -1008,7 +1008,7 @@ mentioned below. >>> Copyright (c) 2021 Yusuke Endoh - {MIT License}[rdoc-label:label-MIT+License] + {MIT License}[rdoc-ref:@MIT+License] [benchmark/so_ackermann.rb] [benchmark/so_array.rb] diff --git a/NEWS.md b/NEWS.md index ea274937758cba..77e48d8ca0ef38 100644 --- a/NEWS.md +++ b/NEWS.md @@ -54,13 +54,14 @@ The following default gems are updated. The following bundled gems are updated. * minitest 5.22.3 +* rake 13.2.0 * test-unit 3.6.2 * net-ftp 0.3.4 * net-imap 0.4.10 * net-smtp 0.5.0 * rbs 3.4.4 * typeprof 0.21.11 -* debug 1.9.1 +* debug 1.9.2 The following bundled gems are promoted from default gems. diff --git a/array.c b/array.c index 246b0b26774af7..4040bfcdae0825 100644 --- a/array.c +++ b/array.c @@ -6647,6 +6647,7 @@ ary_sample(rb_execution_context_t *ec, VALUE ary, VALUE randgen, VALUE nv, VALUE }); DATA_PTR(vmemo) = 0; st_free_table(memo); + RB_GC_GUARD(vmemo); } else { result = rb_ary_dup(ary); diff --git a/bootstraptest/test_finalizer.rb b/bootstraptest/test_finalizer.rb index 22a16b1220e784..ccfa0b55d613f2 100644 --- a/bootstraptest/test_finalizer.rb +++ b/bootstraptest/test_finalizer.rb @@ -6,3 +6,11 @@ ObjectSpace.define_finalizer(a2,proc{a1.inspect}) ObjectSpace.define_finalizer(a1,proc{}) }, '[ruby-dev:35778]' + +assert_equal 'true', %q{ + obj = Object.new + id = obj.object_id + + ObjectSpace.define_finalizer(obj, proc { |i| print(id == i) }) + nil +} diff --git a/bootstraptest/test_syntax.rb b/bootstraptest/test_syntax.rb index 18528db7a53396..44bd697d4f9848 100644 --- a/bootstraptest/test_syntax.rb +++ b/bootstraptest/test_syntax.rb @@ -529,7 +529,7 @@ def lines } def assert_syntax_error expected, code, message = '' assert_match /^#{Regexp.escape(expected)}/, - "begin eval(%q{#{code}}, nil, '', 0)"'; rescue SyntaxError => e; e.message[/(?:\A:(?:\d+:)?(?: syntax error,)?|\^) (.*)/, 1] end', message + "begin eval(%q{#{code}}, nil, '', 0)"'; rescue SyntaxError => e; e.message[/(?:\^|\A:(?:\d+:)?(?! syntax errors? found)(?: syntax error,)?) (.*)/, 1] end', message end assert_syntax_error "unterminated string meets end of file", '().."', '[ruby-dev:29732]' assert_equal %q{[]}, %q{$&;[]}, '[ruby-dev:31068]' diff --git a/bootstraptest/test_yjit.rb b/bootstraptest/test_yjit.rb index e5a160fa9fa342..6b7d4839265d04 100644 --- a/bootstraptest/test_yjit.rb +++ b/bootstraptest/test_yjit.rb @@ -4769,3 +4769,20 @@ def tests tests } + +# test integer left shift with constant rhs +assert_equal [0x80000000000, 'a+', :ok].inspect, %q{ + def shift(val) = val << 43 + + def tests + int = shift(1) + str = shift("a") + + Integer.define_method(:<<) { |_| :ok } + redef = shift(1) + + [int, str, redef] + end + + tests +} diff --git a/common.mk b/common.mk index 9dbca5b731da3f..173b4fb2a725bd 100644 --- a/common.mk +++ b/common.mk @@ -730,10 +730,10 @@ clean-local:: clean-runnable bin/clean-runnable:: PHONY $(Q)$(CHDIR) bin 2>$(NULL) && $(RM) $(PROGRAM) $(WPROGRAM) $(GORUBY)$(EXEEXT) bin/*.$(DLEXT) 2>$(NULL) || $(NULLCMD) lib/clean-runnable:: PHONY - $(Q)$(CHDIR) lib 2>$(NULL) && $(RM) $(LIBRUBY_A) $(LIBRUBY) $(LIBRUBY_ALIASES) $(RUBY_BASE_NAME)/$(RUBY_PROGRAM_VERSION) $(RUBY_BASE_NAME)/vendor_ruby 2>$(NULL) || $(NULLCMD) + $(Q)$(CHDIR) lib 2>$(NULL) && $(RM) $(LIBRUBY_A) $(LIBRUBY) $(LIBRUBY_ALIASES) $(RUBY_BASE_NAME)/$(ruby_version) $(RUBY_BASE_NAME)/vendor_ruby 2>$(NULL) || $(NULLCMD) clean-runnable:: bin/clean-runnable lib/clean-runnable PHONY $(Q)$(RMDIR) lib/$(RUBY_BASE_NAME) lib bin 2>$(NULL) || $(NULLCMD) - -$(Q)$(RM) $(EXTOUT)/$(arch)/rbconfig.rb + -$(Q)$(RM) $(EXTOUT)/$(arch)/rbconfig.rb $(EXTOUT)/common/$(arch) -$(Q)$(RMALL) exe/ clean-ext:: PHONY clean-golf: PHONY diff --git a/compile.c b/compile.c index b4f1573499ed0d..3d81499a23c82f 100644 --- a/compile.c +++ b/compile.c @@ -1000,6 +1000,7 @@ rb_iseq_translate_threaded_code(rb_iseq_t *iseq) #if USE_YJIT rb_yjit_live_iseq_count++; + rb_yjit_iseq_alloc_count++; #endif return COMPILE_OK; @@ -11322,6 +11323,7 @@ iseq_build_from_ary_body(rb_iseq_t *iseq, LINK_ANCHOR *const anchor, } } DATA_PTR(labels_wrapper) = 0; + RB_GC_GUARD(labels_wrapper); validate_labels(iseq, labels_table); if (!ret) return ret; return iseq_setup(iseq, anchor); diff --git a/configure.ac b/configure.ac index 530c073764caa2..b09f7f300f50f3 100644 --- a/configure.ac +++ b/configure.ac @@ -75,8 +75,10 @@ AC_ARG_WITH(baseruby, [ AC_PATH_PROG([BASERUBY], [ruby], [false]) ]) -# BASERUBY must be >= 3.0.0. Note that `"3.0.0" > "3.0"` is true. -AS_IF([test "$HAVE_BASERUBY" != no -a "`RUBYOPT=- $BASERUBY --disable=gems -e 'print true if RUBY_VERSION > "3.0"' 2>/dev/null`" = true], [ +AS_IF([test "$HAVE_BASERUBY" != no], [ + RUBYOPT=- $BASERUBY --disable=gems "${tooldir}/missing-baseruby.bat" || HAVE_BASERUBY=no +]) +AS_IF([test "${HAVE_BASERUBY:=no}" != no], [ AS_CASE(["$build_os"], [mingw*], [ # Can MSys shell run a command with a drive letter? RUBYOPT=- `cygpath -ma "$BASERUBY"` --disable=gems -e exit 2>/dev/null || HAVE_BASERUBY=no @@ -84,12 +86,10 @@ AS_IF([test "$HAVE_BASERUBY" != no -a "`RUBYOPT=- $BASERUBY --disable=gems -e 'p RUBY_APPEND_OPTION(BASERUBY, "--disable=gems") BASERUBY_VERSION=`$BASERUBY -v` $BASERUBY -C "$srcdir" tool/downloader.rb -d tool -e gnu config.guess config.sub >&AS_MESSAGE_FD -], [ - HAVE_BASERUBY=no ]) AS_IF([test "$HAVE_BASERUBY" = no], [ AS_IF([test "$cross_compiling" = yes], [AC_MSG_ERROR([executable host ruby is required for cross-compiling])]) - BASERUBY=$srcdir/tool/missing-baseruby.bat + BASERUBY=${tooldir}/missing-baseruby.bat ]) AC_SUBST(BASERUBY) AC_SUBST(HAVE_BASERUBY) @@ -1432,7 +1432,7 @@ AC_SYS_LARGEFILE # which is not added by AC_SYS_LARGEFILE. AS_IF([test x"$enable_largefile" != xno], [ AS_CASE(["$target_os"], [solaris*], [ - AC_MSG_CHECKING([wheather _LARGEFILE_SOURCE should be defined]) + AC_MSG_CHECKING([whether _LARGEFILE_SOURCE should be defined]) AS_CASE(["${ac_cv_sys_file_offset_bits}:${ac_cv_sys_large_files}"], ["64:"|"64:no"|"64:unknown"], [ # insert _LARGEFILE_SOURCE before _FILE_OFFSET_BITS line @@ -2973,7 +2973,7 @@ AS_IF([test "x$ac_cv_func_ioctl" = xyes], [ } [begin]_group "runtime section" && { -dnl wheather use dln_a_out or not +dnl whether use dln_a_out or not AC_ARG_WITH(dln-a-out, AS_HELP_STRING([--with-dln-a-out], [dln_a_out is deprecated]), [ diff --git a/darray.h b/darray.h index bf3dd9954272f8..41d386e4563347 100644 --- a/darray.h +++ b/darray.h @@ -45,16 +45,12 @@ * void rb_darray_append(rb_darray(T) *ptr_to_ary, T element); */ #define rb_darray_append(ptr_to_ary, element) \ - rb_darray_append_impl(ptr_to_ary, element, rb_xrealloc_mul_add) + rb_darray_append_impl(ptr_to_ary, element) -#define rb_darray_append_without_gc(ptr_to_ary, element) \ - rb_darray_append_impl(ptr_to_ary, element, rb_darray_realloc_mul_add_without_gc) - -#define rb_darray_append_impl(ptr_to_ary, element, realloc_func) do { \ +#define rb_darray_append_impl(ptr_to_ary, element) do { \ rb_darray_ensure_space((ptr_to_ary), \ sizeof(**(ptr_to_ary)), \ - sizeof((*(ptr_to_ary))->data[0]), \ - realloc_func); \ + sizeof((*(ptr_to_ary))->data[0])); \ rb_darray_set(*(ptr_to_ary), \ (*(ptr_to_ary))->meta.size, \ (element)); \ @@ -79,21 +75,15 @@ * void rb_darray_make(rb_darray(T) *ptr_to_ary, size_t size); */ #define rb_darray_make(ptr_to_ary, size) \ - rb_darray_make_impl((ptr_to_ary), size, sizeof(**(ptr_to_ary)), \ - sizeof((*(ptr_to_ary))->data[0]), rb_xcalloc_mul_add) - -#define rb_darray_make_without_gc(ptr_to_ary, size) \ - rb_darray_make_impl((ptr_to_ary), size, sizeof(**(ptr_to_ary)), \ - sizeof((*(ptr_to_ary))->data[0]), rb_darray_calloc_mul_add_without_gc) + rb_darray_make_impl((ptr_to_ary), size, sizeof(**(ptr_to_ary)), sizeof((*(ptr_to_ary))->data[0])) /* Resize the darray to a new capacity. The new capacity must be greater than * or equal to the size of the darray. * * void rb_darray_resize_capa(rb_darray(T) *ptr_to_ary, size_t capa); */ -#define rb_darray_resize_capa_without_gc(ptr_to_ary, capa) \ - rb_darray_resize_capa_impl((ptr_to_ary), rb_darray_next_power_of_two(capa), sizeof(**(ptr_to_ary)), \ - sizeof((*(ptr_to_ary))->data[0]), rb_darray_realloc_mul_add_without_gc) +#define rb_darray_resize_capa(ptr_to_ary, capa) \ + rb_darray_resize_capa_impl((ptr_to_ary), capa, sizeof(**(ptr_to_ary)), sizeof((*(ptr_to_ary))->data[0])) #define rb_darray_data_ptr(ary) ((ary)->data) @@ -139,56 +129,15 @@ rb_darray_free(void *ary) if (meta) ruby_sized_xfree(ary, meta->capa); } -static inline void -rb_darray_free_without_gc(void *ary) -{ - free(ary); -} - -/* Internal function. Like rb_xcalloc_mul_add but does not trigger GC and does - * not check for overflow in arithmetic. */ -static inline void * -rb_darray_calloc_mul_add_without_gc(size_t x, size_t y, size_t z) -{ - size_t size = (x * y) + z; - - void *ptr = calloc(1, size); - if (ptr == NULL) rb_bug("rb_darray_calloc_mul_add_without_gc: failed"); - - return ptr; -} - -/* Internal function. Like rb_xrealloc_mul_add but does not trigger GC and does - * not check for overflow in arithmetic. */ -static inline void * -rb_darray_realloc_mul_add_without_gc(const void *orig_ptr, size_t x, size_t y, size_t z) -{ - size_t size = (x * y) + z; - - void *ptr = realloc((void *)orig_ptr, size); - if (ptr == NULL) rb_bug("rb_darray_realloc_mul_add_without_gc: failed"); - - return ptr; -} - -/* Internal function. Returns the next power of two that is greater than or - * equal to n. */ -static inline size_t -rb_darray_next_power_of_two(size_t n) -{ - return (size_t)(1 << (64 - nlz_int64(n))); -} - /* Internal function. Resizes the capacity of a darray. The new capacity must * be greater than or equal to the size of the darray. */ static inline void -rb_darray_resize_capa_impl(void *ptr_to_ary, size_t new_capa, size_t header_size, size_t element_size, - void *(*realloc_mul_add_impl)(const void *, size_t, size_t, size_t)) +rb_darray_resize_capa_impl(void *ptr_to_ary, size_t new_capa, size_t header_size, size_t element_size) { rb_darray_meta_t **ptr_to_ptr_to_meta = ptr_to_ary; rb_darray_meta_t *meta = *ptr_to_ptr_to_meta; - rb_darray_meta_t *new_ary = realloc_mul_add_impl(meta, new_capa, element_size, header_size); + rb_darray_meta_t *new_ary = rb_xrealloc_mul_add(meta, new_capa, element_size, header_size); if (meta == NULL) { /* First allocation. Initialize size. On subsequence allocations @@ -209,8 +158,7 @@ rb_darray_resize_capa_impl(void *ptr_to_ary, size_t new_capa, size_t header_size // Ensure there is space for one more element. // Note: header_size can be bigger than sizeof(rb_darray_meta_t) when T is __int128_t, for example. static inline void -rb_darray_ensure_space(void *ptr_to_ary, size_t header_size, size_t element_size, - void *(*realloc_mul_add_impl)(const void *, size_t, size_t, size_t)) +rb_darray_ensure_space(void *ptr_to_ary, size_t header_size, size_t element_size) { rb_darray_meta_t **ptr_to_ptr_to_meta = ptr_to_ary; rb_darray_meta_t *meta = *ptr_to_ptr_to_meta; @@ -220,12 +168,11 @@ rb_darray_ensure_space(void *ptr_to_ary, size_t header_size, size_t element_size // Double the capacity size_t new_capa = current_capa == 0 ? 1 : current_capa * 2; - rb_darray_resize_capa_impl(ptr_to_ary, new_capa, header_size, element_size, realloc_mul_add_impl); + rb_darray_resize_capa_impl(ptr_to_ary, new_capa, header_size, element_size); } static inline void -rb_darray_make_impl(void *ptr_to_ary, size_t array_size, size_t header_size, size_t element_size, - void *(*calloc_mul_add_impl)(size_t, size_t, size_t)) +rb_darray_make_impl(void *ptr_to_ary, size_t array_size, size_t header_size, size_t element_size) { rb_darray_meta_t **ptr_to_ptr_to_meta = ptr_to_ary; if (array_size == 0) { @@ -233,7 +180,7 @@ rb_darray_make_impl(void *ptr_to_ary, size_t array_size, size_t header_size, siz return; } - rb_darray_meta_t *meta = calloc_mul_add_impl(array_size, element_size, header_size); + rb_darray_meta_t *meta = rb_xcalloc_mul_add(array_size, element_size, header_size); meta->size = array_size; meta->capa = array_size; diff --git a/doc/ChangeLog/ChangeLog-2.1.0 b/doc/ChangeLog/ChangeLog-2.1.0 index 5b670b31c97c8c..7964a682ebd9c8 100644 --- a/doc/ChangeLog/ChangeLog-2.1.0 +++ b/doc/ChangeLog/ChangeLog-2.1.0 @@ -659,7 +659,7 @@ Mon Dec 9 02:10:32 2013 NARUSE, Yui * lib/net/http/responses.rb: Add `HTTPIMUsed`, as it is also supported by rack/rails. - RFC - http://tools.ietf.org/html/rfc3229 + RFC - https://www.rfc-editor.org/rfc/rfc3229 by Vipul A M https://github.com/ruby/ruby/pull/447 fix GH-447 diff --git a/doc/ChangeLog/ChangeLog-2.2.0 b/doc/ChangeLog/ChangeLog-2.2.0 index 5a7dbf826d3135..0edcf0122b190d 100644 --- a/doc/ChangeLog/ChangeLog-2.2.0 +++ b/doc/ChangeLog/ChangeLog-2.2.0 @@ -5648,7 +5648,7 @@ Wed Aug 6 04:16:05 2014 NARUSE, Yui * lib/net/http/requests.rb (Net::HTTP::Options::RESPONSE_HAS_BODY): OPTIONS requests may have response bodies. [Feature #8429] - http://tools.ietf.org/html/rfc7231#section-4.3.7 + https://www.rfc-editor.org/rfc/rfc7231#section-4.3.7 Wed Aug 6 03:18:04 2014 NARUSE, Yui diff --git a/doc/ChangeLog/ChangeLog-2.4.0 b/doc/ChangeLog/ChangeLog-2.4.0 index a297a579d1d4ba..30e9ccab3d9aac 100644 --- a/doc/ChangeLog/ChangeLog-2.4.0 +++ b/doc/ChangeLog/ChangeLog-2.4.0 @@ -7356,7 +7356,7 @@ Thu Mar 17 11:51:48 2016 NARUSE, Yui Note: CryptGenRandom function is PRNG and doesn't check its entropy, so it won't block. [Bug #12139] https://msdn.microsoft.com/ja-jp/library/windows/desktop/aa379942.aspx - https://tools.ietf.org/html/rfc4086#section-7.1.3 + https://www.rfc-editor.org/rfc/rfc4086#section-7.1.3 https://eprint.iacr.org/2007/419.pdf http://www.cs.huji.ac.il/~dolev/pubs/thesis/msc-thesis-leo.pdf diff --git a/doc/contributing/building_ruby.md b/doc/contributing/building_ruby.md index 18464458896b74..2991385b94a6e6 100644 --- a/doc/contributing/building_ruby.md +++ b/doc/contributing/building_ruby.md @@ -181,12 +181,16 @@ mkdir build && cd build ../configure CC=clang cflags="-fsanitize=address -fno-omit-frame-pointer -DUSE_MN_THREADS=0" # and any other options you might like make ``` -The compiled Ruby will now automatically crash with a report and a backtrace if ASAN detects a memory safety issue. +The compiled Ruby will now automatically crash with a report and a backtrace if ASAN detects a memory safety issue. To run Ruby's test suite under ASAN, issue the following command. Note that this will take quite a long time (over two hours on my laptop); the `RUBY_TEST_TIMEOUT_SCALE` and `SYNTAX_SUGEST_TIMEOUT` variables are required to make sure tests don't spuriously fail with timeouts when in fact they're just slow. + +``` shell +RUBY_TEST_TIMEOUT_SCALE=5 SYNTAX_SUGGEST_TIMEOUT=600 make check +``` Please note, however, the following caveats! -* ASAN will not work properly on any currently released version of Ruby; the necessary support is currently only present on Ruby's master branch. -* Due to [this bug](https://bugs.ruby-lang.org/issues/20243), Clang generates code for threadlocal variables which doesn't work with M:N threading. Thus, it's necessary to disable M:N threading support at build time for now. +* ASAN will not work properly on any currently released version of Ruby; the necessary support is currently only present on Ruby's master branch (and the whole test suite passes only as of commit [9d0a5148ae062a0481a4a18fbeb9cfd01dc10428](https://bugs.ruby-lang.org/projects/ruby-master/repository/git/revisions/9d0a5148ae062a0481a4a18fbeb9cfd01dc10428)) +* Due to [this bug](https://bugs.ruby-lang.org/issues/20243), Clang generates code for threadlocal variables which doesn't work with M:N threading. Thus, it's necessary to disable M:N threading support at build time for now (with the `-DUSE_MN_THREADS=0` configure argument). * Currently, ASAN will only work correctly when using a recent head build of LLVM/Clang - it requires [this bugfix](https://github.com/llvm/llvm-project/pull/75290) related to multithreaded `fork`, which is not yet in any released version. See [here](https://llvm.org/docs/CMake.html) for instructions on how to build LLVM/Clang from source (note you will need at least the `clang` and `compiler-rt` projects enabled). Then, you will need to replace `CC=clang` in the instructions with an explicit path to your built Clang binary. * ASAN has only been tested so far with Clang on Linux. It may or may not work with other compilers or on other platforms - please file an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if you run into problems with such configurations (or, to report that they actually work properly!) * In particular, although I have not yet tried it, I have reason to believe ASAN will _not_ work properly on macOS yet - the fix for the multithreaded fork issue was actually reverted for macOS (see [here](https://github.com/llvm/llvm-project/commit/2a03854e4ce9bb1bcd79a211063bc63c4657f92c)). Please open an issue on [https://bugs.ruby-lang.org](https://bugs.ruby-lang.org) if this is a problem for you. diff --git a/doc/csv/recipes/generating.rdoc b/doc/csv/recipes/generating.rdoc index a6bd88a7147a1e..e61838d31a5b3c 100644 --- a/doc/csv/recipes/generating.rdoc +++ b/doc/csv/recipes/generating.rdoc @@ -163,7 +163,7 @@ This example defines and uses two custom write converters to strip and upcase ge === RFC 4180 Compliance By default, \CSV generates data that is compliant with -{RFC 4180}[https://tools.ietf.org/html/rfc4180] +{RFC 4180}[https://www.rfc-editor.org/rfc/rfc4180] with respect to: - Column separator. - Quote character. diff --git a/doc/csv/recipes/parsing.rdoc b/doc/csv/recipes/parsing.rdoc index f3528fbdf1bc8b..1b7071e33f6543 100644 --- a/doc/csv/recipes/parsing.rdoc +++ b/doc/csv/recipes/parsing.rdoc @@ -191,7 +191,7 @@ Output: === RFC 4180 Compliance By default, \CSV parses data that is compliant with -{RFC 4180}[https://tools.ietf.org/html/rfc4180] +{RFC 4180}[https://www.rfc-editor.org/rfc/rfc4180] with respect to: - Row separator. - Column separator. diff --git a/doc/strftime_formatting.rdoc b/doc/strftime_formatting.rdoc index 7694752a21c1ec..5c7b33155df9ec 100644 --- a/doc/strftime_formatting.rdoc +++ b/doc/strftime_formatting.rdoc @@ -356,7 +356,7 @@ each based on an external standard. == HTTP Format The HTTP date format is based on -{RFC 2616}[https://datatracker.ietf.org/doc/html/rfc2616], +{RFC 2616}[https://www.rfc-editor.org/rfc/rfc2616], and treats dates in the format '%a, %d %b %Y %T GMT': d = Date.new(2001, 2, 3) # => # @@ -371,7 +371,7 @@ and treats dates in the format '%a, %d %b %Y %T GMT': == RFC 3339 Format The RFC 3339 date format is based on -{RFC 3339}[https://datatracker.ietf.org/doc/html/rfc3339]: +{RFC 3339}[https://www.rfc-editor.org/rfc/rfc3339]: d = Date.new(2001, 2, 3) # => # # Return 3339-formatted string. @@ -385,7 +385,7 @@ The RFC 3339 date format is based on == RFC 2822 Format The RFC 2822 date format is based on -{RFC 2822}[https://datatracker.ietf.org/doc/html/rfc2822], +{RFC 2822}[https://www.rfc-editor.org/rfc/rfc2822], and treats dates in the format '%a, %-d %b %Y %T %z']: d = Date.new(2001, 2, 3) # => # diff --git a/enc/ebcdic.h b/enc/ebcdic.h index a3b380a32760ad..5109bf7065abb1 100644 --- a/enc/ebcdic.h +++ b/enc/ebcdic.h @@ -7,5 +7,5 @@ ENC_ALIAS("ebcdic-cp-us", "IBM037"); * hopefully the most widely used one. * * See http://www.iana.org/assignments/character-sets/character-sets.xhtml - * http://tools.ietf.org/html/rfc1345 + * https://www.rfc-editor.org/rfc/rfc1345 */ diff --git a/ext/openssl/ossl_kdf.c b/ext/openssl/ossl_kdf.c index 48b161d4f47749..ba197a659ec777 100644 --- a/ext/openssl/ossl_kdf.c +++ b/ext/openssl/ossl_kdf.c @@ -18,7 +18,7 @@ static VALUE mKDF, eKDF; * of _length_ bytes. * * For more information about PBKDF2, see RFC 2898 Section 5.2 - * (https://tools.ietf.org/html/rfc2898#section-5.2). + * (https://www.rfc-editor.org/rfc/rfc2898#section-5.2). * * === Parameters * pass :: The password. @@ -81,10 +81,10 @@ kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self) * bcrypt. * * The keyword arguments _N_, _r_ and _p_ can be used to tune scrypt. RFC 7914 - * (published on 2016-08, https://tools.ietf.org/html/rfc7914#section-2) states + * (published on 2016-08, https://www.rfc-editor.org/rfc/rfc7914#section-2) states * that using values r=8 and p=1 appears to yield good results. * - * See RFC 7914 (https://tools.ietf.org/html/rfc7914) for more information. + * See RFC 7914 (https://www.rfc-editor.org/rfc/rfc7914) for more information. * * === Parameters * pass :: Passphrase. @@ -147,7 +147,7 @@ kdf_scrypt(int argc, VALUE *argv, VALUE self) * KDF.hkdf(ikm, salt:, info:, length:, hash:) -> String * * HMAC-based Extract-and-Expand Key Derivation Function (HKDF) as specified in - * {RFC 5869}[https://tools.ietf.org/html/rfc5869]. + * {RFC 5869}[https://www.rfc-editor.org/rfc/rfc5869]. * * New in OpenSSL 1.1.0. * @@ -165,7 +165,7 @@ kdf_scrypt(int argc, VALUE *argv, VALUE self) * The hash function. * * === Example - * # The values from https://datatracker.ietf.org/doc/html/rfc5869#appendix-A.1 + * # The values from https://www.rfc-editor.org/rfc/rfc5869#appendix-A.1 * ikm = ["0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b"].pack("H*") * salt = ["000102030405060708090a0b0c"].pack("H*") * info = ["f0f1f2f3f4f5f6f7f8f9"].pack("H*") diff --git a/ext/openssl/ossl_ns_spki.c b/ext/openssl/ossl_ns_spki.c index 9bed1f330ec5e2..9d70b5d87a5d59 100644 --- a/ext/openssl/ossl_ns_spki.c +++ b/ext/openssl/ossl_ns_spki.c @@ -365,8 +365,8 @@ ossl_spki_verify(VALUE self, VALUE key) * * OpenSSL::Netscape is a namespace for SPKI (Simple Public Key * Infrastructure) which implements Signed Public Key and Challenge. - * See {RFC 2692}[http://tools.ietf.org/html/rfc2692] and {RFC - * 2693}[http://tools.ietf.org/html/rfc2692] for details. + * See {RFC 2692}[https://www.rfc-editor.org/rfc/rfc2692] and {RFC + * 2693}[https://www.rfc-editor.org/rfc/rfc2692] for details. */ /* Document-class: OpenSSL::Netscape::SPKIError diff --git a/ext/ripper/ripper_init.c.tmpl b/ext/ripper/ripper_init.c.tmpl index 894cbc0b447b51..c9d381da5b151e 100644 --- a/ext/ripper/ripper_init.c.tmpl +++ b/ext/ripper/ripper_init.c.tmpl @@ -2,14 +2,13 @@ #include "ruby/ruby.h" #include "ruby/encoding.h" #include "internal.h" -#include "internal/imemo.h" /* needed by ruby_parser.h */ +#include "rubyparser.h" +#define YYSTYPE_IS_DECLARED +#include "parse.h" #include "internal/parse.h" #include "internal/ruby_parser.h" #include "node.h" -#include "rubyparser.h" #include "eventids1.h" -#define YYSTYPE_IS_DECLARED -#include "parse.h" #include "eventids2.h" #include "ripper_init.h" diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c index 1ce90030a82126..f0e5ee4e85ff09 100644 --- a/ext/stringio/stringio.c +++ b/ext/stringio/stringio.c @@ -15,6 +15,8 @@ static const char *const STRINGIO_VERSION = "3.1.1"; +#include + #include "ruby.h" #include "ruby/io.h" #include "ruby/encoding.h" @@ -49,6 +51,19 @@ static long strio_write(VALUE self, VALUE str); #define IS_STRIO(obj) (rb_typeddata_is_kind_of((obj), &strio_data_type)) #define error_inval(msg) (rb_syserr_fail(EINVAL, msg)) #define get_enc(ptr) ((ptr)->enc ? (ptr)->enc : !NIL_P((ptr)->string) ? rb_enc_get((ptr)->string) : NULL) +#ifndef HAVE_RB_STR_CHILLED_P +static bool +rb_str_chilled_p(VALUE str) +{ + return false; +} +#endif + +static bool +readonly_string_p(VALUE string) +{ + return OBJ_FROZEN_RAW(string) && !rb_str_chilled_p(string); +} static struct StringIO * strio_alloc(void) @@ -166,7 +181,13 @@ writable(VALUE strio) static void check_modifiable(struct StringIO *ptr) { - if (OBJ_FROZEN(ptr->string)) { + if (NIL_P(ptr->string)) { + /* Null device StringIO */ + } + else if (rb_str_chilled_p(ptr->string)) { + rb_str_modify(ptr->string); + } + else if (OBJ_FROZEN_RAW(ptr->string)) { rb_raise(rb_eIOError, "not modifiable string"); } } @@ -287,7 +308,8 @@ strio_init(int argc, VALUE *argv, struct StringIO *ptr, VALUE self) else if (!argc) { string = rb_enc_str_new("", 0, rb_default_external_encoding()); } - if (!NIL_P(string) && OBJ_FROZEN_RAW(string)) { + + if (!NIL_P(string) && readonly_string_p(string)) { if (ptr->flags & FMODE_WRITABLE) { rb_syserr_fail(EACCES, 0); } @@ -481,7 +503,7 @@ strio_set_string(VALUE self, VALUE string) rb_io_taint_check(self); ptr->flags &= ~FMODE_READWRITE; StringValue(string); - ptr->flags = OBJ_FROZEN(string) ? FMODE_READABLE : FMODE_READWRITE; + ptr->flags = readonly_string_p(string) ? FMODE_READABLE : FMODE_READWRITE; ptr->pos = 0; ptr->lineno = 0; RB_OBJ_WRITE(self, &ptr->string, string); @@ -973,10 +995,10 @@ strio_ungetbyte(VALUE self, VALUE c) if (NIL_P(ptr->string)) return Qnil; if (NIL_P(c)) return Qnil; if (RB_INTEGER_TYPE_P(c)) { - /* rb_int_and() not visible from exts */ - VALUE v = rb_funcall(c, '&', 1, INT2FIX(0xff)); - const char cc = NUM2INT(v) & 0xFF; - strio_unget_bytes(ptr, &cc, 1); + /* rb_int_and() not visible from exts */ + VALUE v = rb_funcall(c, '&', 1, INT2FIX(0xff)); + const char cc = NUM2INT(v) & 0xFF; + strio_unget_bytes(ptr, &cc, 1); } else { long cl; @@ -1157,41 +1179,41 @@ prepare_getline_args(struct StringIO *ptr, struct getline_arg *arg, int argc, VA break; case 1: - if (!NIL_P(rs) && !RB_TYPE_P(rs, T_STRING)) { - VALUE tmp = rb_check_string_type(rs); + if (!NIL_P(rs) && !RB_TYPE_P(rs, T_STRING)) { + VALUE tmp = rb_check_string_type(rs); if (NIL_P(tmp)) { - limit = NUM2LONG(rs); - rs = rb_rs; + limit = NUM2LONG(rs); + rs = rb_rs; } else { - rs = tmp; + rs = tmp; } } break; case 2: - if (!NIL_P(rs)) StringValue(rs); + if (!NIL_P(rs)) StringValue(rs); if (!NIL_P(lim)) limit = NUM2LONG(lim); break; } if (!NIL_P(ptr->string) && !NIL_P(rs)) { - rb_encoding *enc_rs, *enc_io; - enc_rs = rb_enc_get(rs); - enc_io = get_enc(ptr); - if (enc_rs != enc_io && - (rb_enc_str_coderange(rs) != ENC_CODERANGE_7BIT || - (RSTRING_LEN(rs) > 0 && !rb_enc_asciicompat(enc_io)))) { - if (rs == rb_rs) { - rs = rb_enc_str_new(0, 0, enc_io); - rb_str_buf_cat_ascii(rs, "\n"); - rs = rs; - } - else { - rb_raise(rb_eArgError, "encoding mismatch: %s IO with %s RS", - rb_enc_name(enc_io), - rb_enc_name(enc_rs)); - } - } + rb_encoding *enc_rs, *enc_io; + enc_rs = rb_enc_get(rs); + enc_io = get_enc(ptr); + if (enc_rs != enc_io && + (rb_enc_str_coderange(rs) != ENC_CODERANGE_7BIT || + (RSTRING_LEN(rs) > 0 && !rb_enc_asciicompat(enc_io)))) { + if (rs == rb_rs) { + rs = rb_enc_str_new(0, 0, enc_io); + rb_str_buf_cat_ascii(rs, "\n"); + rs = rs; + } + else { + rb_raise(rb_eArgError, "encoding mismatch: %s IO with %s RS", + rb_enc_name(enc_io), + rb_enc_name(enc_rs)); + } + } } arg->rs = rs; arg->limit = limit; @@ -1203,9 +1225,9 @@ prepare_getline_args(struct StringIO *ptr, struct getline_arg *arg, int argc, VA keywords[0] = rb_intern_const("chomp"); } rb_get_kwargs(opts, keywords, 0, 1, &vchomp); - if (respect_chomp) { + if (respect_chomp) { arg->chomp = (vchomp != Qundef) && RTEST(vchomp); - } + } } return arg; } @@ -1245,7 +1267,7 @@ strio_getline(struct getline_arg *arg, struct StringIO *ptr) str = strio_substr(ptr, ptr->pos, e - s - w, enc); } else if ((n = RSTRING_LEN(str)) == 0) { - const char *paragraph_end = NULL; + const char *paragraph_end = NULL; p = s; while (p[(p + 1 < e) && (*p == '\r') && 0] == '\n') { p += *p == '\r'; @@ -1255,18 +1277,18 @@ strio_getline(struct getline_arg *arg, struct StringIO *ptr) } s = p; while ((p = memchr(p, '\n', e - p)) && (p != e)) { - p++; - if (!((p < e && *p == '\n') || - (p + 1 < e && *p == '\r' && *(p+1) == '\n'))) { - continue; - } - paragraph_end = p - ((*(p-2) == '\r') ? 2 : 1); - while ((p < e && *p == '\n') || - (p + 1 < e && *p == '\r' && *(p+1) == '\n')) { - p += (*p == '\r') ? 2 : 1; - } - e = p; - break; + p++; + if (!((p < e && *p == '\n') || + (p + 1 < e && *p == '\r' && *(p+1) == '\n'))) { + continue; + } + paragraph_end = p - ((*(p-2) == '\r') ? 2 : 1); + while ((p < e && *p == '\n') || + (p + 1 < e && *p == '\r' && *(p+1) == '\n')) { + p += (*p == '\r') ? 2 : 1; + } + e = p; + break; } if (arg->chomp && paragraph_end) { w = e - paragraph_end; @@ -1590,7 +1612,7 @@ strio_read(int argc, VALUE *argv, VALUE self) } break; default: - rb_error_arity(argc, 0, 2); + rb_error_arity(argc, 0, 2); } if (NIL_P(str)) { rb_encoding *enc = binary ? rb_ascii8bit_encoding() : get_enc(ptr); @@ -1626,28 +1648,28 @@ strio_pread(int argc, VALUE *argv, VALUE self) long offset = NUM2LONG(rb_offset); if (len < 0) { - rb_raise(rb_eArgError, "negative string size (or size too big): %" PRIsVALUE, rb_len); + rb_raise(rb_eArgError, "negative string size (or size too big): %" PRIsVALUE, rb_len); } if (len == 0) { - if (NIL_P(rb_buf)) { - return rb_str_new("", 0); - } - return rb_buf; + if (NIL_P(rb_buf)) { + return rb_str_new("", 0); + } + return rb_buf; } if (offset < 0) { - rb_syserr_fail_str(EINVAL, rb_sprintf("pread: Invalid offset argument: %" PRIsVALUE, rb_offset)); + rb_syserr_fail_str(EINVAL, rb_sprintf("pread: Invalid offset argument: %" PRIsVALUE, rb_offset)); } struct StringIO *ptr = readable(self); if (offset >= RSTRING_LEN(ptr->string)) { - rb_eof_error(); + rb_eof_error(); } if (NIL_P(rb_buf)) { - return strio_substr(ptr, offset, len, rb_ascii8bit_encoding()); + return strio_substr(ptr, offset, len, rb_ascii8bit_encoding()); } long rest = RSTRING_LEN(ptr->string) - offset; diff --git a/gc.c b/gc.c index f4adc6f42a70e6..c4f2acfbafa734 100644 --- a/gc.c +++ b/gc.c @@ -2439,7 +2439,7 @@ rb_objspace_alloc(void) ccan_list_head_init(&SIZE_POOL_TOMB_HEAP(size_pool)->pages); } - rb_darray_make_without_gc(&objspace->weak_references, 0); + rb_darray_make(&objspace->weak_references, 0); // TODO: debug why on Windows Ruby crashes on boot when GC is on. #ifdef _WIN32 @@ -2512,7 +2512,7 @@ rb_objspace_free(rb_objspace_t *objspace) free_stack_chunks(&objspace->mark_stack); mark_stack_free_cache(&objspace->mark_stack); - rb_darray_free_without_gc(objspace->weak_references); + rb_darray_free(objspace->weak_references); if (!objspace->ractor->objspace_absorbed) { lock_ractor_set(); @@ -8836,7 +8836,11 @@ rb_gc_mark_weak(VALUE *ptr) rgengc_check_relation(objspace, obj); - rb_darray_append_without_gc(&objspace->weak_references, ptr); + DURING_GC_COULD_MALLOC_REGION_START(); + { + rb_darray_append(&objspace->weak_references, ptr); + } + DURING_GC_COULD_MALLOC_REGION_END(); objspace->profile.weak_references_count++; } @@ -10477,7 +10481,12 @@ gc_update_weak_references(rb_objspace_t *objspace) objspace->profile.retained_weak_references_count = retained_weak_references_count; rb_darray_clear(objspace->weak_references); - rb_darray_resize_capa_without_gc(&objspace->weak_references, retained_weak_references_count); + + DURING_GC_COULD_MALLOC_REGION_START(); + { + rb_darray_resize_capa(&objspace->weak_references, retained_weak_references_count); + } + DURING_GC_COULD_MALLOC_REGION_END(); } static int @@ -12350,18 +12359,20 @@ gc_set_candidate_object_i(void *vstart, void *vend, size_t stride, void *data) rb_objspace_t *objspace = &rb_objspace; VALUE v = (VALUE)vstart; for (; v != (VALUE)vend; v += stride) { - switch (BUILTIN_TYPE(v)) { - case T_NONE: - case T_ZOMBIE: - break; - case T_STRING: - // precompute the string coderange. This both save time for when it will be - // eventually needed, and avoid mutating heap pages after a potential fork. - rb_enc_str_coderange(v); - // fall through - default: - if (!RVALUE_OLD_P(v) && !RVALUE_WB_UNPROTECTED(v)) { - RVALUE_AGE_SET_CANDIDATE(objspace, v); + asan_unpoisoning_object(v) { + switch (BUILTIN_TYPE(v)) { + case T_NONE: + case T_ZOMBIE: + break; + case T_STRING: + // precompute the string coderange. This both save time for when it will be + // eventually needed, and avoid mutating heap pages after a potential fork. + rb_enc_str_coderange(v); + // fall through + default: + if (!RVALUE_OLD_P(v) && !RVALUE_WB_UNPROTECTED(v)) { + RVALUE_AGE_SET_CANDIDATE(objspace, v); + } } } } @@ -13225,9 +13236,7 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj) break; case T_SYMBOL: - if (DYNAMIC_SYM_P((VALUE)any)) { - UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr); - } + UPDATE_IF_MOVED(objspace, RSYMBOL(any)->fstr); break; case T_FLOAT: diff --git a/gems/bundled_gems b/gems/bundled_gems index 5dfd755706a379..38c97c77a01754 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -8,7 +8,7 @@ minitest 5.22.3 https://github.com/minitest/minitest 287b35d60c8e19c11ba090efc6eeb225325a8520 power_assert 2.0.3 https://github.com/ruby/power_assert 84e85124c5014a139af39161d484156cfe87a9ed -rake 13.1.0 https://github.com/ruby/rake +rake 13.2.0 https://github.com/ruby/rake test-unit 3.6.2 https://github.com/test-unit/test-unit rexml 3.2.6 https://github.com/ruby/rexml rss 0.3.0 https://github.com/ruby/rss @@ -20,7 +20,7 @@ matrix 0.4.2 https://github.com/ruby/matrix prime 0.1.2 https://github.com/ruby/prime rbs 3.4.4 https://github.com/ruby/rbs ba7872795d5de04adb8ff500c0e6afdc81a041dd typeprof 0.21.11 https://github.com/ruby/typeprof b19a6416da3a05d57fadd6ffdadb382b6d236ca5 -debug 1.9.1 https://github.com/ruby/debug 2d602636d99114d55a32fedd652c9c704446a749 +debug 1.9.2 https://github.com/ruby/debug racc 1.7.3 https://github.com/ruby/racc mutex_m 0.2.0 https://github.com/ruby/mutex_m getoptlong 0.2.1 https://github.com/ruby/getoptlong diff --git a/internal/parse.h b/internal/parse.h index fc78836e5c48e4..f4f2f0c0f521ca 100644 --- a/internal/parse.h +++ b/internal/parse.h @@ -18,8 +18,6 @@ struct rb_iseq_struct; /* in vm_core.h */ -#define STRTERM_HEREDOC IMEMO_FL_USER0 - /* structs for managing terminator of string literal and heredocment */ typedef struct rb_strterm_literal_struct { long nest; @@ -40,7 +38,7 @@ typedef struct rb_strterm_heredoc_struct { #define HERETERM_LENGTH_MAX UINT_MAX typedef struct rb_strterm_struct { - VALUE flags; + bool heredoc; union { rb_strterm_literal_t literal; rb_strterm_heredoc_t heredoc; @@ -67,6 +65,7 @@ int rb_ruby_parser_end_seen_p(rb_parser_t *p); int rb_ruby_parser_set_yydebug(rb_parser_t *p, int flag); rb_parser_string_t *rb_str_to_parser_string(rb_parser_t *p, VALUE str); +void rb_parser_check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc); void rb_parser_warn_duplicate_keys(struct parser_params *p, NODE *hash); int rb_parser_dvar_defined_ref(struct parser_params*, ID, ID**); ID rb_parser_internal_id(struct parser_params*); diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h index 1c59851f0d7eda..6a2dcab5ca8690 100644 --- a/internal/ruby_parser.h +++ b/internal/ruby_parser.h @@ -5,7 +5,6 @@ #include "internal/bignum.h" #include "internal/compilers.h" #include "internal/complex.h" -#include "internal/imemo.h" #include "internal/rational.h" #include "rubyparser.h" #include "vm.h" diff --git a/iseq.c b/iseq.c index a6210242e2b229..f7b1df806d6e83 100644 --- a/iseq.c +++ b/iseq.c @@ -1157,7 +1157,7 @@ iseq_load(VALUE data, const rb_iseq_t *parent, VALUE opt) tmp_loc.end_pos.column = NUM2INT(rb_ary_entry(code_location, 3)); } - if (RTEST(rb_hash_aref(misc, ID2SYM(rb_intern("prism"))))) { + if (SYM2ID(rb_hash_aref(misc, ID2SYM(rb_intern("parser")))) == rb_intern("prism")) { ISEQ_BODY(iseq)->prism = true; } @@ -1434,6 +1434,44 @@ rb_iseqw_new(const rb_iseq_t *iseq) return iseqw_new(iseq); } +/** + * Accept the options given to InstructionSequence.compile and + * InstructionSequence.compile_prism and share the logic for creating the + * instruction sequence. + */ +static VALUE +iseqw_s_compile_parser(int argc, VALUE *argv, VALUE self, bool prism) +{ + VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; + int i; + + i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); + if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); + switch (i) { + case 5: opt = argv[--i]; + case 4: line = argv[--i]; + case 3: path = argv[--i]; + case 2: file = argv[--i]; + } + + if (NIL_P(file)) file = rb_fstring_lit(""); + if (NIL_P(path)) path = file; + if (NIL_P(line)) line = INT2FIX(1); + + Check_Type(path, T_STRING); + Check_Type(file, T_STRING); + + rb_iseq_t *iseq; + if (prism) { + iseq = pm_iseq_compile_with_option(src, file, path, line, opt); + } + else { + iseq = rb_iseq_compile_with_option(src, file, path, line, opt); + } + + return iseqw_new(iseq); +} + /* * call-seq: * InstructionSequence.compile(source[, file[, path[, line[, options]]]]) -> iseq @@ -1474,26 +1512,7 @@ rb_iseqw_new(const rb_iseq_t *iseq) static VALUE iseqw_s_compile(int argc, VALUE *argv, VALUE self) { - VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; - int i; - - i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); - if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); - switch (i) { - case 5: opt = argv[--i]; - case 4: line = argv[--i]; - case 3: path = argv[--i]; - case 2: file = argv[--i]; - } - - if (NIL_P(file)) file = rb_fstring_lit(""); - if (NIL_P(path)) path = file; - if (NIL_P(line)) line = INT2FIX(1); - - Check_Type(path, T_STRING); - Check_Type(file, T_STRING); - - return iseqw_new(rb_iseq_compile_with_option(src, file, path, line, opt)); + return iseqw_s_compile_parser(argc, argv, self, *rb_ruby_prism_ptr()); } /* @@ -1535,26 +1554,7 @@ iseqw_s_compile(int argc, VALUE *argv, VALUE self) static VALUE iseqw_s_compile_prism(int argc, VALUE *argv, VALUE self) { - VALUE src, file = Qnil, path = Qnil, line = Qnil, opt = Qnil; - int i; - - i = rb_scan_args(argc, argv, "1*:", &src, NULL, &opt); - if (i > 4+NIL_P(opt)) rb_error_arity(argc, 1, 5); - switch (i) { - case 5: opt = argv[--i]; - case 4: line = argv[--i]; - case 3: path = argv[--i]; - case 2: file = argv[--i]; - } - - if (NIL_P(file)) file = rb_fstring_lit(""); - if (NIL_P(path)) path = file; - if (NIL_P(line)) line = INT2FIX(1); - - Check_Type(path, T_STRING); - Check_Type(file, T_STRING); - - return iseqw_new(pm_iseq_compile_with_option(src, file, path, line, opt)); + return iseqw_s_compile_parser(argc, argv, self, true); } /* diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index a8ea9fe211c2ee..ed4138f0d9b48b 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -26,6 +26,8 @@ module Gem::BUNDLED_GEMS "resolv-replace" => "3.4.0", "rinda" => "3.4.0", "syslog" => "3.4.0", + "ostruct" => "3.5.0", + "pstore" => "3.5.0", }.freeze EXACT = { @@ -41,6 +43,8 @@ module Gem::BUNDLED_GEMS "resolv-replace" => true, "rinda" => true, "syslog" => true, + "ostruct" => true, + "pstore" => true, }.freeze PREFIXED = { diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb index c7eacd193049a3..ecc65b49560292 100644 --- a/lib/bundler/gem_version_promoter.rb +++ b/lib/bundler/gem_version_promoter.rb @@ -45,17 +45,37 @@ def level=(value) # Given a Resolver::Package and an Array of Specifications of available # versions for a gem, this method will return the Array of Specifications - # sorted (and possibly truncated if strict is true) in an order to give - # preference to the current level (:major, :minor or :patch) when resolution - # is deciding what versions best resolve all dependencies in the bundle. + # sorted in an order to give preference to the current level (:major, :minor + # or :patch) when resolution is deciding what versions best resolve all + # dependencies in the bundle. # @param package [Resolver::Package] The package being resolved. # @param specs [Specification] An array of Specifications for the package. - # @return [Specification] A new instance of the Specification Array sorted and - # possibly filtered. + # @return [Specification] A new instance of the Specification Array sorted. def sort_versions(package, specs) - specs = filter_dep_specs(specs, package) if strict + locked_version = package.locked_version - sort_dep_specs(specs, package) + result = specs.sort do |a, b| + unless package.prerelease_specified? || pre? + a_pre = a.prerelease? + b_pre = b.prerelease? + + next 1 if a_pre && !b_pre + next -1 if b_pre && !a_pre + end + + if major? || locked_version.nil? + b <=> a + elsif either_version_older_than_locked?(a, b, locked_version) + b <=> a + elsif segments_do_not_match?(a, b, :major) + a <=> b + elsif !minor? && segments_do_not_match?(a, b, :minor) + a <=> b + else + b <=> a + end + end + post_sort(result, package.unlock?, locked_version) end # @return [bool] Convenience method for testing value of level variable. @@ -73,9 +93,18 @@ def pre? pre == true end - private + # Given a Resolver::Package and an Array of Specifications of available + # versions for a gem, this method will truncate the Array if strict + # is true. That means filtering out downgrades from the version currently + # locked, and filtering out upgrades that go past the selected level (major, + # minor, or patch). + # @param package [Resolver::Package] The package being resolved. + # @param specs [Specification] An array of Specifications for the package. + # @return [Specification] A new instance of the Specification Array + # truncated. + def filter_versions(package, specs) + return specs unless strict - def filter_dep_specs(specs, package) locked_version = package.locked_version return specs if locked_version.nil? || major? @@ -89,32 +118,7 @@ def filter_dep_specs(specs, package) end end - def sort_dep_specs(specs, package) - locked_version = package.locked_version - - result = specs.sort do |a, b| - unless package.prerelease_specified? || pre? - a_pre = a.prerelease? - b_pre = b.prerelease? - - next -1 if a_pre && !b_pre - next 1 if b_pre && !a_pre - end - - if major? || locked_version.nil? - a <=> b - elsif either_version_older_than_locked?(a, b, locked_version) - a <=> b - elsif segments_do_not_match?(a, b, :major) - b <=> a - elsif !minor? && segments_do_not_match?(a, b, :minor) - b <=> a - else - a <=> b - end - end - post_sort(result, package.unlock?, locked_version) - end + private def either_version_older_than_locked?(a, b, locked_version) a.version < locked_version || b.version < locked_version @@ -133,13 +137,13 @@ def post_sort(result, unlock, locked_version) if unlock || locked_version.nil? result else - move_version_to_end(result, locked_version) + move_version_to_beginning(result, locked_version) end end - def move_version_to_end(result, version) + def move_version_to_beginning(result, version) move, keep = result.partition {|s| s.version.to_s == version.to_s } - keep.concat(move) + move.concat(keep) end end end diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index f60069f421ab08..1a6711ea6fcacd 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -50,26 +50,26 @@ def setup_solver specs[name] = matches.sort_by {|s| [s.version, s.platform.to_s] } end + @all_versions = Hash.new do |candidates, package| + candidates[package] = all_versions_for(package) + end + @sorted_versions = Hash.new do |candidates, package| - candidates[package] = if package.root? - [root_version] - else - all_versions_for(package).sort - end + candidates[package] = filtered_versions_for(package).sort end + @sorted_versions[root] = [root_version] + root_dependencies = prepare_dependencies(@requirements, @packages) @cached_dependencies = Hash.new do |dependencies, package| - dependencies[package] = if package.root? - { root_version => root_dependencies } - else - Hash.new do |versions, version| - versions[version] = to_dependency_hash(version.dependencies.reject {|d| d.name == package.name }, @packages) - end + dependencies[package] = Hash.new do |versions, version| + versions[version] = to_dependency_hash(version.dependencies.reject {|d| d.name == package.name }, @packages) end end + @cached_dependencies[root] = { root_version => root_dependencies } + logger = Bundler::UI::Shell.new logger.level = debug? ? "debug" : "warn" @@ -156,9 +156,15 @@ def parse_dependency(package, dependency) end def versions_for(package, range=VersionRange.any) - versions = range.select_versions(@sorted_versions[package]) + versions = select_sorted_versions(package, range) - sort_versions(package, versions) + # Conditional avoids (among other things) calling + # sort_versions_by_preferred with the root package + if versions.size > 1 + sort_versions_by_preferred(package, versions) + else + versions + end end def no_versions_incompatibility_for(package, unsatisfied_term) @@ -247,7 +253,7 @@ def all_versions_for(package) locked_requirement = base_requirements[name] results = filter_matching_specs(results, locked_requirement) if locked_requirement - versions = results.group_by(&:version).reduce([]) do |groups, (version, specs)| + results.group_by(&:version).reduce([]) do |groups, (version, specs)| platform_specs = package.platforms.map {|platform| select_best_platform_match(specs, platform) } # If package is a top-level dependency, @@ -274,8 +280,6 @@ def all_versions_for(package) groups end - - sort_versions(package, versions) end def source_for(name) @@ -334,6 +338,21 @@ def raise_not_found!(package) private + def filtered_versions_for(package) + @gem_version_promoter.filter_versions(package, @all_versions[package]) + end + + def raise_all_versions_filtered_out!(package) + level = @gem_version_promoter.level + name = package.name + locked_version = package.locked_version + requirement = package.dependency + + raise GemNotFound, + "#{name} is locked to #{locked_version}, while Gemfile is requesting #{requirement}. " \ + "--strict --#{level} was specified, but there are no #{level} level upgrades from #{locked_version} satisfying #{requirement}, so version solving has failed" + end + def filter_matching_specs(specs, requirements) Array(requirements).flat_map do |requirement| specs.select {| spec| requirement_satisfied_by?(requirement, spec) } @@ -357,12 +376,8 @@ def requirement_satisfied_by?(requirement, spec) requirement.satisfied_by?(spec.version) || spec.source.is_a?(Source::Gemspec) end - def sort_versions(package, versions) - if versions.size > 1 - @gem_version_promoter.sort_versions(package, versions).reverse - else - versions - end + def sort_versions_by_preferred(package, versions) + @gem_version_promoter.sort_versions(package, versions) end def repository_for(package) @@ -379,12 +394,19 @@ def prepare_dependencies(requirements, packages) next [dep_package, dep_constraint] if name == "bundler" - versions = versions_for(dep_package, dep_constraint.range) + dep_range = dep_constraint.range + versions = select_sorted_versions(dep_package, dep_range) if versions.empty? && dep_package.ignores_prereleases? + @all_versions.delete(dep_package) @sorted_versions.delete(dep_package) dep_package.consider_prereleases! - versions = versions_for(dep_package, dep_constraint.range) + versions = select_sorted_versions(dep_package, dep_range) end + + if versions.empty? && select_all_versions(dep_package, dep_range).any? + raise_all_versions_filtered_out!(dep_package) + end + next [dep_package, dep_constraint] unless versions.empty? next unless dep_package.current_platform? @@ -393,6 +415,14 @@ def prepare_dependencies(requirements, packages) end.compact.to_h end + def select_sorted_versions(package, range) + range.select_versions(@sorted_versions[package]) + end + + def select_all_versions(package, range) + range.select_versions(@all_versions[package]) + end + def other_specs_matching_message(specs, requirement) message = String.new("The source contains the following gems matching '#{requirement}':\n") message << specs.map {|s| " * #{s.full_name}" }.join("\n") diff --git a/lib/bundler/resolver/candidate.rb b/lib/bundler/resolver/candidate.rb index e695ef08ee0bde..9e8b9133358c69 100644 --- a/lib/bundler/resolver/candidate.rb +++ b/lib/bundler/resolver/candidate.rb @@ -15,7 +15,7 @@ class Resolver # considered separately. # # Some candidates may also keep some information explicitly about the - # package the refer to. These candidates are referred to as "canonical" and + # package they refer to. These candidates are referred to as "canonical" and # are used when materializing resolution results back into RubyGems # specifications that can be installed, written to lock files, and so on. # diff --git a/lib/mkmf.rb b/lib/mkmf.rb index 92af95d4231043..73459ffeb95176 100644 --- a/lib/mkmf.rb +++ b/lib/mkmf.rb @@ -44,6 +44,23 @@ def quote # correctly compile and link the C extension to Ruby and a third-party # library. module MakeMakefile + + target_rbconfig = nil + ARGV.delete_if do |arg| + opt = arg.delete_prefix("--target-rbconfig=") + unless opt == arg + target_rbconfig = opt + end + end + if target_rbconfig + # Load the RbConfig for the target platform into this module. + # Cross-compiling needs the same version of Ruby. + Kernel.load target_rbconfig, self + else + # The RbConfig for the target platform where the built extension runs. + RbConfig = ::RbConfig + end + #### defer until this module become global-state free. # def self.extended(obj) # obj.init_mkmf diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index 1ca99db6810034..0a064a5c941850 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -382,9 +382,6 @@ def dump_options(options) template << "l" values << options.fetch(:line, 1) - template << "L" - values << options.fetch(:offset, 0) - template << "L" if (encoding = options[:encoding]) name = encoding.name diff --git a/lib/random/formatter.rb b/lib/random/formatter.rb index 0548e86ccaafc6..037f9d8748f27b 100644 --- a/lib/random/formatter.rb +++ b/lib/random/formatter.rb @@ -165,7 +165,7 @@ def urlsafe_base64(n=nil, padding=false) # # The result contains 122 random bits (15.25 random bytes). # - # See RFC4122[https://datatracker.ietf.org/doc/html/rfc4122] for details of UUID. + # See RFC4122[https://www.rfc-editor.org/rfc/rfc4122] for details of UUID. # def uuid ary = random_bytes(16) diff --git a/lib/reline.rb b/lib/reline.rb index f3fd28b627736a..b43584fc9bccbf 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -219,26 +219,16 @@ def get_screen_size Reline::DEFAULT_DIALOG_PROC_AUTOCOMPLETE = ->() { # autocomplete - return nil unless config.autocompletion - if just_cursor_moving and completion_journey_data.nil? - # Auto complete starts only when edited - return nil - end - pre, target, post = retrieve_completion_block(true) - if target.nil? or target.empty? or (completion_journey_data&.pointer == -1 and target.size <= 3) - return nil - end - if completion_journey_data and completion_journey_data.list - result = completion_journey_data.list.dup - result.shift - pointer = completion_journey_data.pointer - 1 - else - result = call_completion_proc_with_checking_args(pre, target, post) - pointer = nil - end - if result and result.size == 1 and result[0] == target and pointer != 0 - result = nil - end + return unless config.autocompletion + + journey_data = completion_journey_data + return unless journey_data + + target = journey_data.list[journey_data.pointer] + result = journey_data.list.drop(1) + pointer = journey_data.pointer - 1 + return if target.empty? || (result == [target] && pointer < 0) + target_width = Reline::Unicode.calculate_width(target) x = cursor_pos.x - target_width if x < 0 diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index d202ba02d27a76..d5c158ac746e54 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -41,15 +41,36 @@ module CompletionState NORMAL = :normal COMPLETION = :completion MENU = :menu - JOURNEY = :journey MENU_WITH_PERFECT_MATCH = :menu_with_perfect_match PERFECT_MATCH = :perfect_match end RenderedScreen = Struct.new(:base_y, :lines, :cursor_y, keyword_init: true) - CompletionJourneyData = Struct.new(:preposing, :postposing, :list, :pointer) - MenuInfo = Struct.new(:target, :list) + CompletionJourneyState = Struct.new(:line_index, :pre, :target, :post, :list, :pointer) + + class MenuInfo + attr_reader :list + + def initialize(list) + @list = list + end + + def lines(screen_width) + return [] if @list.empty? + + list = @list.sort + sizes = list.map { |item| Reline::Unicode.calculate_width(item) } + item_width = sizes.max + 2 + num_cols = [screen_width / item_width, 1].max + num_rows = list.size.fdiv(num_cols).ceil + list_with_padding = list.zip(sizes).map { |item, size| item + ' ' * (item_width - size) } + aligned = (list_with_padding + [nil] * (num_rows * num_cols - list_with_padding.size)).each_slice(num_rows).to_a.transpose + aligned.map do |row| + row.join.rstrip + end + end + end MINIMUM_SCROLLBAR_HEIGHT = 1 @@ -199,7 +220,7 @@ def reset_variables(prompt = '', encoding:) @waiting_proc = nil @waiting_operator_proc = nil @waiting_operator_vi_arg = nil - @completion_journey_data = nil + @completion_journey_state = nil @completion_state = CompletionState::NORMAL @perfect_matched = nil @menu_info = nil @@ -458,7 +479,7 @@ def render_differential [[0, Reline::Unicode.calculate_width(l, true), l]] end if @menu_info - @menu_info.list.sort!.each do |item| + @menu_info.lines(screen_width).each do |item| new_lines << [[0, Reline::Unicode.calculate_width(item), item]] end @menu_info = nil # TODO: do not change state here @@ -537,6 +558,8 @@ def rerender end class DialogProcScope + CompletionJourneyData = Struct.new(:preposing, :postposing, :list, :pointer) + def initialize(line_editor, config, proc_to_exec, context) @line_editor = line_editor @config = config @@ -600,7 +623,7 @@ def preferred_dialog_height end def completion_journey_data - @line_editor.instance_variable_get(:@completion_journey_data) + @line_editor.dialog_proc_scope_completion_journey_data end def config @@ -779,8 +802,8 @@ def editing_mode @config.editing_mode end - private def menu(target, list) - @menu_info = MenuInfo.new(target, list) + private def menu(_target, list) + @menu_info = MenuInfo.new(list) end private def complete_internal_proc(list, is_menu) @@ -829,9 +852,9 @@ def editing_mode [target, preposing, completed, postposing] end - private def complete(list, just_show_list = false) + private def complete(list, just_show_list) case @completion_state - when CompletionState::NORMAL, CompletionState::JOURNEY + when CompletionState::NORMAL @completion_state = CompletionState::COMPLETION when CompletionState::PERFECT_MATCH @dig_perfect_match_proc&.(@perfect_matched) @@ -871,46 +894,44 @@ def editing_mode end end - private def move_completed_list(list, direction) - case @completion_state - when CompletionState::NORMAL, CompletionState::COMPLETION, - CompletionState::MENU, CompletionState::MENU_WITH_PERFECT_MATCH - @completion_state = CompletionState::JOURNEY - result = retrieve_completion_block - return if result.nil? - preposing, target, postposing = result - @completion_journey_data = CompletionJourneyData.new( - preposing, postposing, - [target] + list.select{ |item| item.start_with?(target) }, 0) - if @completion_journey_data.list.size == 1 - @completion_journey_data.pointer = 0 - else - case direction - when :up - @completion_journey_data.pointer = @completion_journey_data.list.size - 1 - when :down - @completion_journey_data.pointer = 1 - end - end - @completion_state = CompletionState::JOURNEY - else - case direction - when :up - @completion_journey_data.pointer -= 1 - if @completion_journey_data.pointer < 0 - @completion_journey_data.pointer = @completion_journey_data.list.size - 1 - end - when :down - @completion_journey_data.pointer += 1 - if @completion_journey_data.pointer >= @completion_journey_data.list.size - @completion_journey_data.pointer = 0 - end - end + def dialog_proc_scope_completion_journey_data + return nil unless @completion_journey_state + line_index = @completion_journey_state.line_index + pre_lines = @buffer_of_lines[0...line_index].map { |line| line + "\n" } + post_lines = @buffer_of_lines[(line_index + 1)..-1].map { |line| line + "\n" } + DialogProcScope::CompletionJourneyData.new( + pre_lines.join + @completion_journey_state.pre, + @completion_journey_state.post + post_lines.join, + @completion_journey_state.list, + @completion_journey_state.pointer + ) + end + + private def move_completed_list(direction) + @completion_journey_state ||= retrieve_completion_journey_state + return false unless @completion_journey_state + + if (delta = { up: -1, down: +1 }[direction]) + @completion_journey_state.pointer = (@completion_journey_state.pointer + delta) % @completion_journey_state.list.size end - completed = @completion_journey_data.list[@completion_journey_data.pointer] - line_to_pointer = (@completion_journey_data.preposing + completed).split("\n")[@line_index] || String.new(encoding: @encoding) - new_line = line_to_pointer + (@completion_journey_data.postposing.split("\n").first || '') - set_current_line(new_line, line_to_pointer.bytesize) + completed = @completion_journey_state.list[@completion_journey_state.pointer] + set_current_line(@completion_journey_state.pre + completed + @completion_journey_state.post, @completion_journey_state.pre.bytesize + completed.bytesize) + true + end + + private def retrieve_completion_journey_state + preposing, target, postposing = retrieve_completion_block + list = call_completion_proc + return unless list.is_a?(Array) + + candidates = list.select{ |item| item.start_with?(target) } + return if candidates.empty? + + pre = preposing.split("\n", -1).last || '' + post = postposing.split("\n", -1).first || '' + CompletionJourneyState.new( + @line_index, pre, target, post, [target] + candidates, 0 + ) end private def run_for_operators(key, method_symbol, &block) @@ -1099,50 +1120,56 @@ def input_key(key) @first_char = false completion_occurs = false if @config.editing_mode_is?(:emacs, :vi_insert) and key.char == "\C-i".ord - unless @config.disable_completion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - if @config.autocompletion - move_completed_list(result, :down) - else - complete(result) + if !@config.disable_completion + process_insert(force: true) + if @config.autocompletion + @completion_state = CompletionState::NORMAL + completion_occurs = move_completed_list(:down) + else + @completion_journey_state = nil + result = call_completion_proc + if result.is_a?(Array) + completion_occurs = true + complete(result, false) end end end elsif @config.editing_mode_is?(:emacs, :vi_insert) and key.char == :completion_journey_up if not @config.disable_completion and @config.autocompletion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - move_completed_list(result, :up) - end + process_insert(force: true) + @completion_state = CompletionState::NORMAL + completion_occurs = move_completed_list(:up) end - elsif not @config.disable_completion and @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) - unless @config.disable_completion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - move_completed_list(result, "\C-p".ord == key.char ? :up : :down) - end + elsif @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) + # In vi mode, move completed list even if autocompletion is off + if not @config.disable_completion + process_insert(force: true) + @completion_state = CompletionState::NORMAL + completion_occurs = move_completed_list("\C-p".ord == key.char ? :up : :down) end elsif Symbol === key.char and respond_to?(key.char, true) process_key(key.char, key.char) else normal_char(key) end + unless completion_occurs @completion_state = CompletionState::NORMAL - @completion_journey_data = nil + @completion_journey_state = nil end + if @in_pasting clear_dialogs - else - return old_lines != @buffer_of_lines + return + end + + modified = old_lines != @buffer_of_lines + if !completion_occurs && modified && !@config.disable_completion && @config.autocompletion + # Auto complete starts only when edited + process_insert(force: true) + @completion_journey_state = retrieve_completion_journey_state end + modified end def scroll_into_view @@ -2020,7 +2047,7 @@ def finish private def em_delete_or_list(key) if current_line.empty? or @byte_pointer < current_line.bytesize em_delete(key) - else # show completed list + elsif !@config.autocompletion # show completed list result = call_completion_proc if result.is_a?(Array) complete(result, true) diff --git a/lib/rubygems/command_manager.rb b/lib/rubygems/command_manager.rb index c2e4f4ce49ebc4..8e578dc1966172 100644 --- a/lib/rubygems/command_manager.rb +++ b/lib/rubygems/command_manager.rb @@ -60,6 +60,7 @@ class Gem::CommandManager :push, :query, :rdoc, + :rebuild, :search, :server, :signin, diff --git a/lib/rubygems/commands/build_command.rb b/lib/rubygems/commands/build_command.rb index 0ebdec565b5606..2ec83241418d13 100644 --- a/lib/rubygems/commands/build_command.rb +++ b/lib/rubygems/commands/build_command.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true require_relative "../command" +require_relative "../gemspec_helpers" require_relative "../package" require_relative "../version_option" class Gem::Commands::BuildCommand < Gem::Command include Gem::VersionOption + include Gem::GemspecHelpers def initialize super "build", "Build a gem from a gemspec" @@ -75,17 +77,6 @@ def execute private - def find_gemspec(glob = "*.gemspec") - gemspecs = Dir.glob(glob).sort - - if gemspecs.size > 1 - alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" - terminate_interaction(1) - end - - gemspecs.first - end - def build_gem gemspec = resolve_gem_name diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb new file mode 100644 index 00000000000000..77a474ef1ddf32 --- /dev/null +++ b/lib/rubygems/commands/rebuild_command.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +require "date" +require "digest" +require "fileutils" +require "tmpdir" +require_relative "../gemspec_helpers" +require_relative "../package" + +class Gem::Commands::RebuildCommand < Gem::Command + include Gem::GemspecHelpers + + def initialize + super "rebuild", "Attempt to reproduce a build of a gem." + + add_option "--diff", "If the files don't match, compare them using diffoscope." do |_value, options| + options[:diff] = true + end + + add_option "--force", "Skip validation of the spec." do |_value, options| + options[:force] = true + end + + add_option "--strict", "Consider warnings as errors when validating the spec." do |_value, options| + options[:strict] = true + end + + add_option "--source GEM_SOURCE", "Specify the source to download the gem from." do |value, options| + options[:source] = value + end + + add_option "--original GEM_FILE", "Specify a local file to compare against (instead of downloading it)." do |value, options| + options[:original_gem_file] = value + end + + add_option "--gemspec GEMSPEC_FILE", "Specify the name of the gemspec file." do |value, options| + options[:gemspec_file] = value + end + + add_option "-C PATH", "Run as if gem build was started in instead of the current working directory." do |value, options| + options[:build_path] = value + end + end + + def arguments # :nodoc: + "GEM_NAME gem name on gem server\n" \ + "GEM_VERSION gem version you are attempting to rebuild" + end + + def description # :nodoc: + <<-EOF +The rebuild command allows you to (attempt to) reproduce a build of a gem +from a ruby gemspec. + +This command assumes the gemspec can be built with the `gem build` command. +If you use any of `gem build`, `rake build`, or`rake release` in the +build/release process for a gem, it is a potential candidate. + +You will need to match the RubyGems version used, since this is included in +the Gem metadata. + +If the gem includes lockfiles (e.g. Gemfile.lock) and similar, it will +require more effort to reproduce a build. For example, it might require +more precisely matched versions of Ruby and/or Bundler to be used. + EOF + end + + def usage # :nodoc: + "#{program_name} GEM_NAME GEM_VERSION" + end + + def execute + gem_name, gem_version = get_gem_name_and_version + + old_dir, new_dir = prep_dirs + + gem_filename = "#{gem_name}-#{gem_version}.gem" + old_file = File.join(old_dir, gem_filename) + new_file = File.join(new_dir, gem_filename) + + if options[:original_gem_file] + FileUtils.copy_file(options[:original_gem_file], old_file) + else + download_gem(gem_name, gem_version, old_file) + end + + rg_version = rubygems_version(old_file) + unless rg_version == Gem::VERSION + alert_error <<-EOF +You need to use the same RubyGems version #{gem_name} v#{gem_version} was built with. + +#{gem_name} v#{gem_version} was built using RubyGems v#{rg_version}. +Gem files include the version of RubyGems used to build them. +This means in order to reproduce #{gem_filename}, you must also use RubyGems v#{rg_version}. + +You're using RubyGems v#{Gem::VERSION}. + +Please install RubyGems v#{rg_version} and try again. + EOF + terminate_interaction 1 + end + + source_date_epoch = get_timestamp(old_file).to_s + + if build_path = options[:build_path] + Dir.chdir(build_path) { build_gem(gem_name, source_date_epoch, new_file) } + else + build_gem(gem_name, source_date_epoch, new_file) + end + + compare(source_date_epoch, old_file, new_file) + end + + private + + def sha256(file) + Digest::SHA256.hexdigest(Gem.read_binary(file)) + end + + def get_timestamp(file) + mtime = nil + File.open(file, Gem.binary_mode) do |f| + Gem::Package::TarReader.new(f) do |tar| + mtime = tar.seek("metadata.gz") {|tf| tf.header.mtime } + end + end + + mtime + end + + def compare(source_date_epoch, old_file, new_file) + date = Time.at(source_date_epoch.to_i).strftime("%F %T %Z") + + old_hash = sha256(old_file) + new_hash = sha256(new_file) + + say + say "Built at: #{date} (#{source_date_epoch})" + say "Original build saved to: #{old_file}" + say "Reproduced build saved to: #{new_file}" + say "Working directory: #{options[:build_path] || Dir.pwd}" + say + say "Hash comparison:" + say " #{old_hash}\t#{old_file}" + say " #{new_hash}\t#{new_file}" + say + + if old_hash == new_hash + say "SUCCESS - original and rebuild hashes matched" + else + say "FAILURE - original and rebuild hashes did not match" + say + + if options[:diff] + if system("diffoscope", old_file, new_file).nil? + alert_error "error: could not find `diffoscope` executable" + end + else + say "Pass --diff for more details (requires diffoscope to be installed)." + end + + terminate_interaction 1 + end + end + + def prep_dirs + rebuild_dir = Dir.mktmpdir("gem_rebuild") + old_dir = File.join(rebuild_dir, "old") + new_dir = File.join(rebuild_dir, "new") + + FileUtils.mkdir_p(old_dir) + FileUtils.mkdir_p(new_dir) + + [old_dir, new_dir] + end + + def get_gem_name_and_version + args = options[:args] || [] + if args.length == 2 + gem_name, gem_version = args + elsif args.length > 2 + raise Gem::CommandLineError, "Too many arguments" + else + raise Gem::CommandLineError, "Expected GEM_NAME and GEM_VERSION arguments (gem rebuild GEM_NAME GEM_VERSION)" + end + + [gem_name, gem_version] + end + + def build_gem(gem_name, source_date_epoch, output_file) + gemspec = options[:gemspec_file] || find_gemspec("#{gem_name}.gemspec") + + if gemspec + build_package(gemspec, source_date_epoch, output_file) + else + alert_error error_message(gem_name) + terminate_interaction(1) + end + end + + def build_package(gemspec, source_date_epoch, output_file) + with_source_date_epoch(source_date_epoch) do + spec = Gem::Specification.load(gemspec) + if spec + Gem::Package.build( + spec, + options[:force], + options[:strict], + output_file + ) + else + alert_error "Error loading gemspec. Aborting." + terminate_interaction 1 + end + end + end + + def with_source_date_epoch(source_date_epoch) + old_sde = ENV["SOURCE_DATE_EPOCH"] + ENV["SOURCE_DATE_EPOCH"] = source_date_epoch.to_s + + yield + ensure + ENV["SOURCE_DATE_EPOCH"] = old_sde + end + + def error_message(gem_name) + if gem_name + "Couldn't find a gemspec file matching '#{gem_name}' in #{Dir.pwd}" + else + "Couldn't find a gemspec file in #{Dir.pwd}" + end + end + + def download_gem(gem_name, gem_version, old_file) + # This code was based loosely off the `gem fetch` command. + version = "= #{gem_version}" + dep = Gem::Dependency.new gem_name, version + + specs_and_sources, errors = + Gem::SpecFetcher.fetcher.spec_for_dependency dep + + # There should never be more than one item in specs_and_sources, + # since we search for an exact version. + spec, source = specs_and_sources[0] + + if spec.nil? + show_lookup_failure gem_name, version, errors, options[:domain] + terminate_interaction 1 + end + + download_path = source.download spec + + FileUtils.move(download_path, old_file) + + say "Downloaded #{gem_name} version #{gem_version} as #{old_file}." + end + + def rubygems_version(gem_file) + Gem::Package.new(gem_file).spec.rubygems_version + end +end diff --git a/lib/rubygems/gemspec_helpers.rb b/lib/rubygems/gemspec_helpers.rb new file mode 100644 index 00000000000000..2b20fcafa12120 --- /dev/null +++ b/lib/rubygems/gemspec_helpers.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative "../rubygems" + +## +# Mixin methods for commands that work with gemspecs. + +module Gem::GemspecHelpers + def find_gemspec(glob = "*.gemspec") + gemspecs = Dir.glob(glob).sort + + if gemspecs.size > 1 + alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" + terminate_interaction(1) + end + + gemspecs.first + end +end diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 72a179da376d87..1d5d7642376a55 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -59,7 +59,7 @@ class FormatError < Error def initialize(message, source = nil) if source - @path = source.path + @path = source.is_a?(String) ? source : source.path message += " in #{path}" if path end diff --git a/lib/uri.rb b/lib/uri.rb index cd8083b65aa071..dfdb052a79057c 100644 --- a/lib/uri.rb +++ b/lib/uri.rb @@ -1,6 +1,6 @@ # frozen_string_literal: false # URI is a module providing classes to handle Uniform Resource Identifiers -# (RFC2396[https://datatracker.ietf.org/doc/html/rfc2396]). +# (RFC2396[https://www.rfc-editor.org/rfc/rfc2396]). # # == Features # @@ -47,14 +47,14 @@ # A good place to view an RFC spec is http://www.ietf.org/rfc.html. # # Here is a list of all related RFC's: -# - RFC822[https://datatracker.ietf.org/doc/html/rfc822] -# - RFC1738[https://datatracker.ietf.org/doc/html/rfc1738] -# - RFC2255[https://datatracker.ietf.org/doc/html/rfc2255] -# - RFC2368[https://datatracker.ietf.org/doc/html/rfc2368] -# - RFC2373[https://datatracker.ietf.org/doc/html/rfc2373] -# - RFC2396[https://datatracker.ietf.org/doc/html/rfc2396] -# - RFC2732[https://datatracker.ietf.org/doc/html/rfc2732] -# - RFC3986[https://datatracker.ietf.org/doc/html/rfc3986] +# - RFC822[https://www.rfc-editor.org/rfc/rfc822] +# - RFC1738[https://www.rfc-editor.org/rfc/rfc1738] +# - RFC2255[https://www.rfc-editor.org/rfc/rfc2255] +# - RFC2368[https://www.rfc-editor.org/rfc/rfc2368] +# - RFC2373[https://www.rfc-editor.org/rfc/rfc2373] +# - RFC2396[https://www.rfc-editor.org/rfc/rfc2396] +# - RFC2732[https://www.rfc-editor.org/rfc/rfc2732] +# - RFC3986[https://www.rfc-editor.org/rfc/rfc3986] # # == Class tree # diff --git a/lib/uri/generic.rb b/lib/uri/generic.rb index baa6a4c34c3de3..bdd366661ecfd4 100644 --- a/lib/uri/generic.rb +++ b/lib/uri/generic.rb @@ -945,7 +945,7 @@ def fragment=(v) # == Description # # URI has components listed in order of decreasing significance from left to right, - # see RFC3986 https://tools.ietf.org/html/rfc3986 1.2.3. + # see RFC3986 https://www.rfc-editor.org/rfc/rfc3986 1.2.3. # # == Usage # diff --git a/lib/uri/http.rb b/lib/uri/http.rb index 306daf19656d93..900b132c8c1cbf 100644 --- a/lib/uri/http.rb +++ b/lib/uri/http.rb @@ -85,7 +85,7 @@ def request_uri # == Description # # Returns the authority for an HTTP uri, as defined in - # https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2. + # https://www.rfc-editor.org/rfc/rfc3986#section-3.2. # # # Example: @@ -106,7 +106,7 @@ def authority # == Description # # Returns the origin for an HTTP uri, as defined in - # https://datatracker.ietf.org/doc/html/rfc6454. + # https://www.rfc-editor.org/rfc/rfc6454. # # # Example: diff --git a/node.h b/node.h index 221c9f4eecbd18..191887c15b1b4c 100644 --- a/node.h +++ b/node.h @@ -73,10 +73,6 @@ VALUE rb_parser_dump_tree(const NODE *node, int comment); const struct kwtable *rb_reserved_word(const char *, unsigned int); struct parser_params; -void *rb_parser_malloc(struct parser_params *, size_t); -void *rb_parser_realloc(struct parser_params *, void *, size_t); -void *rb_parser_calloc(struct parser_params *, size_t, size_t); -void rb_parser_free(struct parser_params *, void *); PRINTF_ARGS(void rb_parser_printf(struct parser_params *parser, const char *fmt, ...), 2, 3); VALUE rb_node_set_type(NODE *n, enum node_type t); diff --git a/parse.y b/parse.y index 585130c3465ed6..7bea55cb2b6052 100644 --- a/parse.y +++ b/parse.y @@ -19,8 +19,6 @@ #define YYDEBUG 1 #define YYERROR_VERBOSE 1 #define YYSTACK_USE_ALLOCA 0 -#define YYLTYPE rb_code_location_t -#define YYLTYPE_IS_DECLARED 1 /* For Ripper */ #ifdef RUBY_EXTCONF_H @@ -51,7 +49,6 @@ #include "internal/encoding.h" #include "internal/error.h" #include "internal/hash.h" -#include "internal/imemo.h" #include "internal/io.h" #include "internal/numeric.h" #include "internal/parse.h" @@ -73,6 +70,22 @@ #include "ruby/ractor.h" #include "symbol.h" +#ifndef RIPPER +static VALUE +syntax_error_new(void) +{ + return rb_class_new_instance(0, 0, rb_eSyntaxError); +} +#endif + +static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, const YYLTYPE *loc); + +#define compile_callback rb_suppress_tracing +VALUE rb_io_gets_internal(VALUE io); + +VALUE rb_node_case_when_optimizable_literal(const NODE *const node); +#endif /* !UNIVERSAL_PARSER */ + #ifndef RIPPER static int rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2); @@ -117,15 +130,8 @@ rb_parser_regx_hash_cmp(rb_node_regx_t *n1, rb_node_regx_t *n2) rb_parser_string_hash_cmp(n1->string, n2->string)); } -static int -node_integer_line_cmp(const NODE *node_i, const NODE *line) -{ - VALUE num = rb_node_integer_literal_val(node_i); - - return !(FIXNUM_P(num) && line->nd_loc.beg_pos.lineno == FIX2INT(num)); -} - static st_index_t rb_parser_str_hash(rb_parser_string_t *str); +static st_index_t rb_char_p_hash(const char *c); static int literal_cmp(st_data_t val, st_data_t lit) @@ -137,22 +143,6 @@ literal_cmp(st_data_t val, st_data_t lit) enum node_type type_val = nd_type(node_val); enum node_type type_lit = nd_type(node_lit); - /* Special case for Integer and __LINE__ */ - if (type_val == NODE_INTEGER && type_lit == NODE_LINE) { - return node_integer_line_cmp(node_val, node_lit); - } - if (type_lit == NODE_INTEGER && type_val == NODE_LINE) { - return node_integer_line_cmp(node_lit, node_val); - } - - /* Special case for String and __FILE__ */ - if (type_val == NODE_STR && type_lit == NODE_FILE) { - return rb_parser_string_hash_cmp(RNODE_STR(node_val)->string, RNODE_FILE(node_lit)->path); - } - if (type_lit == NODE_STR && type_val == NODE_FILE) { - return rb_parser_string_hash_cmp(RNODE_STR(node_lit)->string, RNODE_FILE(node_val)->path); - } - if (type_val != type_lit) { return -1; } @@ -179,7 +169,11 @@ literal_cmp(st_data_t val, st_data_t lit) case NODE_ENCODING: return RNODE_ENCODING(node_val)->enc != RNODE_ENCODING(node_lit)->enc; default: +#ifdef UNIVERSAL_PARSER + abort(); +#else rb_bug("unexpected node: %s, %s", ruby_node_name(type_val), ruby_node_name(type_lit)); +#endif } } @@ -187,23 +181,17 @@ static st_index_t literal_hash(st_data_t a) { NODE *node = (NODE *)a; - VALUE val; enum node_type type = nd_type(node); switch (type) { case NODE_INTEGER: - val = rb_node_integer_literal_val(node); - if (!FIXNUM_P(val)) val = rb_big_hash(val); - return FIX2LONG(val); + return rb_char_p_hash(RNODE_INTEGER(node)->val); case NODE_FLOAT: - val = rb_node_float_literal_val(node); - return rb_dbl_long_hash(RFLOAT_VALUE(val)); + return rb_char_p_hash(RNODE_FLOAT(node)->val); case NODE_RATIONAL: - val = rb_node_rational_literal_val(node); - return rb_rational_hash(val); + return rb_char_p_hash(RNODE_RATIONAL(node)->val); case NODE_IMAGINARY: - val = rb_node_imaginary_literal_val(node); - return rb_complex_hash(val); + return rb_char_p_hash(RNODE_IMAGINARY(node)->val); case NODE_STR: return rb_parser_str_hash(RNODE_STR(node)->string); case NODE_SYM: @@ -211,32 +199,20 @@ literal_hash(st_data_t a) case NODE_REGX: return rb_parser_str_hash(RNODE_REGX(node)->string); case NODE_LINE: - /* Same with NODE_INTEGER FIXNUM case */ return (st_index_t)node->nd_loc.beg_pos.lineno; case NODE_FILE: - /* Same with NODE_STR */ return rb_parser_str_hash(RNODE_FILE(node)->path); case NODE_ENCODING: return (st_index_t)RNODE_ENCODING(node)->enc; default: +#ifdef UNIVERSAL_PARSER + abort(); +#else rb_bug("unexpected node: %s", ruby_node_name(type)); +#endif } } - -static VALUE -syntax_error_new(void) -{ - return rb_class_new_instance(0, 0, rb_eSyntaxError); -} -#endif - -static NODE *reg_named_capture_assign(struct parser_params* p, VALUE regexp, const YYLTYPE *loc); - -#define compile_callback rb_suppress_tracing -VALUE rb_io_gets_internal(VALUE io); - -VALUE rb_node_case_when_optimizable_literal(const NODE *const node); -#endif /* !UNIVERSAL_PARSER */ +#endif /* !RIPPER */ static inline int parse_isascii(int c) @@ -377,10 +353,6 @@ RBIMPL_WARNING_POP() #define yydebug (p->debug) /* disable the global variable definition */ -#define YYMALLOC(size) rb_parser_malloc(p, (size)) -#define YYREALLOC(ptr, size) rb_parser_realloc(p, (ptr), (size)) -#define YYCALLOC(nelem, size) rb_parser_calloc(p, (nelem), (size)) -#define YYFREE(ptr) rb_parser_free(p, (ptr)) #define YYFPRINTF(out, ...) rb_parser_printf(p, __VA_ARGS__) #define YY_LOCATION_PRINT(File, loc, p) \ rb_parser_printf(p, "%d.%d-%d.%d", \ @@ -522,8 +494,6 @@ typedef struct parser_string_buffer { token */ struct parser_params { - rb_imemo_tmpbuf_t *heap; - YYSTYPE *lval; YYLTYPE *yylloc; @@ -567,7 +537,7 @@ struct parser_params { VALUE ruby_sourcefile_string; rb_encoding *enc; token_info *token_info; - VALUE case_labels; + st_table *case_labels; rb_node_exits_t *exits; VALUE debug_buffer; @@ -1532,8 +1502,6 @@ int reg_fragment_check(struct parser_params*, rb_parser_string_t*, int); static int literal_concat0(struct parser_params *p, rb_parser_string_t *head, rb_parser_string_t *tail); static NODE *heredoc_dedent(struct parser_params*,NODE*); -static void check_literal_when(struct parser_params *p, NODE *args, const YYLTYPE *loc); - #ifdef RIPPER static VALUE var_field(struct parser_params *p, VALUE a); #define get_value(idx) (rb_ary_entry(p->s_value_stack, idx)) @@ -1615,6 +1583,9 @@ static void numparam_pop(struct parser_params *p, NODE *prev_inner); #define RE_OPTION_MASK 0xff #define RE_OPTION_ARG_ENCODING_NONE 32 +#define CHECK_LITERAL_WHEN (st_table *)1 +#define CASE_LABELS_ENABLED_P(case_labels) (case_labels && case_labels != CHECK_LITERAL_WHEN) + #define yytnamerr(yyres, yystr) (YYSIZE_T)rb_yytnamerr(p, yyres, yystr) size_t rb_yytnamerr(struct parser_params *p, char *yyres, const char *yystr); @@ -2079,7 +2050,6 @@ get_nd_args(struct parser_params *p, NODE *node) } #ifndef RIPPER -#ifndef UNIVERSAL_PARSER static st_index_t djb2(const uint8_t *str, size_t len) { @@ -2098,7 +2068,6 @@ parser_memhash(const void *ptr, long len) return djb2(ptr, len); } #endif -#endif #define PARSER_STRING_PTR(str) (str->ptr) #define PARSER_STRING_LEN(str) (str->len) @@ -2163,13 +2132,17 @@ rb_parser_string_free(rb_parser_t *p, rb_parser_string_t *str) } #ifndef RIPPER -#ifndef UNIVERSAL_PARSER static st_index_t rb_parser_str_hash(rb_parser_string_t *str) { return parser_memhash((const void *)PARSER_STRING_PTR(str), PARSER_STRING_LEN(str)); } -#endif + +static st_index_t +rb_char_p_hash(const char *c) +{ + return parser_memhash((const void *)c, strlen(c)); +} #endif static size_t @@ -2567,7 +2540,6 @@ rb_parser_str_resize(struct parser_params *p, rb_parser_string_t *str, long len) } #ifndef RIPPER -#ifndef UNIVERSAL_PARSER # define PARSER_ENC_STRING_GETMEM(str, ptrvar, lenvar, encvar) \ ((ptrvar) = str->ptr, \ (lenvar) = str->len, \ @@ -2587,7 +2559,6 @@ rb_parser_string_hash_cmp(rb_parser_string_t *str1, rb_parser_string_t *str2) enc1 != enc2 || memcmp(ptr1, ptr2, len1) != 0); } -#endif static void rb_parser_ary_extend(rb_parser_t *p, rb_parser_ary_t *ary, long len) @@ -2694,6 +2665,10 @@ rb_parser_tokens_free(rb_parser_t *p, rb_parser_ary_t *tokens) rb_parser_printf(p, "$%c", (int)RNODE_BACK_REF($$)->nd_nth); } tBACK_REF +%destructor { + if (CASE_LABELS_ENABLED_P($$)) st_free_table($$); +} + %lex-param {struct parser_params *p} %parse-param {struct parser_params *p} %initial-action @@ -2707,7 +2682,6 @@ rb_parser_tokens_free(rb_parser_t *p, rb_parser_ary_t *tokens) %after-pop-stack after_pop_stack %union { - VALUE val; NODE *node; rb_node_fcall_t *node_fcall; rb_node_args_t *node_args; @@ -2721,6 +2695,7 @@ rb_parser_tokens_free(rb_parser_t *p, rb_parser_ary_t *tokens) ID id; int num; st_table *tbl; + st_table *labels; const struct vtable *vars; struct rb_strterm_struct *strterm; struct lex_context ctxt; @@ -4536,28 +4511,28 @@ primary : literal } | k_case expr_value terms? { - $$ = p->case_labels; - p->case_labels = Qnil; - } + $$ = p->case_labels; + p->case_labels = CHECK_LITERAL_WHEN; + } case_body k_end { - if (RTEST(p->case_labels)) rb_hash_clear(p->case_labels); - p->case_labels = $4; + if (CASE_LABELS_ENABLED_P(p->case_labels)) st_free_table(p->case_labels); + p->case_labels = $4; $$ = NEW_CASE($2, $5, &@$); fixpos($$, $2); /*% ripper: case!($:2, $:5) %*/ } | k_case terms? { - $$ = p->case_labels; + $$ = p->case_labels; p->case_labels = 0; - } + } case_body k_end { - if (RTEST(p->case_labels)) rb_hash_clear(p->case_labels); - p->case_labels = $3; + if (p->case_labels) st_free_table(p->case_labels); + p->case_labels = $3; $$ = NEW_CASE2($4, &@$); /*% ripper: case!(Qnil, $:4) %*/ } @@ -5415,7 +5390,7 @@ do_body : { case_args : arg_value { - check_literal_when(p, $1, &@1); + rb_parser_check_literal_when(p, $1, &@1); $$ = NEW_LIST($1, &@$); /*% ripper: args_add!(args_new!, $:1) %*/ } @@ -5426,7 +5401,7 @@ case_args : arg_value } | case_args ',' arg_value { - check_literal_when(p, $3, &@3); + rb_parser_check_literal_when(p, $3, &@3); $$ = last_arg_append(p, $1, $3, &@$); /*% ripper: args_add!($:1, $:3) %*/ } @@ -7923,7 +7898,7 @@ parser_str_new(struct parser_params *p, const char *ptr, long len, rb_encoding * static int strterm_is_heredoc(rb_strterm_t *strterm) { - return strterm->flags & STRTERM_HEREDOC; + return strterm->heredoc; } static rb_strterm_t * @@ -7940,7 +7915,7 @@ static rb_strterm_t * new_heredoc(struct parser_params *p) { rb_strterm_t *strterm = ZALLOC(rb_strterm_t); - strterm->flags |= STRTERM_HEREDOC; + strterm->heredoc = true; return strterm; } @@ -10164,6 +10139,7 @@ parse_numeric(struct parser_params *p, int c) /* prefixed octal */ c = nextc(p); if (c == -1 || c == '_' || !ISDIGIT(c)) { + tokfix(p); return no_digits(p); } } @@ -11567,7 +11543,7 @@ yylex(YYSTYPE *lval, YYLTYPE *yylloc, struct parser_params *p) enum yytokentype t; p->lval = lval; - lval->val = Qundef; + lval->node = 0; p->yylloc = yylloc; t = parser_yylex(p); @@ -13486,36 +13462,35 @@ new_xstring(struct parser_params *p, NODE *node, const YYLTYPE *loc) } #ifndef RIPPER -VALUE -rb_parser_node_case_when_optimizable_literal(struct parser_params *p, const NODE *const node) -{ - return rb_node_case_when_optimizable_literal(node); -} -#endif +static const +struct st_hash_type literal_type = { + literal_cmp, + literal_hash, +}; -static void -check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc) -{ - VALUE lit; +static int nd_type_st_key_enable_p(NODE *node); +void +rb_parser_check_literal_when(struct parser_params *p, NODE *arg, const YYLTYPE *loc) +{ + /* See https://bugs.ruby-lang.org/issues/20331 for discussion about what is warned. */ if (!arg || !p->case_labels) return; + if (!nd_type_st_key_enable_p(arg)) return; - lit = rb_parser_node_case_when_optimizable_literal(p, arg); - if (UNDEF_P(lit)) return; - - if (NIL_P(p->case_labels)) { - p->case_labels = rb_obj_hide(rb_hash_new()); + if (p->case_labels == CHECK_LITERAL_WHEN) { + p->case_labels = st_init_table(&literal_type); } else { - VALUE line = rb_hash_lookup(p->case_labels, lit); - if (!NIL_P(line)) { + st_data_t line; + if (st_lookup(p->case_labels, (st_data_t)arg, &line)) { rb_warning1("duplicated 'when' clause with line %d is ignored", - WARN_IVAL(line)); + WARN_IVAL(INT2NUM((int)line))); return; } } - rb_hash_aset(p->case_labels, lit, INT2NUM(p->ruby_sourceline)); + st_insert(p->case_labels, (st_data_t)arg, (st_data_t)p->ruby_sourceline); } +#endif #ifdef RIPPER static int @@ -15228,14 +15203,7 @@ nd_value(struct parser_params *p, NODE *node) void rb_parser_warn_duplicate_keys(struct parser_params *p, NODE *hash) { -#ifndef UNIVERSAL_PARSER - static const -#endif - struct st_hash_type literal_type = { - literal_cmp, - literal_hash, - }; - + /* See https://bugs.ruby-lang.org/issues/20331 for discussion about what is warned. */ st_table *literal_keys = st_init_table_with_size(&literal_type, RNODE_LIST(hash)->as.nd_alen / 2); while (hash && RNODE_LIST(hash)->nd_next) { NODE *head = RNODE_LIST(hash)->nd_head; @@ -16158,7 +16126,6 @@ rb_ruby_parser_mark(void *ptr) rb_gc_mark(p->lex.input); rb_gc_mark(p->ruby_sourcefile_string); rb_gc_mark((VALUE)p->ast); - rb_gc_mark(p->case_labels); rb_gc_mark(p->delayed.token); #ifndef RIPPER rb_gc_mark(p->debug_lines); @@ -16173,9 +16140,6 @@ rb_ruby_parser_mark(void *ptr) #endif rb_gc_mark(p->debug_buffer); rb_gc_mark(p->debug_output); -#ifdef YYMALLOC - rb_gc_mark((VALUE)p->heap); -#endif } void @@ -16212,6 +16176,10 @@ rb_ruby_parser_free(void *ptr) st_free_table(p->pvtbl); } + if (CASE_LABELS_ENABLED_P(p->case_labels)) { + st_free_table(p->case_labels); + } + xfree(ptr); } @@ -16632,69 +16600,6 @@ rb_ruby_ripper_parser_allocate(void) #endif /* RIPPER */ #ifndef RIPPER -#ifdef YYMALLOC -#define HEAPCNT(n, size) ((n) * (size) / sizeof(YYSTYPE)) -/* Keep the order; NEWHEAP then xmalloc and ADD2HEAP to get rid of - * potential memory leak */ -#define NEWHEAP() rb_imemo_tmpbuf_parser_heap(0, p->heap, 0) -#define ADD2HEAP(new, cnt, ptr) ((p->heap = (new))->ptr = (ptr), \ - (new)->cnt = (cnt), (ptr)) - -void * -rb_parser_malloc(struct parser_params *p, size_t size) -{ - size_t cnt = HEAPCNT(1, size); - rb_imemo_tmpbuf_t *n = NEWHEAP(); - void *ptr = xmalloc(size); - - return ADD2HEAP(n, cnt, ptr); -} - -void * -rb_parser_calloc(struct parser_params *p, size_t nelem, size_t size) -{ - size_t cnt = HEAPCNT(nelem, size); - rb_imemo_tmpbuf_t *n = NEWHEAP(); - void *ptr = xcalloc(nelem, size); - - return ADD2HEAP(n, cnt, ptr); -} - -void * -rb_parser_realloc(struct parser_params *p, void *ptr, size_t size) -{ - rb_imemo_tmpbuf_t *n; - size_t cnt = HEAPCNT(1, size); - - if (ptr && (n = p->heap) != NULL) { - do { - if (n->ptr == ptr) { - n->ptr = ptr = xrealloc(ptr, size); - if (n->cnt) n->cnt = cnt; - return ptr; - } - } while ((n = n->next) != NULL); - } - n = NEWHEAP(); - ptr = xrealloc(ptr, size); - return ADD2HEAP(n, cnt, ptr); -} - -void -rb_parser_free(struct parser_params *p, void *ptr) -{ - rb_imemo_tmpbuf_t **prev = &p->heap, *n; - - while ((n = *prev) != NULL) { - if (n->ptr == ptr) { - *prev = n->next; - break; - } - prev = &n->next; - } -} -#endif - void rb_parser_printf(struct parser_params *p, const char *fmt, ...) { diff --git a/prism/config.yml b/prism/config.yml index bb1dd6e6b02ec0..c813b02d9f15b2 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -1,5 +1,6 @@ errors: - ALIAS_ARGUMENT + - ALIAS_ARGUMENT_NUMBERED_REFERENCE - AMPAMPEQ_MULTI_ASSIGN - ARGUMENT_AFTER_BLOCK - ARGUMENT_AFTER_FORWARDING_ELLIPSES @@ -46,6 +47,7 @@ errors: - CLASS_SUPERCLASS - CLASS_TERM - CLASS_UNEXPECTED_END + - CLASS_VARIABLE_BARE - CONDITIONAL_ELSIF_PREDICATE - CONDITIONAL_IF_PREDICATE - CONDITIONAL_PREDICATE_TERM @@ -107,6 +109,7 @@ errors: - FOR_IN - FOR_INDEX - FOR_TERM + - GLOBAL_VARIABLE_BARE - HASH_EXPRESSION_AFTER_LABEL - HASH_KEY - HASH_ROCKET @@ -118,6 +121,7 @@ errors: - INCOMPLETE_VARIABLE_CLASS_3_3_0 - INCOMPLETE_VARIABLE_INSTANCE - INCOMPLETE_VARIABLE_INSTANCE_3_3_0 + - INSTANCE_VARIABLE_BARE - INVALID_CHARACTER - INVALID_ENCODING_MAGIC_COMMENT - INVALID_FLOAT_EXPONENT @@ -166,7 +170,7 @@ errors: - PARAMETER_BLOCK_MULTI - PARAMETER_CIRCULAR - PARAMETER_METHOD_NAME - - PARAMETER_NAME_REPEAT + - PARAMETER_NAME_DUPLICATED - PARAMETER_NO_DEFAULT - PARAMETER_NO_DEFAULT_KW - PARAMETER_NUMBERED_RESERVED @@ -175,6 +179,7 @@ errors: - PARAMETER_STAR - PARAMETER_UNEXPECTED_FWD - PARAMETER_WILD_LOOSE_COMMA + - PATTERN_CAPTURE_DUPLICATE - PATTERN_EXPRESSION_AFTER_BRACKET - PATTERN_EXPRESSION_AFTER_COMMA - PATTERN_EXPRESSION_AFTER_HROCKET @@ -186,6 +191,7 @@ errors: - PATTERN_EXPRESSION_AFTER_RANGE - PATTERN_EXPRESSION_AFTER_REST - PATTERN_HASH_KEY + - PATTERN_HASH_KEY_DUPLICATE - PATTERN_HASH_KEY_LABEL - PATTERN_IDENT_AFTER_HROCKET - PATTERN_LABEL_AFTER_COMMA @@ -206,6 +212,7 @@ errors: - RESCUE_TERM - RESCUE_VARIABLE - RETURN_INVALID + - SCRIPT_NOT_FOUND - SINGLETON_FOR_LITERALS - STATEMENT_ALIAS - STATEMENT_POSTEXE_END @@ -255,6 +262,7 @@ warnings: - KEYWORD_EOL - LITERAL_IN_CONDITION_DEFAULT - LITERAL_IN_CONDITION_VERBOSE + - SHEBANG_CARRIAGE_RETURN - UNEXPECTED_CARRIAGE_RETURN tokens: - name: EOF @@ -629,9 +637,9 @@ flags: - name: InterpolatedStringNodeFlags values: - name: FROZEN - comment: "frozen by virtue of a `frozen_string_literal: true` comment or `--enable-frozen-string-literal`" + comment: "frozen by virtue of a `frozen_string_literal: true` comment or `--enable-frozen-string-literal`; only for adjacent string literals like `'a' 'b'`" - name: MUTABLE - comment: "mutable by virtue of a `frozen_string_literal: false` comment or `--disable-frozen-string-literal`" + comment: "mutable by virtue of a `frozen_string_literal: false` comment or `--disable-frozen-string-literal`; only for adjacent string literals like `'a' 'b'`" comment: Flags for interpolated string nodes that indicated mutability if they are also marked as literals. - name: KeywordHashNodeFlags values: diff --git a/prism/extension.c b/prism/extension.c index 5b5c0593c91719..27e799a8da645e 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -28,7 +28,6 @@ ID rb_option_id_encoding; ID rb_option_id_filepath; ID rb_option_id_frozen_string_literal; ID rb_option_id_line; -ID rb_option_id_offset; ID rb_option_id_scopes; ID rb_option_id_version; @@ -139,8 +138,6 @@ build_options_i(VALUE key, VALUE value, VALUE argument) { if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value))); } else if (key_id == rb_option_id_line) { if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value)); - } else if (key_id == rb_option_id_offset) { - if (!NIL_P(value)) pm_options_offset_set(options, NUM2UINT(value)); } else if (key_id == rb_option_id_frozen_string_literal) { if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value)); } else if (key_id == rb_option_id_version) { @@ -448,12 +445,15 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) { VALUE level = Qnil; switch (error->level) { - case PM_ERROR_LEVEL_FATAL: - level = ID2SYM(rb_intern("fatal")); + case PM_ERROR_LEVEL_SYNTAX: + level = ID2SYM(rb_intern("syntax")); break; case PM_ERROR_LEVEL_ARGUMENT: level = ID2SYM(rb_intern("argument")); break; + case PM_ERROR_LEVEL_LOAD: + level = ID2SYM(rb_intern("load")); + break; default: rb_raise(rb_eRuntimeError, "Unknown level: %" PRIu8, error->level); } @@ -1184,7 +1184,7 @@ format_errors(VALUE self, VALUE source, VALUE colorize) { pm_node_t *node = pm_parse(&parser); pm_buffer_t buffer = { 0 }; - pm_parser_errors_format(&parser, &buffer, RTEST(colorize)); + pm_parser_errors_format(&parser, &parser.error_list, &buffer, RTEST(colorize), true); rb_encoding *encoding = rb_enc_find(parser.encoding->name); VALUE result = rb_enc_str_new(pm_buffer_value(&buffer), pm_buffer_length(&buffer), encoding); @@ -1347,7 +1347,6 @@ Init_prism(void) { rb_option_id_filepath = rb_intern_const("filepath"); rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal"); rb_option_id_line = rb_intern_const("line"); - rb_option_id_offset = rb_intern_const("offset"); rb_option_id_scopes = rb_intern_const("scopes"); rb_option_id_version = rb_intern_const("version"); diff --git a/prism/options.c b/prism/options.c index cd575357aae58f..2854b765b9323b 100644 --- a/prism/options.c +++ b/prism/options.c @@ -24,14 +24,6 @@ pm_options_line_set(pm_options_t *options, int32_t line) { options->line = line; } -/** - * Set the offset option on the given options struct. - */ -PRISM_EXPORTED_FUNCTION void -pm_options_offset_set(pm_options_t *options, uint32_t offset) { - options->offset = offset; -} - /** * Set the frozen string literal option on the given options struct. */ @@ -201,9 +193,6 @@ pm_options_read(pm_options_t *options, const char *data) { options->line = pm_options_read_s32(data); data += 4; - options->offset = pm_options_read_u32(data); - data += 4; - uint32_t encoding_length = pm_options_read_u32(data); data += 4; diff --git a/prism/options.h b/prism/options.h index 9a4d6969c3c67b..d0b46a086491c7 100644 --- a/prism/options.h +++ b/prism/options.h @@ -66,12 +66,6 @@ typedef struct { */ int32_t line; - /** - * The offset within the file that the parse starts on. This value is - * 0-indexed. - */ - uint32_t offset; - /** * The name of the encoding that the source file is in. Note that this must * correspond to a name that can be found with Encoding.find in Ruby. @@ -164,14 +158,6 @@ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, cons */ PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t line); -/** - * Set the offset option on the given options struct. - * - * @param options The options struct to set the offset on. - * @param offset The offset to set. - */ -PRISM_EXPORTED_FUNCTION void pm_options_offset_set(pm_options_t *options, uint32_t offset); - /** * Set the encoding option on the given options struct. * @@ -267,7 +253,6 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options); * | `4` | the length of the filepath | * | ... | the filepath bytes | * | `4` | the line number | - * | `4` | the offset | * | `4` | the length the encoding | * | ... | the encoding bytes | * | `1` | frozen string literal | diff --git a/prism/prism.c b/prism/prism.c index c6353f451b586c..d03a11583f7a4d 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -260,10 +260,13 @@ lex_mode_push_list(pm_parser_t *parser, bool interpolation, uint8_t delimiter) { // We'll use strpbrk to find the first of these characters. uint8_t *breakpoints = lex_mode.as.list.breakpoints; memcpy(breakpoints, "\\ \t\f\r\v\n\0\0\0", sizeof(lex_mode.as.list.breakpoints)); - - // Now we'll add the terminator to the list of breakpoints. size_t index = 7; - breakpoints[index++] = terminator; + + // Now we'll add the terminator to the list of breakpoints. If the + // terminator is not already a NULL byte, add it to the list. + if (terminator != '\0') { + breakpoints[index++] = terminator; + } // If interpolation is allowed, then we're going to check for the # // character. Otherwise we'll only look for escapes and the terminator. @@ -309,13 +312,16 @@ lex_mode_push_regexp(pm_parser_t *parser, uint8_t incrementor, uint8_t terminato // characters. uint8_t *breakpoints = lex_mode.as.regexp.breakpoints; memcpy(breakpoints, "\r\n\\#\0\0", sizeof(lex_mode.as.regexp.breakpoints)); + size_t index = 4; // First we'll add the terminator. - breakpoints[4] = terminator; + if (terminator != '\0') { + breakpoints[index++] = terminator; + } // Next, if there is an incrementor, then we'll check for that as well. if (incrementor != '\0') { - breakpoints[5] = incrementor; + breakpoints[index++] = incrementor; } return lex_mode_push(parser, lex_mode); @@ -341,10 +347,13 @@ lex_mode_push_string(pm_parser_t *parser, bool interpolation, bool label_allowed // string. We'll use strpbrk to find the first of these characters. uint8_t *breakpoints = lex_mode.as.string.breakpoints; memcpy(breakpoints, "\r\n\\\0\0\0", sizeof(lex_mode.as.string.breakpoints)); - - // Now add in the terminator. size_t index = 3; - breakpoints[index++] = terminator; + + // Now add in the terminator. If the terminator is not already a NULL byte, + // then we'll add it. + if (terminator != '\0') { + breakpoints[index++] = terminator; + } // If interpolation is allowed, then we're going to check for the # // character. Otherwise we'll only look for escapes and the terminator. @@ -500,6 +509,9 @@ debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * call /** True if the -p command line option was given. */ #define PM_PARSER_COMMAND_LINE_OPTION_P(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_P) +/** True if the -x command line option was given. */ +#define PM_PARSER_COMMAND_LINE_OPTION_X(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_X) + /******************************************************************************/ /* Diagnostic-related functions */ /******************************************************************************/ @@ -812,12 +824,21 @@ pm_parser_warn_conditional_predicate_literal(pm_parser_t *parser, pm_node_t *nod } /** - * Add a warning to the parser if the value that is being written inside of a - * predicate to a conditional is a literal. + * Return true if the value being written within the predicate of a conditional + * is a literal value. */ -static void -pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, pm_node_t *node) { +static bool +pm_conditional_predicate_warn_write_literal_p(const pm_node_t *node) { switch (PM_NODE_TYPE(node)) { + case PM_ARRAY_NODE: { + const pm_array_node_t *cast = (const pm_array_node_t *) node; + for (size_t index = 0; index < cast->elements.size; index++) { + if (!pm_conditional_predicate_warn_write_literal_p(cast->elements.nodes[index])) { + return false; + } + } + return true; + } case PM_FALSE_NODE: case PM_FLOAT_NODE: case PM_IMAGINARY_NODE: @@ -831,10 +852,20 @@ pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, pm_node_t *node case PM_STRING_NODE: case PM_SYMBOL_NODE: case PM_TRUE_NODE: - pm_parser_warn_node(parser, node, parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3_0 : PM_WARN_EQUAL_IN_CONDITIONAL); - break; + return true; default: - break; + return false; + } +} + +/** + * Add a warning to the parser if the value that is being written inside of a + * predicate to a conditional is a literal. + */ +static inline void +pm_conditional_predicate_warn_write_literal(pm_parser_t *parser, const pm_node_t *node) { + if (pm_conditional_predicate_warn_write_literal_p(node)) { + pm_parser_warn_node(parser, node, parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_WARN_EQUAL_IN_CONDITIONAL_3_3_0 : PM_WARN_EQUAL_IN_CONDITIONAL); } } @@ -1162,6 +1193,77 @@ token_is_setter_name(pm_token_t *token) { ); } +/** + * Returns true if the given local variable is a keyword. + */ +static bool +pm_local_is_keyword(const char *source, size_t length) { +#define KEYWORD(name) if (memcmp(source, name, length) == 0) return true + + switch (length) { + case 2: + switch (source[0]) { + case 'd': KEYWORD("do"); return false; + case 'i': KEYWORD("if"); KEYWORD("in"); return false; + case 'o': KEYWORD("or"); return false; + default: return false; + } + case 3: + switch (source[0]) { + case 'a': KEYWORD("and"); return false; + case 'd': KEYWORD("def"); return false; + case 'e': KEYWORD("end"); return false; + case 'f': KEYWORD("for"); return false; + case 'n': KEYWORD("nil"); KEYWORD("not"); return false; + default: return false; + } + case 4: + switch (source[0]) { + case 'c': KEYWORD("case"); return false; + case 'e': KEYWORD("else"); return false; + case 'n': KEYWORD("next"); return false; + case 'r': KEYWORD("redo"); return false; + case 's': KEYWORD("self"); return false; + case 't': KEYWORD("then"); KEYWORD("true"); return false; + case 'w': KEYWORD("when"); return false; + default: return false; + } + case 5: + switch (source[0]) { + case 'a': KEYWORD("alias"); return false; + case 'b': KEYWORD("begin"); KEYWORD("break"); return false; + case 'c': KEYWORD("class"); return false; + case 'e': KEYWORD("elsif"); return false; + case 'f': KEYWORD("false"); return false; + case 'r': KEYWORD("retry"); return false; + case 's': KEYWORD("super"); return false; + case 'u': KEYWORD("undef"); KEYWORD("until"); return false; + case 'w': KEYWORD("while"); return false; + case 'y': KEYWORD("yield"); return false; + default: return false; + } + case 6: + switch (source[0]) { + case 'e': KEYWORD("ensure"); return false; + case 'm': KEYWORD("module"); return false; + case 'r': KEYWORD("rescue"); KEYWORD("return"); return false; + case 'u': KEYWORD("unless"); return false; + default: return false; + } + case 8: + KEYWORD("__LINE__"); + KEYWORD("__FILE__"); + return false; + case 12: + KEYWORD("__ENCODING__"); + return false; + default: + return false; + } + +#undef KEYWORD +} + /******************************************************************************/ /* Node flag handling functions */ /******************************************************************************/ @@ -1886,7 +1988,6 @@ pm_block_parameters_node_closing_set(pm_block_parameters_node_t *node, const pm_ */ static pm_block_local_variable_node_t * pm_block_local_variable_node_create(pm_parser_t *parser, const pm_token_t *name) { - assert(name->type == PM_TOKEN_IDENTIFIER || name->type == PM_TOKEN_MISSING); pm_block_local_variable_node_t *node = PM_ALLOC_NODE(parser, pm_block_local_variable_node_t); *node = (pm_block_local_variable_node_t) { @@ -3323,6 +3424,15 @@ pm_double_parse(pm_parser_t *parser, const pm_token_t *token) { char *buffer = xmalloc(sizeof(char) * (length + 1)); memcpy((void *) buffer, token->start, length); + // Next, determine if we need to replace the decimal point because of + // locale-specific options, and then normalize them if we have to. + char decimal_point = *localeconv()->decimal_point; + if (decimal_point != '.') { + for (size_t index = 0; index < length; index++) { + if (buffer[index] == '.') buffer[index] = decimal_point; + } + } + // Next, handle underscores by removing them from the buffer. for (size_t index = 0; index < length; index++) { if (buffer[index] == '_') { @@ -4837,7 +4947,8 @@ pm_refute_numbered_parameter(pm_parser_t *parser, const uint8_t *start, const ui * name and depth. */ static pm_local_variable_target_node_t * -pm_local_variable_target_node_create_values(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { +pm_local_variable_target_node_create(pm_parser_t *parser, const pm_location_t *location, pm_constant_id_t name, uint32_t depth) { + pm_refute_numbered_parameter(parser, location->start, location->end); pm_local_variable_target_node_t *node = PM_ALLOC_NODE(parser, pm_local_variable_target_node_t); *node = (pm_local_variable_target_node_t) { @@ -4852,36 +4963,6 @@ pm_local_variable_target_node_create_values(pm_parser_t *parser, const pm_locati return node; } -/** - * Allocate and initialize a new LocalVariableTargetNode node. - */ -static pm_local_variable_target_node_t * -pm_local_variable_target_node_create(pm_parser_t *parser, const pm_token_t *name) { - pm_refute_numbered_parameter(parser, name->start, name->end); - - return pm_local_variable_target_node_create_values( - parser, - &(pm_location_t) { .start = name->start, .end = name->end }, - pm_parser_constant_id_token(parser, name), - 0 - ); -} - -/** - * Allocate and initialize a new LocalVariableTargetNode node with the given depth. - */ -static pm_local_variable_target_node_t * -pm_local_variable_target_node_create_depth(pm_parser_t *parser, const pm_token_t *name, uint32_t depth) { - pm_refute_numbered_parameter(parser, name->start, name->end); - - return pm_local_variable_target_node_create_values( - parser, - &(pm_location_t) { .start = name->start, .end = name->end }, - pm_parser_constant_id_token(parser, name), - depth - ); -} - /** * Allocate and initialize a new MatchPredicateNode node. */ @@ -6984,9 +7065,9 @@ pm_parser_local_add_location(pm_parser_t *parser, const uint8_t *start, const ui /** * Add a local variable from a token to the current scope. */ -static inline void +static inline pm_constant_id_t pm_parser_local_add_token(pm_parser_t *parser, pm_token_t *token) { - pm_parser_local_add_location(parser, token->start, token->end); + return pm_parser_local_add_location(parser, token->start, token->end); } /** @@ -7074,7 +7155,7 @@ pm_parser_parameter_name_check(pm_parser_t *parser, const pm_token_t *name) { if (pm_constant_id_list_includes(&parser->current_scope->locals, constant_id)) { // Add an error if the parameter doesn't start with _ and has been seen before if ((name->start < name->end) && (*name->start != '_')) { - pm_parser_err_token(parser, name, PM_ERR_PARAMETER_NAME_REPEAT); + pm_parser_err_token(parser, name, PM_ERR_PARAMETER_NAME_DUPLICATED); } return true; } @@ -7489,7 +7570,7 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) { // Allocate a new magic comment node to append to the parser's list. pm_magic_comment_t *magic_comment; - if ((magic_comment = (pm_magic_comment_t *) xcalloc(sizeof(pm_magic_comment_t), 1)) != NULL) { + if ((magic_comment = (pm_magic_comment_t *) xcalloc(1, sizeof(pm_magic_comment_t))) != NULL) { magic_comment->key_start = key_start; magic_comment->value_start = value_start; magic_comment->key_length = (uint32_t) key_length; @@ -7922,8 +8003,7 @@ lex_numeric(pm_parser_t *parser) { static pm_token_type_t lex_global_variable(pm_parser_t *parser) { if (parser->current.end >= parser->end) { - pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0 : PM_ERR_INVALID_VARIABLE_GLOBAL; - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); + pm_parser_err_token(parser, &parser->current, PM_ERR_GLOBAL_VARIABLE_BARE); return PM_TOKEN_GLOBAL_VARIABLE; } @@ -7994,10 +8074,11 @@ lex_global_variable(pm_parser_t *parser) { parser->current.end += width; } while (parser->current.end < parser->end && (width = char_is_identifier(parser, parser->current.end)) > 0); } else { - // If we get here, then we have a $ followed by something that isn't - // recognized as a global variable. + // If we get here, then we have a $ followed by something that + // isn't recognized as a global variable. pm_diagnostic_id_t diag_id = parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0 ? PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0 : PM_ERR_INVALID_VARIABLE_GLOBAL; - PM_PARSER_ERR_TOKEN_FORMAT_CONTENT(parser, parser->current, diag_id); + size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); + PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); } return PM_TOKEN_GLOBAL_VARIABLE; @@ -8950,7 +9031,7 @@ lex_at_variable(pm_parser_t *parser) { while (parser->current.end < parser->end && (width = char_is_identifier(parser, parser->current.end)) > 0) { parser->current.end += width; } - } else { + } else if (parser->current.end < parser->end && pm_char_is_decimal_digit(*parser->current.end)) { pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE; if (parser->version == PM_OPTIONS_VERSION_CRUBY_3_3_0) { diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0 : PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0; @@ -8958,6 +9039,9 @@ lex_at_variable(pm_parser_t *parser) { size_t width = parser->encoding->char_width(parser->current.end, parser->end - parser->current.end); PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, diag_id, (int) ((parser->current.end + width) - parser->current.start), (const char *) parser->current.start); + } else { + pm_diagnostic_id_t diag_id = (type == PM_TOKEN_CLASS_VARIABLE) ? PM_ERR_CLASS_VARIABLE_BARE : PM_ERR_INSTANCE_VARIABLE_BARE; + pm_parser_err_token(parser, &parser->current, diag_id); } // If we're lexing an embedded variable, then we need to pop back into the @@ -8984,7 +9068,7 @@ parser_lex_callback(pm_parser_t *parser) { */ static inline pm_comment_t * parser_comment(pm_parser_t *parser, pm_comment_type_t type) { - pm_comment_t *comment = (pm_comment_t *) xcalloc(sizeof(pm_comment_t), 1); + pm_comment_t *comment = (pm_comment_t *) xcalloc(1, sizeof(pm_comment_t)); if (comment == NULL) return NULL; *comment = (pm_comment_t) { @@ -10554,19 +10638,19 @@ parser_lex(pm_parser_t *parser) { pm_token_type_t type = lex_identifier(parser, previous_command_start); - // If we've hit a __END__ and it was at the start of the line or the - // start of the file and it is followed by either a \n or a \r\n, then - // this is the last token of the file. + // If we've hit a __END__ and it was at the start of the + // line or the start of the file and it is followed by + // either a \n or a \r\n, then this is the last token of the + // file. if ( ((parser->current.end - parser->current.start) == 7) && current_token_starts_line(parser) && (memcmp(parser->current.start, "__END__", 7) == 0) && (parser->current.end == parser->end || match_eol(parser)) - ) - { - // Since we know we're about to add an __END__ comment, we know we - // need to add all of the newlines to get the correct column - // information for it. + ) { + // Since we know we're about to add an __END__ comment, + // we know we need to add all of the newlines to get the + // correct column information for it. const uint8_t *cursor = parser->current.end; while ((cursor = next_newline(cursor, parser->end - cursor)) != NULL) { pm_newline_list_append(&parser->newline_list, cursor++); @@ -10659,12 +10743,6 @@ parser_lex(pm_parser_t *parser) { pm_token_buffer_t token_buffer = { 0 }; while (breakpoint != NULL) { - // If we hit a null byte, skip directly past it. - if (*breakpoint == '\0') { - breakpoint = pm_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1), true); - continue; - } - // If we hit whitespace, then we must have received content by // now, so we can return an element of the list. if (pm_char_is_whitespace(*breakpoint)) { @@ -10701,6 +10779,12 @@ parser_lex(pm_parser_t *parser) { LEX(PM_TOKEN_STRING_END); } + // If we hit a null byte, skip directly past it. + if (*breakpoint == '\0') { + breakpoint = pm_strpbrk(parser, breakpoint + 1, breakpoints, parser->end - (breakpoint + 1), true); + continue; + } + // If we hit escapes, then we need to treat the next token // literally. In this case we'll skip past the next character // and find the next breakpoint. @@ -13514,7 +13598,28 @@ parse_block_parameters( if (accept1(parser, PM_TOKEN_SEMICOLON)) { do { - expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE); + switch (parser->current.type) { + case PM_TOKEN_CONSTANT: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_CONSTANT); + parser_lex(parser); + break; + case PM_TOKEN_INSTANCE_VARIABLE: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_IVAR); + parser_lex(parser); + break; + case PM_TOKEN_GLOBAL_VARIABLE: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_GLOBAL); + parser_lex(parser); + break; + case PM_TOKEN_CLASS_VARIABLE: + pm_parser_err_current(parser, PM_ERR_ARGUMENT_FORMAL_CLASS); + parser_lex(parser); + break; + default: + expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE); + break; + } + bool repeated = pm_parser_parameter_name_check(parser, &parser->previous); pm_parser_local_add_token(parser, &parser->previous); @@ -14523,13 +14628,30 @@ parse_heredoc_dedent(pm_parser_t *parser, pm_node_list_t *nodes, size_t common_w } static pm_node_t * -parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id); +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id); + +/** + * Add the newly created local to the list of captures for this pattern matching + * expression. If it is duplicated from a previous local, then we'll need to add + * an error to the parser. + */ +static void +parse_pattern_capture(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_constant_id_t capture, const pm_location_t *location) { + // Skip this capture if it starts with an underscore. + if (*location->start == '_') return; + + if (pm_constant_id_list_includes(captures, capture)) { + pm_parser_err(parser, location->start, location->end, PM_ERR_PATTERN_CAPTURE_DUPLICATE); + } else { + pm_constant_id_list_append(captures, capture); + } +} /** * Accept any number of constants joined by :: delimiters. */ static pm_node_t * -parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { +parse_pattern_constant_path(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node_t *node) { // Now, if there are any :: operators that follow, parse them as constant // path nodes. while (accept1(parser, PM_TOKEN_COLON_COLON)) { @@ -14537,7 +14659,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { expect1(parser, PM_TOKEN_CONSTANT, PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT); pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); - node = (pm_node_t *)pm_constant_path_node_create(parser, node, &delimiter, child); + node = (pm_node_t *) pm_constant_path_node_create(parser, node, &delimiter, child); } // If there is a [ or ( that follows, then this is part of a larger pattern @@ -14556,7 +14678,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { - inner = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); } @@ -14568,7 +14690,7 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { accept1(parser, PM_TOKEN_NEWLINE); if (!accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { - inner = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); } @@ -14651,18 +14773,30 @@ parse_pattern_constant_path(pm_parser_t *parser, pm_node_t *node) { * Parse a rest pattern. */ static pm_splat_node_t * -parse_pattern_rest(pm_parser_t *parser) { +parse_pattern_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { assert(parser->previous.type == PM_TOKEN_USTAR); pm_token_t operator = parser->previous; pm_node_t *name = NULL; // Rest patterns don't necessarily have a name associated with them. So we - // will check for that here. If they do, then we'll add it to the local table - // since this pattern will cause it to become a local variable. + // will check for that here. If they do, then we'll add it to the local + // table since this pattern will cause it to become a local variable. if (accept1(parser, PM_TOKEN_IDENTIFIER)) { pm_token_t identifier = parser->previous; - pm_parser_local_add_token(parser, &identifier); - name = (pm_node_t *) pm_local_variable_target_node_create(parser, &identifier); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &identifier); + + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); + } + + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&identifier)); + name = (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&identifier), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); } // Finally we can return the created node. @@ -14673,7 +14807,7 @@ parse_pattern_rest(pm_parser_t *parser) { * Parse a keyword rest node. */ static pm_node_t * -parse_pattern_keyword_rest(pm_parser_t *parser) { +parse_pattern_keyword_rest(pm_parser_t *parser, pm_constant_id_list_t *captures) { assert(parser->current.type == PM_TOKEN_USTAR_STAR); parser_lex(parser); @@ -14685,8 +14819,20 @@ parse_pattern_keyword_rest(pm_parser_t *parser) { } if (accept1(parser, PM_TOKEN_IDENTIFIER)) { - pm_parser_local_add_token(parser, &parser->previous); - value = (pm_node_t *) pm_local_variable_target_node_create(parser, &parser->previous); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); + + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); + } + + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + value = (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&parser->previous), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); } return (pm_node_t *) pm_assoc_splat_node_create(parser, value, &operator); @@ -14697,30 +14843,44 @@ parse_pattern_keyword_rest(pm_parser_t *parser) { * value. This will use an implicit local variable target. */ static pm_node_t * -parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_symbol_node_t *key) { +parse_pattern_hash_implicit_value(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_symbol_node_t *key) { const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - pm_constant_id_t name = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); - - int current_depth = pm_parser_local_depth_constant_id(parser, name); - uint32_t depth; + pm_constant_id_t constant_id = pm_parser_constant_id_location(parser, value_loc->start, value_loc->end); - if (current_depth == -1) { - pm_parser_local_add_location(parser, value_loc->start, value_loc->end); - depth = 0; - } else { - depth = (uint32_t) current_depth; + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); } - pm_local_variable_target_node_t *target = pm_local_variable_target_node_create_values(parser, value_loc, name, depth); + parse_pattern_capture(parser, captures, constant_id, value_loc); + pm_local_variable_target_node_t *target = pm_local_variable_target_node_create( + parser, + value_loc, + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); + return (pm_node_t *) pm_implicit_node_create(parser, (pm_node_t *) target); } +/** + * Add a node to the list of keys for a hash pattern, and if it is a duplicate + * then add an error to the parser. + */ +static void +parse_pattern_hash_key(pm_parser_t *parser, pm_static_literals_t *keys, pm_node_t *node) { + if (pm_static_literals_add(parser, keys, node) != NULL) { + pm_parser_err_node(parser, node, PM_ERR_PATTERN_HASH_KEY_DUPLICATE); + } +} + /** * Parse a hash pattern. */ static pm_hash_pattern_node_t * -parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { +parse_pattern_hash(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_node_t *first_node) { pm_node_list_t assocs = { 0 }; + pm_static_literals_t keys = { 0 }; pm_node_t *rest = NULL; switch (PM_NODE_TYPE(first_node)) { @@ -14730,16 +14890,17 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { break; case PM_SYMBOL_NODE: { if (pm_symbol_node_label_p(first_node)) { + parse_pattern_hash_key(parser, &keys, first_node); pm_node_t *value; - if (!match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - // Here we have a value for the first assoc in the list, so - // we will parse it now. - value = parse_pattern(parser, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); - } else { + if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { // Otherwise, we will create an implicit local variable // target for the value. - value = parse_pattern_hash_implicit_value(parser, (pm_symbol_node_t *) first_node); + value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) first_node); + } else { + // Here we have a value for the first assoc in the list, so + // we will parse it now. + value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -14773,7 +14934,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { } if (match1(parser, PM_TOKEN_USTAR_STAR)) { - pm_node_t *assoc = parse_pattern_keyword_rest(parser); + pm_node_t *assoc = parse_pattern_keyword_rest(parser, captures); if (rest == NULL) { rest = assoc; @@ -14784,14 +14945,14 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { } else { expect1(parser, PM_TOKEN_LABEL, PM_ERR_PATTERN_LABEL_AFTER_COMMA); pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); + + parse_pattern_hash_key(parser, &keys, key); pm_node_t *value = NULL; - if (!match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { - value = parse_pattern(parser, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); + if (match7(parser, PM_TOKEN_COMMA, PM_TOKEN_KEYWORD_THEN, PM_TOKEN_BRACE_RIGHT, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { + value = parse_pattern_hash_implicit_value(parser, captures, (pm_symbol_node_t *) key); } else { - const pm_location_t *value_loc = &((pm_symbol_node_t *) key)->value_loc; - pm_parser_local_add_location(parser, value_loc->start, value_loc->end); - value = parse_pattern_hash_implicit_value(parser, (pm_symbol_node_t *) key); + value = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_KEY); } pm_token_t operator = not_provided(parser); @@ -14808,6 +14969,7 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { pm_hash_pattern_node_t *node = pm_hash_pattern_node_node_list_create(parser, &assocs, rest); xfree(assocs.nodes); + pm_static_literals_free(&keys); return node; } @@ -14815,18 +14977,25 @@ parse_pattern_hash(pm_parser_t *parser, pm_node_t *first_node) { * Parse a pattern expression primitive. */ static pm_node_t * -parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { +parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_diagnostic_id_t diag_id) { switch (parser->current.type) { case PM_TOKEN_IDENTIFIER: case PM_TOKEN_METHOD_NAME: { parser_lex(parser); - pm_token_t name = parser->previous; - int depth = pm_parser_local_depth(parser, &name); - if (depth < 0) { - depth = 0; - pm_parser_local_add_token(parser, &name); + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); + + int depth; + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); } - return (pm_node_t *) pm_local_variable_target_node_create_depth(parser, &name, (uint32_t) depth); + + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + return (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&parser->previous), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); } case PM_TOKEN_BRACKET_LEFT_ARRAY: { pm_token_t opening = parser->current; @@ -14835,15 +15004,14 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { if (accept1(parser, PM_TOKEN_BRACKET_RIGHT)) { // If we have an empty array pattern, then we'll just return a new // array pattern node. - return (pm_node_t *)pm_array_pattern_node_empty_create(parser, &opening, &parser->previous); + return (pm_node_t *) pm_array_pattern_node_empty_create(parser, &opening, &parser->previous); } // Otherwise, we'll parse the inner pattern, then deal with it depending // on the type it returns. - pm_node_t *inner = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); + pm_node_t *inner = parse_pattern(parser, captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET); accept1(parser, PM_TOKEN_NEWLINE); - expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_PATTERN_TERM_BRACKET); pm_token_t closing = parser->previous; @@ -14905,7 +15073,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { first_node = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); break; case PM_TOKEN_USTAR_STAR: - first_node = parse_pattern_keyword_rest(parser); + first_node = parse_pattern_keyword_rest(parser, captures); break; case PM_TOKEN_STRING_BEGIN: first_node = parse_expression(parser, PM_BINDING_POWER_MAX, false, PM_ERR_PATTERN_HASH_KEY); @@ -14919,7 +15087,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { } } - node = parse_pattern_hash(parser, first_node); + node = parse_pattern_hash(parser, captures, first_node); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_PATTERN_TERM_BRACE); @@ -15066,14 +15234,14 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { pm_node_t *child = (pm_node_t *) pm_constant_read_node_create(parser, &parser->previous); pm_constant_path_node_t *node = pm_constant_path_node_create(parser, NULL, &delimiter, child); - return parse_pattern_constant_path(parser, (pm_node_t *)node); + return parse_pattern_constant_path(parser, captures, (pm_node_t *) node); } case PM_TOKEN_CONSTANT: { pm_token_t constant = parser->current; parser_lex(parser); pm_node_t *node = (pm_node_t *) pm_constant_read_node_create(parser, &constant); - return parse_pattern_constant_path(parser, node); + return parse_pattern_constant_path(parser, captures, node); } default: pm_parser_err_current(parser, diag_id); @@ -15086,7 +15254,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { * assignment. */ static pm_node_t * -parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { +parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, pm_diagnostic_id_t diag_id) { pm_node_t *node = NULL; do { @@ -15103,9 +15271,9 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { case PM_TOKEN_UDOT_DOT_DOT: case PM_CASE_PRIMITIVE: { if (node == NULL) { - node = parse_pattern_primitive(parser, diag_id); + node = parse_pattern_primitive(parser, captures, diag_id); } else { - pm_node_t *right = parse_pattern_primitive(parser, PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE); + pm_node_t *right = parse_pattern_primitive(parser, captures, PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE); node = (pm_node_t *) pm_alternation_pattern_node_create(parser, node, right, &operator); } @@ -15115,7 +15283,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { pm_token_t opening = parser->current; parser_lex(parser); - pm_node_t *body = parse_pattern(parser, false, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); + pm_node_t *body = parse_pattern(parser, captures, false, PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN); accept1(parser, PM_TOKEN_NEWLINE); expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_PATTERN_TERM_PAREN); pm_node_t *right = (pm_node_t *) pm_parentheses_node_create(parser, &opening, body, &parser->previous); @@ -15147,16 +15315,23 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { // In this case we should create an assignment node. while (accept1(parser, PM_TOKEN_EQUAL_GREATER)) { pm_token_t operator = parser->previous; - expect1(parser, PM_TOKEN_IDENTIFIER, PM_ERR_PATTERN_IDENT_AFTER_HROCKET); - pm_token_t identifier = parser->previous; - int depth = pm_parser_local_depth(parser, &identifier); - if (depth < 0) { - depth = 0; - pm_parser_local_add_token(parser, &identifier); + + pm_constant_id_t constant_id = pm_parser_constant_id_token(parser, &parser->previous); + int depth; + + if ((depth = pm_parser_local_depth_constant_id(parser, constant_id)) == -1) { + pm_parser_local_add(parser, constant_id); } - pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create_depth(parser, &identifier, (uint32_t) depth); + parse_pattern_capture(parser, captures, constant_id, &PM_LOCATION_TOKEN_VALUE(&parser->previous)); + pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create( + parser, + &PM_LOCATION_TOKEN_VALUE(&parser->previous), + constant_id, + (uint32_t) (depth == -1 ? 0 : depth) + ); + node = (pm_node_t *) pm_capture_pattern_node_create(parser, node, target, &operator); } @@ -15167,7 +15342,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_diagnostic_id_t diag_id) { * Parse a pattern matching expression. */ static pm_node_t * -parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) { +parse_pattern(pm_parser_t *parser, pm_constant_id_list_t *captures, bool top_pattern, pm_diagnostic_id_t diag_id) { pm_node_t *node = NULL; bool leading_rest = false; @@ -15177,30 +15352,30 @@ parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) case PM_TOKEN_LABEL: { parser_lex(parser); pm_node_t *key = (pm_node_t *) pm_symbol_node_label_create(parser, &parser->previous); - return (pm_node_t *) parse_pattern_hash(parser, key); + return (pm_node_t *) parse_pattern_hash(parser, captures, key); } case PM_TOKEN_USTAR_STAR: { - node = parse_pattern_keyword_rest(parser); - return (pm_node_t *) parse_pattern_hash(parser, node); + node = parse_pattern_keyword_rest(parser, captures); + return (pm_node_t *) parse_pattern_hash(parser, captures, node); } case PM_TOKEN_USTAR: { if (top_pattern) { parser_lex(parser); - node = (pm_node_t *) parse_pattern_rest(parser); + node = (pm_node_t *) parse_pattern_rest(parser, captures); leading_rest = true; break; } } /* fallthrough */ default: - node = parse_pattern_primitives(parser, diag_id); + node = parse_pattern_primitives(parser, captures, diag_id); break; } // If we got a dynamic label symbol, then we need to treat it like the // beginning of a hash pattern. if (pm_symbol_node_label_p(node)) { - return (pm_node_t *) parse_pattern_hash(parser, node); + return (pm_node_t *) parse_pattern_hash(parser, captures, node); } if (top_pattern && match1(parser, PM_TOKEN_COMMA)) { @@ -15220,7 +15395,7 @@ parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) } if (accept1(parser, PM_TOKEN_USTAR)) { - node = (pm_node_t *) parse_pattern_rest(parser); + node = (pm_node_t *) parse_pattern_rest(parser, captures); // If we have already parsed a splat pattern, then this is an error. We // will continue to parse the rest of the patterns, but we will indicate @@ -15231,7 +15406,7 @@ parse_pattern(pm_parser_t *parser, bool top_pattern, pm_diagnostic_id_t diag_id) trailing_rest = true; } else { - node = parse_pattern_primitives(parser, PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA); + node = parse_pattern_primitives(parser, captures, PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA); } pm_node_list_append(&nodes, node); @@ -16157,7 +16332,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b case PM_GLOBAL_VARIABLE_READ_NODE: { if (PM_NODE_TYPE_P(old_name, PM_BACK_REFERENCE_READ_NODE) || PM_NODE_TYPE_P(old_name, PM_NUMBERED_REFERENCE_READ_NODE) || PM_NODE_TYPE_P(old_name, PM_GLOBAL_VARIABLE_READ_NODE)) { if (PM_NODE_TYPE_P(old_name, PM_NUMBERED_REFERENCE_READ_NODE)) { - pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT); + pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT_NUMBERED_REFERENCE); } } else { pm_parser_err_node(parser, old_name, PM_ERR_ALIAS_ARGUMENT); @@ -16198,8 +16373,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b return (pm_node_t *) pm_case_node_create(parser, &case_keyword, predicate, &parser->previous); } - // At this point we can create a case node, though we don't yet know if it - // is a case-in or case-when node. + // At this point we can create a case node, though we don't yet know + // if it is a case-in or case-when node. pm_token_t end_keyword = not_provided(parser); pm_node_t *node; @@ -16277,8 +16452,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pm_parser_err_token(parser, &case_keyword, PM_ERR_CASE_MATCH_MISSING_PREDICATE); } - // At this point we expect that we're parsing a case-in node. We will - // continue to parse the in nodes until we hit the end of the list. + // At this point we expect that we're parsing a case-in node. We + // will continue to parse the in nodes until we hit the end of + // the list. while (match1(parser, PM_TOKEN_KEYWORD_IN)) { bool previous_pattern_matching_newlines = parser->pattern_matching_newlines; parser->pattern_matching_newlines = true; @@ -16288,11 +16464,16 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b parser_lex(parser); pm_token_t in_keyword = parser->previous; - pm_node_t *pattern = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + + pm_constant_id_list_t captures = { 0 }; + pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + parser->pattern_matching_newlines = previous_pattern_matching_newlines; + pm_constant_id_list_free(&captures); - // Since we're in the top-level of the case-in node we need to check - // for guard clauses in the form of `if` or `unless` statements. + // Since we're in the top-level of the case-in node we need + // to check for guard clauses in the form of `if` or + // `unless` statements. if (accept1(parser, PM_TOKEN_KEYWORD_IF_MODIFIER)) { pm_token_t keyword = parser->previous; pm_node_t *predicate = parse_value_expression(parser, PM_BINDING_POWER_COMPOSITION, true, PM_ERR_CONDITIONAL_IF_PREDICATE); @@ -16303,9 +16484,9 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b pattern = (pm_node_t *) pm_unless_node_modifier_create(parser, pattern, &keyword, predicate); } - // Now we need to check for the terminator of the in node's pattern. - // It can be a newline or semicolon optionally followed by a `then` - // keyword. + // Now we need to check for the terminator of the in node's + // pattern. It can be a newline or semicolon optionally + // followed by a `then` keyword. pm_token_t then_keyword; if (accept2(parser, PM_TOKEN_NEWLINE, PM_TOKEN_SEMICOLON)) { if (accept1(parser, PM_TOKEN_KEYWORD_THEN)) { @@ -16318,8 +16499,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b then_keyword = parser->previous; } - // Now we can actually parse the statements associated with the in - // node. + // Now we can actually parse the statements associated with + // the in node. pm_statements_node_t *statements; if (match3(parser, PM_TOKEN_KEYWORD_IN, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_END)) { statements = NULL; @@ -16327,8 +16508,8 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b statements = parse_statements(parser, PM_CONTEXT_CASE_IN); } - // Now that we have the full pattern and statements, we can create the - // node and attach it to the case node. + // Now that we have the full pattern and statements, we can + // create the node and attach it to the case node. pm_node_t *condition = (pm_node_t *) pm_in_node_create(parser, pattern, statements, &in_keyword, &then_keyword); pm_case_match_node_condition_append(case_node, condition); } @@ -17984,18 +18165,32 @@ parse_call_operator_write(pm_parser_t *parser, pm_call_node_t *call_node, const } } +/** + * Returns true if the name of the capture group is a valid local variable that + * can be written to. + */ static bool -name_is_identifier(pm_parser_t *parser, const uint8_t *source, size_t length) { +parse_regular_expression_named_capture(pm_parser_t *parser, const uint8_t *source, size_t length) { if (length == 0) { return false; } + // First ensure that it starts with a valid identifier starting character. size_t width = char_is_identifier_start(parser, source); if (!width) { return false; } - uint8_t *cursor = ((uint8_t *)source) + width; + // Next, ensure that it's not an uppercase character. + if (parser->encoding_changed) { + if (parser->encoding->isupper_char(source, (ptrdiff_t) length)) return false; + } else { + if (pm_encoding_utf_8_isupper_char(source, (ptrdiff_t) length)) return false; + } + + // Next, iterate through all of the bytes of the string to ensure that they + // are all valid identifier characters. + const uint8_t *cursor = source + width; while (cursor < source + length && (width = char_is_identifier(parser, cursor))) { cursor += width; } @@ -18029,14 +18224,13 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * // If the name of the capture group isn't a valid identifier, we do // not add it to the local table. - if (!name_is_identifier(parser, source, length)) continue; + if (!parse_regular_expression_named_capture(parser, source, length)) continue; if (content->type == PM_STRING_SHARED) { // If the unescaped string is a slice of the source, then we can // copy the names directly. The pointers will line up. location = (pm_location_t) { .start = source, .end = source + length }; name = pm_parser_constant_id_location(parser, location.start, location.end); - pm_refute_numbered_parameter(parser, source, source + length); } else { // Otherwise, the name is a slice of the malloc-ed owned string, // in which case we need to copy it out into a new string. @@ -18047,11 +18241,6 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * memcpy(memory, source, length); name = pm_parser_constant_id_owned(parser, (uint8_t *) memory, length); - - if (pm_token_is_numbered_parameter(source, source + length)) { - const pm_location_t *location = &call->receiver->location; - PM_PARSER_ERR_LOCATION_FORMAT(parser, location, PM_ERR_PARAMETER_NUMBERED_RESERVED, location->start); - } } if (name != 0) { @@ -18060,19 +18249,22 @@ parse_regular_expression_named_captures(pm_parser_t *parser, const pm_string_t * if (pm_constant_id_list_includes(&names, name)) continue; pm_constant_id_list_append(&names, name); - // Here we lazily create the MatchWriteNode since we know we're - // about to add a target. - if (match == NULL) match = pm_match_write_node_create(parser, call); - - // First, find the depth of the local that is being assigned. int depth; if ((depth = pm_parser_local_depth_constant_id(parser, name)) == -1) { + // If the identifier is not already a local, then we'll add + // it to the local table unless it's a keyword. + if (pm_local_is_keyword((const char *) source, length)) continue; + pm_parser_local_add(parser, name); } + // Here we lazily create the MatchWriteNode since we know we're + // about to add a target. + if (match == NULL) match = pm_match_write_node_create(parser, call); + // Next, create the local variable target and add it to the // list of targets for the match. - pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create_values(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); + pm_node_t *target = (pm_node_t *) pm_local_variable_target_node_create(parser, &location, name, depth == -1 ? 0 : (uint32_t) depth); pm_node_list_append(&match->targets, target); } } @@ -18844,11 +19036,13 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_token_t operator = parser->current; parser->command_start = false; lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); - parser_lex(parser); - pm_node_t *pattern = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + pm_constant_id_list_t captures = { 0 }; + pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_IN); + parser->pattern_matching_newlines = previous_pattern_matching_newlines; + pm_constant_id_list_free(&captures); return (pm_node_t *) pm_match_predicate_node_create(parser, node, pattern, &operator); } @@ -18859,11 +19053,13 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t pm_token_t operator = parser->current; parser->command_start = false; lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); - parser_lex(parser); - pm_node_t *pattern = parse_pattern(parser, true, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); + pm_constant_id_list_t captures = { 0 }; + pm_node_t *pattern = parse_pattern(parser, &captures, true, PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET); + parser->pattern_matching_newlines = previous_pattern_matching_newlines; + pm_constant_id_list_free(&captures); return (pm_node_t *) pm_match_required_node_create(parser, node, pattern, &operator); } @@ -19103,6 +19299,38 @@ parse_program(pm_parser_t *parser) { /* External functions */ /******************************************************************************/ +/** + * A vendored version of strnstr that is used to find a substring within a + * string with a given length. This function is used to search for the Ruby + * engine name within a shebang when the -x option is passed to Ruby. + * + * The only modification that we made here is that we don't do NULL byte checks + * because we know the little parameter will not have a NULL byte and we allow + * the big parameter to have them. + */ +static const char * +pm_strnstr(const char *big, const char *little, size_t big_length) { + size_t little_length = strlen(little); + + for (const char *big_end = big + big_length; big < big_end; big++) { + if (*big == *little && memcmp(big, little, little_length) == 0) return big; + } + + return NULL; +} + +/** + * Potentially warn the user if the shebang that has been found to include + * "ruby" has a carriage return at the end, as that can cause problems on some + * platforms. + */ +static void +pm_parser_warn_shebang_carriage_return(pm_parser_t *parser, const uint8_t *start, size_t length) { + if (length > 2 && start[length - 1] == '\n' && start[length - 2] == '\r') { + pm_parser_warn(parser, start, start + length, PM_WARN_SHEBANG_CARRIAGE_RETURN); + } +} + /** * Initialize a parser with the given start and end pointers. */ @@ -19189,22 +19417,6 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm // line option parser->start_line = options->line; - // offset option - if (options->offset != 0) { - const uint8_t *cursor = parser->start; - const uint8_t *offset = cursor + options->offset; - - const uint8_t *newline = NULL; - while ((newline = next_newline(cursor, parser->end - cursor)) != NULL) { - if (newline > offset) break; - pm_newline_list_append(&parser->newline_list, newline); - cursor = newline + 1; - } - - parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = offset, .end = offset }; - parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = offset, .end = offset }; - } - // encoding option size_t encoding_length = pm_string_length(&options->encoding); if (encoding_length > 0) { @@ -19251,16 +19463,81 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm if (size >= 3 && source[0] == 0xef && source[1] == 0xbb && source[2] == 0xbf) { parser->current.end += 3; parser->encoding_comment_start += 3; + + if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) { + parser->encoding = PM_ENCODING_UTF_8_ENTRY; + if (parser->encoding_changed_callback != NULL) parser->encoding_changed_callback(parser); + } } + // If the -x command line flag is set, or the first shebang of the file does + // not include "ruby", then we'll search for a shebang that does include + // "ruby" and start parsing from there. + bool search_shebang = PM_PARSER_COMMAND_LINE_OPTION_X(parser); + // If the first two bytes of the source are a shebang, then we'll indicate // that the encoding comment is at the end of the shebang. if (peek(parser) == '#' && peek_offset(parser, 1) == '!') { - const uint8_t *encoding_comment_start = next_newline(source, (ptrdiff_t) size); - if (encoding_comment_start) { - parser->encoding_comment_start = encoding_comment_start + 1; + const uint8_t *newline = next_newline(parser->start, parser->end - parser->start); + size_t length = (size_t) ((newline != NULL ? newline : parser->end) - parser->start); + + if (pm_strnstr((const char *) parser->start, "ruby", length) != NULL) { + pm_parser_warn_shebang_carriage_return(parser, parser->start, length); + if (newline != NULL) parser->encoding_comment_start = newline + 1; + search_shebang = false; + } else { + search_shebang = true; } } + + // Here we're going to find the first shebang that includes "ruby" and start + // parsing from there. + if (search_shebang) { + // If a shebang that includes "ruby" is not found, then we're going to a + // a load error to the list of errors on the parser. + bool found_shebang = false; + + // This is going to point to the start of each line as we check it. + // We'll maintain a moving window looking at each line at they come. + const uint8_t *cursor = parser->start; + + // The newline pointer points to the end of the current line that we're + // considering. If it is NULL, then we're at the end of the file. + const uint8_t *newline = next_newline(cursor, parser->end - cursor); + + while (newline != NULL) { + pm_newline_list_append(&parser->newline_list, newline); + + cursor = newline + 1; + newline = next_newline(cursor, parser->end - cursor); + + size_t length = (size_t) ((newline != NULL ? newline : parser->end) - cursor); + if (length > 2 && cursor[0] == '#' && cursor[1] == '!') { + if (parser->newline_list.size == 1) { + pm_parser_warn_shebang_carriage_return(parser, cursor, length); + } + + if (pm_strnstr((const char *) cursor, "ruby", length) != NULL) { + found_shebang = true; + parser->encoding_comment_start = newline + 1; + break; + } + } + } + + if (found_shebang) { + parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; + parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor }; + } else { + pm_parser_err(parser, parser->start, parser->start, PM_ERR_SCRIPT_NOT_FOUND); + pm_newline_list_clear(&parser->newline_list); + } + } + + // The encoding comment can start after any amount of inline whitespace, so + // here we'll advance it to the first non-inline-whitespace character so + // that it is ready for future comparisons. + parser->encoding_comment_start += pm_strspn_inline_whitespace(parser->encoding_comment_start, parser->end - parser->encoding_comment_start); } /** @@ -19671,8 +19948,7 @@ pm_parser_errors_format_line(const pm_parser_t *parser, const pm_newline_list_t * Format the errors on the parser into the given buffer. */ PRISM_EXPORTED_FUNCTION void -pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize) { - const pm_list_t *error_list = &parser->error_list; +pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages) { assert(error_list->size != 0); // First, we're going to sort all of the errors by line number using an @@ -19687,7 +19963,15 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col // blank lines based on the maximum number of digits in the line numbers // that are going to be displaid. pm_error_format_t error_format; - int32_t max_line_number = errors[error_list->size - 1].line - start_line; + int32_t first_line_number = errors[0].line; + int32_t last_line_number = errors[error_list->size - 1].line; + + // If we have a maximum line number that is negative, then we're going to + // use the absolute value for comparison but multiple by 10 to additionally + // have a column for the negative sign. + if (first_line_number < 0) first_line_number = (-first_line_number) * 10; + if (last_line_number < 0) last_line_number = (-last_line_number) * 10; + int32_t max_line_number = first_line_number > last_line_number ? first_line_number : last_line_number; if (max_line_number < 10) { if (colorize) { @@ -19831,16 +20115,18 @@ pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool col column += (char_width == 0 ? 1 : char_width); } - pm_buffer_append_byte(buffer, ' '); + if (inline_messages) { + pm_buffer_append_byte(buffer, ' '); + const char *message = error->error->message; + pm_buffer_append_string(buffer, message, strlen(message)); + } - const char *message = error->error->message; - pm_buffer_append_string(buffer, message, strlen(message)); pm_buffer_append_byte(buffer, '\n'); // Here we determine how many lines of padding to display after the // error, depending on where the next error is in source. last_line = error->line; - int32_t next_line = (index == error_list->size - 1) ? ((int32_t) newline_list->size) : errors[index + 1].line; + int32_t next_line = (index == error_list->size - 1) ? (((int32_t) newline_list->size) + parser->start_line) : errors[index + 1].line; if (next_line - last_line > 1) { pm_buffer_append_string(buffer, " ", 2); diff --git a/prism/prism.h b/prism/prism.h index 34540b9441954a..59067c3021a70a 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -26,6 +26,7 @@ #include #include +#include #include #include #include @@ -222,10 +223,12 @@ const char * pm_token_type_human(pm_token_type_t token_type); * Format the errors on the parser into the given buffer. * * @param parser The parser to format the errors for. + * @param error_list The list of errors to format. * @param buffer The buffer to format the errors into. * @param colorize Whether or not to colorize the errors with ANSI escape sequences. + * @param inline_messages Whether or not to inline the messages with the source. */ -PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize); +PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, const pm_list_t *error_list, pm_buffer_t *buffer, bool colorize, bool inline_messages); // We optionally support dumping to JSON. For systems that don't want or need // this functionality, it can be turned off with the PRISM_EXCLUDE_JSON define. diff --git a/prism/static_literals.c b/prism/static_literals.c index 1708e31a1674de..469bdfd5ea2bd4 100644 --- a/prism/static_literals.c +++ b/prism/static_literals.c @@ -95,13 +95,17 @@ node_hash(const pm_parser_t *parser, const pm_node_t *node) { // Strings hash their value and mix in their flags so that different // encodings are not considered equal. const pm_string_t *value = &((const pm_string_node_t *) node)->unescaped; - return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)) ^ murmur_scramble((uint32_t) node->flags); + + pm_node_flags_t flags = node->flags; + flags &= (PM_STRING_FLAGS_FORCED_BINARY_ENCODING | PM_STRING_FLAGS_FORCED_UTF8_ENCODING); + + return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)) ^ murmur_scramble((uint32_t) flags); } case PM_SOURCE_FILE_NODE: { // Source files hash their value and mix in their flags so that // different encodings are not considered equal. const pm_string_t *value = &((const pm_source_file_node_t *) node)->filepath; - return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)) ^ murmur_scramble((uint32_t) node->flags); + return murmur_hash(pm_string_source(value), pm_string_length(value) * sizeof(uint8_t)); } case PM_REGULAR_EXPRESSION_NODE: { // Regular expressions hash their value and mix in their flags so @@ -316,8 +320,6 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_parser_t *pa */ pm_node_t * pm_static_literals_add(const pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *node) { - if (!PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)) return NULL; - switch (PM_NODE_TYPE(node)) { case PM_INTEGER_NODE: case PM_SOURCE_LINE_NODE: @@ -435,8 +437,6 @@ pm_rational_inspect(pm_buffer_t *buffer, pm_rational_node_t *node) { */ PRISM_EXPORTED_FUNCTION void pm_static_literal_inspect(pm_buffer_t *buffer, const pm_parser_t *parser, const pm_node_t *node) { - assert(PM_NODE_FLAG_P(node, PM_NODE_FLAG_STATIC_LITERAL)); - switch (PM_NODE_TYPE(node)) { case PM_FALSE_NODE: pm_buffer_append_string(buffer, "false", 5); diff --git a/prism/templates/include/prism/diagnostic.h.erb b/prism/templates/include/prism/diagnostic.h.erb index 53a700d409397d..07bbc8fae79264 100644 --- a/prism/templates/include/prism/diagnostic.h.erb +++ b/prism/templates/include/prism/diagnostic.h.erb @@ -66,11 +66,14 @@ typedef struct { * The levels of errors generated during parsing. */ typedef enum { - /** For errors that cannot be recovered from. */ - PM_ERROR_LEVEL_FATAL = 0, + /** For errors that should raise a syntax error. */ + PM_ERROR_LEVEL_SYNTAX = 0, /** For errors that should raise an argument error. */ - PM_ERROR_LEVEL_ARGUMENT = 1 + PM_ERROR_LEVEL_ARGUMENT = 1, + + /** For errors that should raise a load error. */ + PM_ERROR_LEVEL_LOAD = 2 } pm_error_level_t; /** diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 1a723f64afcd3f..ac7ab4fff3c65d 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -131,8 +131,8 @@ module Prism comments = load_comments magic_comments = Array.new(load_varuint) { MagicComment.new(load_location_object, load_location_object) } data_loc = load_optional_location_object - errors = Array.new(load_varuint) { ParseError.new(DIAGNOSTIC_TYPES[io.getbyte], load_embedded_string, load_location_object, load_error_level) } - warnings = Array.new(load_varuint) { ParseWarning.new(DIAGNOSTIC_TYPES[io.getbyte], load_embedded_string, load_location_object, load_warning_level) } + errors = Array.new(load_varuint) { ParseError.new(DIAGNOSTIC_TYPES[load_varuint], load_embedded_string, load_location_object, load_error_level) } + warnings = Array.new(load_varuint) { ParseWarning.new(DIAGNOSTIC_TYPES[load_varuint], load_embedded_string, load_location_object, load_warning_level) } [comments, magic_comments, data_loc, errors, warnings] end @@ -296,9 +296,11 @@ module Prism case level when 0 - :fatal + :syntax when 1 :argument + when 2 + :load else raise "Unknown level: #{level}" end diff --git a/prism/templates/src/diagnostic.c.erb b/prism/templates/src/diagnostic.c.erb index 818b12d98be8c1..3d5073147a9ffb 100644 --- a/prism/templates/src/diagnostic.c.erb +++ b/prism/templates/src/diagnostic.c.erb @@ -65,8 +65,9 @@ typedef struct { * * For errors, they are: * - * * `PM_ERROR_LEVEL_FATAL` - The default level for errors. + * * `PM_ERROR_LEVEL_SYNTAX` - Errors that should raise SyntaxError. * * `PM_ERROR_LEVEL_ARGUMENT` - Errors that should raise ArgumentError. + * * `PM_ERROR_LEVEL_LOAD` - Errors that should raise LoadError. * * For warnings, they are: * @@ -75,242 +76,251 @@ typedef struct { */ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { // Special error that can be replaced - [PM_ERR_CANNOT_PARSE_EXPRESSION] = { "cannot parse the expression", PM_ERROR_LEVEL_FATAL }, + [PM_ERR_CANNOT_PARSE_EXPRESSION] = { "cannot parse the expression", PM_ERROR_LEVEL_SYNTAX }, // Errors that should raise argument errors [PM_ERR_INVALID_ENCODING_MAGIC_COMMENT] = { "unknown or invalid encoding in the magic comment", PM_ERROR_LEVEL_ARGUMENT }, + // Errors that should raise load errors + [PM_ERR_SCRIPT_NOT_FOUND] = { "no Ruby script found in input", PM_ERROR_LEVEL_LOAD }, + // Errors that should raise syntax errors - [PM_ERR_ALIAS_ARGUMENT] = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "multiple block arguments; only one block is allowed", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_CLASS] = { "invalid formal argument; formal argument cannot be a class variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = { "invalid formal argument; formal argument cannot be a constant", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = { "unexpected `&` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected `...` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR] = { "unexpected `**` when the parent method is not forwarding", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = { "unexpected `*` splat argument after a `*` splat argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_TERM_PAREN] = { "expected a `)` to close the arguments", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = { "unexpected `{` after a method call without parenthesis", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_ELEMENT] = { "expected an element for the array", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_EXPRESSION] = { "expected an expression for the array element", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = { "expected an expression after `*` in the array", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_SEPARATOR] = { "expected a `,` separator for the array elements", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ARRAY_TERM] = { "expected a `]` to close the array", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_LONELY_ELSE] = { "unexpected `else` in `begin` block; else without rescue is useless", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_TERM] = { "expected an `end` to close the `begin` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_UPCASE_BRACE] = { "expected a `{` after `BEGIN`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_UPCASE_TERM] = { "expected a `}` to close the `BEGIN` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BEGIN_UPCASE_TOPLEVEL] = { "BEGIN is permitted only at toplevel", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE] = { "expected a local variable name in the block parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_PARAM_PIPE_TERM] = { "expected the block parameters to end with `|`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_TERM_BRACE] = { "expected a block beginning with `{` to end with `}`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_BLOCK_TERM_END] = { "expected a block beginning with `do` to end with `end`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CANNOT_PARSE_STRING_PART] = { "cannot parse the string part", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_EXPRESSION_AFTER_CASE] = { "expected an expression after `case`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_EXPRESSION_AFTER_WHEN] = { "expected an expression after `when`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = { "expected a predicate for a case matching statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_MISSING_CONDITIONS] = { "expected a `when` or `in` clause after `case`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CASE_TERM] = { "expected an `end` to close the `case` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in a method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_NAME] = { "expected a constant name after `class`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CLASS_UNEXPECTED_END] = { "unexpected `end`, expecting ';' or '\\n'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = { "expected a predicate expression for the `elsif` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_IF_PREDICATE] = { "expected a predicate expression for the `if` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_PREDICATE_TERM] = { "expected `then` or `;` or '\\n'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_TERM] = { "expected an `end` to close the conditional clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_TERM_ELSE] = { "expected an `end` to close the `else` clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_UNLESS_PREDICATE] = { "expected a predicate expression for the `unless` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_UNTIL_PREDICATE] = { "expected a predicate expression for the `until` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = { "expected a predicate expression for the `while` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = { "expected a constant after the `::` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_ENDLESS] = { "could not parse the endless method body", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_NAME] = { "expected a method name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_NAME_AFTER_RECEIVER] = { "expected a method name after the receiver", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "expected a `)` to close the parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_RECEIVER] = { "expected a receiver for the method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_RECEIVER_TERM] = { "expected a `.` or `::` after the receiver in a method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEF_TERM] = { "expected an `end` to close the `def` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_DEFINED_EXPRESSION] = { "expected an expression after `defined?`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EMBDOC_TERM] = { "could not find a terminator for the embedded document", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EMBEXPR_END] = { "expected a `}` to close the embedded expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EMBVAR_INVALID] = { "invalid embedded variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_END_UPCASE_BRACE] = { "expected a `{` after `END`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_END_UPCASE_TERM] = { "expected a `}` to close the `END` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_CONTROL] = { "invalid control escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = { "invalid control escape sequence; control cannot be repeated", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hexadecimal escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_META] = { "invalid meta escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_META_REPEAT] = { "invalid meta escape sequence; meta cannot be repeated", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE] = { "invalid Unicode escape sequence", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = { "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = { "invalid Unicode escape sequence; maximum length is 6 digits", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "invalid Unicode escape sequence; needs closing `}`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_ARGUMENT] = { "expected an argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = { "unexpected %s, expecting end-of-input", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = { "expected an expression after `&&=`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = { "expected an expression after `||=`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA] = { "expected an expression after `,`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = { "expected an expression after `=`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = { "expected an expression after `<<`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = { "expected an expression after `(`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "expected an expression after the operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = { "expected an expression after `*` splat in an argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = { "expected an expression after `**` in a hash", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = { "expected an expression after `*`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RPAREN] = { "expected a matching `)`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RPAREN_AFTER_MULTI] = { "expected a `)` after multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_RPAREN_REQ_PARAMETER] = { "expected a `)` to end a required parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_STRING_CONTENT] = { "expected string content after opening string delimiter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPECT_WHEN_DELIMITER] = { "expected a delimiter after the predicates of a `when` clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_EXPRESSION_BARE_HASH] = { "unexpected bare hash in expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FLOAT_PARSE] = { "could not parse the float '%.*s'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_COLLECTION] = { "expected a collection after the `in` in a `for` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_INDEX] = { "expected an index after `for`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_IN] = { "expected an `in` after the index in a `for` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_FOR_TERM] = { "expected an `end` to close the `for` loop", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = { "expected an expression after the label in a hash", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_TERM] = { "expected a `}` to close the hash literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HASH_VALUE] = { "expected a value in the hash literal", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_HEREDOC_TERM] = { "could not find a terminator for the heredoc", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_QUESTION_MARK] = { "incomplete expression at `?`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_BINARY] = { "invalid binary number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_DECIMAL] = { "invalid decimal number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = { "invalid hexadecimal number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_OCTAL] = { "invalid octal number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_NUMBER_UNDERSCORE] = { "invalid underscore placement in number", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_CHARACTER] = { "invalid character 0x%X", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_MULTIBYTE_CHAR] = { "invalid multibyte char (%s)", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_MULTIBYTE_CHARACTER] = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_MULTIBYTE_ESCAPE] = { "invalid multibyte escape: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_PRINTABLE_CHARACTER] = { "invalid character `%c`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_PERCENT] = { "invalid `%` token", PM_ERROR_LEVEL_FATAL }, // TODO WHAT? - [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_INVALID_VARIABLE_GLOBAL] = { "'%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when an numbered parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_IT_NOT_ALLOWED_ORDINARY] = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LAMBDA_OPEN] = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LAMBDA_TERM_BRACE] = { "expected a lambda block beginning with `{` to end with `}`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LAMBDA_TERM_END] = { "expected a lambda block beginning with `do` to end with `end`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_LOWER_ELEMENT] = { "expected a symbol in a `%i` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_LOWER_TERM] = { "expected a closing delimiter for the `%i` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_UPPER_ELEMENT] = { "expected a symbol in a `%I` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_I_UPPER_TERM] = { "expected a closing delimiter for the `%I` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_LOWER_ELEMENT] = { "expected a string in a `%w` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_LOWER_TERM] = { "expected a closing delimiter for the `%w` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_UPPER_ELEMENT] = { "expected a string in a `%W` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_LIST_W_UPPER_TERM] = { "expected a closing delimiter for the `%W` list", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in a method definition", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MODULE_NAME] = { "expected a constant name after `module`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MODULE_TERM] = { "expected an `end` to close the `module` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = { "multiple splats in multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST] = { "unexpected '%.*s' resulting in multiple splats in multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NOT_EXPRESSION] = { "expected an expression after `not`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NO_LOCAL_VARIABLE] = { "%.*s: no such local variable", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = { "number literal ending with a `_`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when an 'it' parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBERED_PARAMETER_ORDINARY] = { "numbered parameters are not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = { "numbered parameter is already used in outer scope", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_OPERATOR_MULTI_ASSIGN] = { "unexpected operator for a multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = { "unexpected operator after a call with arguments", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_OPERATOR_WRITE_BLOCK] = { "unexpected operator after a call with a block", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = { "unexpected multiple `**` splat parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_CIRCULAR] = { "parameter default value references itself", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NAME_REPEAT] = { "repeated parameter name", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NO_DEFAULT_KW] = { "expected a default value for the keyword parameter", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_NUMBERED_RESERVED] = { "%.2s is reserved for numbered parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_ORDER] = { "unexpected parameter order", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_SPLAT_MULTI] = { "unexpected multiple `*` splat parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_STAR] = { "unexpected parameter `*`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_UNEXPECTED_FWD] = { "unexpected `...` in parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = { "unexpected `,` in parameters", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = { "expected a pattern expression after `=>`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_IN] = { "expected a pattern expression after the `in` keyword", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_KEY] = { "expected a pattern expression after the key", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN] = { "expected a pattern expression after the `(` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_PIN] = { "expected a pattern expression after the `^` pin operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = { "expected a pattern expression after the `|` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_FATAL }, // TODO // THIS // AND // ABOVE // IS WEIRD - [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_REST] = { "unexpected rest pattern", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_TERM_BRACE] = { "expected a `}` to close the pattern expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_TERM_BRACKET] = { "expected a `]` to close the pattern expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PATTERN_TERM_PAREN] = { "expected a `)` to close the pattern expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = { "unexpected `||=` in a multiple assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH] = { "regexp encoding option '%c' differs from source encoding '%s'", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_TERM] = { "expected a closing delimiter for the regular expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_MODIFIER_VALUE] = { "expected a value after the `rescue` modifier", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_TERM] = { "expected a closing delimiter for the `rescue` clause", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RESCUE_VARIABLE] = { "expected an exception variable after `=>` in a rescue statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_RETURN_INVALID] = { "invalid `return` in a class or module body", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_SINGLETON_FOR_LITERALS] = { "cannot define singleton method for literals", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_ALIAS] = { "unexpected an `alias` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_POSTEXE_END] = { "unexpected an `END` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_PREEXE_BEGIN] = { "unexpected a `BEGIN` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STATEMENT_UNDEF] = { "unexpected an `undef` at a non-statement position", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_CONCATENATION] = { "expected a string for concatenation", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_INTERPOLATED_TERM] = { "expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_FATAL }, // TODO expected symbol? prism.c ~9719 - [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_TERNARY_EXPRESSION_FALSE] = { "expected an expression after `:` in the ternary operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_TERNARY_EXPRESSION_TRUE] = { "expected an expression after `?` in the ternary operator", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNDEF_ARGUMENT] = { "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNARY_RECEIVER] = { "unexpected %s, expected a receiver for unary `%c`", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNEXPECTED_TOKEN_IGNORE] = { "unexpected %s, ignoring it", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_UNTIL_TERM] = { "expected an `end` to close the `until` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_VOID_EXPRESSION] = { "unexpected void value expression", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WHILE_TERM] = { "expected an `end` to close the `while` statement", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WRITE_TARGET_IN_METHOD] = { "dynamic constant assignment", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WRITE_TARGET_READONLY] = { "Can't set variable %.*s", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_WRITE_TARGET_UNEXPECTED] = { "unexpected write target", PM_ERROR_LEVEL_FATAL }, - [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_FATAL }, + [PM_ERR_ALIAS_ARGUMENT] = { "invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ALIAS_ARGUMENT_NUMBERED_REFERENCE] = { "invalid argument being passed to `alias`; can't make alias for the number variables", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_AMPAMPEQ_MULTI_ASSIGN] = { "unexpected `&&=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_AFTER_BLOCK] = { "unexpected argument after a block argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES] = { "unexpected argument after `...`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BARE_HASH] = { "unexpected bare hash argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BLOCK_FORWARDING] = { "both a block argument and a forwarding argument; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_BLOCK_MULTI] = { "both block arg and actual block given; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_CLASS] = { "invalid formal argument; formal argument cannot be a class variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_CONSTANT] = { "invalid formal argument; formal argument cannot be a constant", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_GLOBAL] = { "invalid formal argument; formal argument cannot be a global variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORMAL_IVAR] = { "invalid formal argument; formal argument cannot be an instance variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_FORWARDING_UNBOUND] = { "unexpected `...` in an non-parenthesized call", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_IN] = { "unexpected `in` keyword in arguments", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_AMP] = { "unexpected `&` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES] = { "unexpected `...` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_STAR] = { "unexpected `*` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_NO_FORWARDING_STAR_STAR] = { "unexpected `**` when the parent method is not forwarding", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT] = { "unexpected `*` splat argument after a `**` keyword splat argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_SPLAT_AFTER_SPLAT] = { "unexpected `*` splat argument after a `*` splat argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_TERM_PAREN] = { "expected a `)` to close the arguments", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARGUMENT_UNEXPECTED_BLOCK] = { "unexpected `{` after a method call without parenthesis", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_ELEMENT] = { "expected an element for the array", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_EXPRESSION] = { "expected an expression for the array element", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_EXPRESSION_AFTER_STAR] = { "expected an expression after `*` in the array", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_SEPARATOR] = { "expected a `,` separator for the array elements", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ARRAY_TERM] = { "expected a `]` to close the array", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_LONELY_ELSE] = { "unexpected `else` in `begin` block; else without rescue is useless", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_TERM] = { "expected an `end` to close the `begin` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_UPCASE_BRACE] = { "expected a `{` after `BEGIN`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_UPCASE_TERM] = { "expected a `}` to close the `BEGIN` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BEGIN_UPCASE_TOPLEVEL] = { "BEGIN is permitted only at toplevel", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_PARAM_LOCAL_VARIABLE] = { "expected a local variable name in the block parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_PARAM_PIPE_TERM] = { "expected the block parameters to end with `|`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_TERM_BRACE] = { "expected a block beginning with `{` to end with `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_BLOCK_TERM_END] = { "expected a block beginning with `do` to end with `end`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CANNOT_PARSE_STRING_PART] = { "cannot parse the string part", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_EXPRESSION_AFTER_CASE] = { "expected an expression after `case`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_EXPRESSION_AFTER_WHEN] = { "expected an expression after `when`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_MATCH_MISSING_PREDICATE] = { "expected a predicate for a case matching statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_MISSING_CONDITIONS] = { "expected a `when` or `in` clause after `case`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CASE_TERM] = { "expected an `end` to close the `case` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_IN_METHOD] = { "unexpected class definition in method body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_NAME] = { "unexpected constant path after `class`; class/module name must be CONSTANT", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_SUPERCLASS] = { "expected a superclass after `<`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_TERM] = { "expected an `end` to close the `class` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_UNEXPECTED_END] = { "unexpected `end`, expecting ';' or '\\n'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CLASS_VARIABLE_BARE] = { "'@@' without identifiers is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_ELSIF_PREDICATE] = { "expected a predicate expression for the `elsif` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_IF_PREDICATE] = { "expected a predicate expression for the `if` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_PREDICATE_TERM] = { "expected `then` or `;` or '\\n'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_TERM] = { "expected an `end` to close the conditional clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_TERM_ELSE] = { "expected an `end` to close the `else` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_UNLESS_PREDICATE] = { "expected a predicate expression for the `unless` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_UNTIL_PREDICATE] = { "expected a predicate expression for the `until` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONDITIONAL_WHILE_PREDICATE] = { "expected a predicate expression for the `while` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = { "expected a constant after the `::` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_ENDLESS] = { "could not parse the endless method body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_ENDLESS_SETTER] = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_NAME] = { "expected a method name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_NAME_AFTER_RECEIVER] = { "expected a method name after the receiver", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_PARAMS_TERM] = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_PARAMS_TERM_PAREN] = { "expected a `)` to close the parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_RECEIVER] = { "expected a receiver for the method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_RECEIVER_TERM] = { "expected a `.` or `::` after the receiver in a method definition", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEF_TERM] = { "expected an `end` to close the `def` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_DEFINED_EXPRESSION] = { "expected an expression after `defined?`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EMBDOC_TERM] = { "could not find a terminator for the embedded document", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EMBEXPR_END] = { "expected a `}` to close the embedded expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EMBVAR_INVALID] = { "invalid embedded variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_END_UPCASE_BRACE] = { "expected a `{` after `END`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_END_UPCASE_TERM] = { "expected a `}` to close the `END` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_CONTROL] = { "invalid control escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_CONTROL_REPEAT] = { "invalid control escape sequence; control cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_HEXADECIMAL] = { "invalid hexadecimal escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_META] = { "invalid meta escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_META_REPEAT] = { "invalid meta escape sequence; meta cannot be repeated", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE] = { "invalid Unicode escape sequence", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS] = { "invalid Unicode escape sequence; Unicode cannot be combined with control or meta flags", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_LITERAL] = { "invalid Unicode escape sequence; multiple codepoints are not allowed in a character literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_LONG] = { "invalid Unicode escape sequence; maximum length is 6 digits", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_ESCAPE_INVALID_UNICODE_TERM] = { "invalid Unicode escape sequence; needs closing `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_ARGUMENT] = { "expected an argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EOL_AFTER_STATEMENT] = { "unexpected %s, expecting end-of-input", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ] = { "expected an expression after `&&=`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ] = { "expected an expression after `||=`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA] = { "expected an expression after `,`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_EQUAL] = { "expected an expression after `=`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS] = { "expected an expression after `<<`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_LPAREN] = { "expected an expression after `(`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR] = { "expected an expression after the operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT] = { "expected an expression after `*` splat in an argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH] = { "expected an expression after `**` in a hash", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_EXPRESSION_AFTER_STAR] = { "expected an expression after `*`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_IDENT_REQ_PARAMETER] = { "expected an identifier for the required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_LPAREN_REQ_PARAMETER] = { "expected a `(` to start a required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RBRACKET] = { "expected a matching `]`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RPAREN] = { "expected a matching `)`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RPAREN_AFTER_MULTI] = { "expected a `)` after multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_RPAREN_REQ_PARAMETER] = { "expected a `)` to end a required parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_STRING_CONTENT] = { "expected string content after opening string delimiter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPECT_WHEN_DELIMITER] = { "expected a delimiter after the predicates of a `when` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_EXPRESSION_BARE_HASH] = { "unexpected bare hash in expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FLOAT_PARSE] = { "could not parse the float '%.*s'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_COLLECTION] = { "expected a collection after the `in` in a `for` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_INDEX] = { "expected an index after `for`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_IN] = { "expected an `in` after the index in a `for` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_FOR_TERM] = { "expected an `end` to close the `for` loop", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_GLOBAL_VARIABLE_BARE] = { "'$' without identifiers is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_EXPRESSION_AFTER_LABEL] = { "expected an expression after the label in a hash", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_KEY] = { "unexpected %s, expecting '}' or a key in the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_ROCKET] = { "expected a `=>` between the hash key and value", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_TERM] = { "expected a `}` to close the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HASH_VALUE] = { "expected a value in the hash literal", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_HEREDOC_TERM] = { "could not find a terminator for the heredoc", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_QUESTION_MARK] = { "incomplete expression at `?`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_CLASS_3_3_0] = { "`%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_CLASS] = { "'%.*s' is not allowed as a class variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE_3_3_0] = { "`%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INCOMPLETE_VARIABLE_INSTANCE] = { "'%.*s' is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INSTANCE_VARIABLE_BARE] = { "'@' without identifiers is not allowed as an instance variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_FLOAT_EXPONENT] = { "invalid exponent", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_BINARY] = { "invalid binary number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_DECIMAL] = { "invalid decimal number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_HEXADECIMAL] = { "invalid hexadecimal number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_OCTAL] = { "invalid octal number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_NUMBER_UNDERSCORE] = { "invalid underscore placement in number", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_CHARACTER] = { "invalid character 0x%X", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_MULTIBYTE_CHAR] = { "invalid multibyte char (%s)", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_MULTIBYTE_CHARACTER] = { "invalid multibyte character 0x%X", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_MULTIBYTE_ESCAPE] = { "invalid multibyte escape: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_PRINTABLE_CHARACTER] = { "invalid character `%c`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_PERCENT] = { "invalid `%` token", PM_ERROR_LEVEL_SYNTAX }, // TODO WHAT? + [PM_ERR_INVALID_VARIABLE_GLOBAL_3_3_0] = { "`%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_INVALID_VARIABLE_GLOBAL] = { "'%.*s' is not allowed as a global variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_IT_NOT_ALLOWED_NUMBERED] = { "`it` is not allowed when an numbered parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_IT_NOT_ALLOWED_ORDINARY] = { "`it` is not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LAMBDA_OPEN] = { "expected a `do` keyword or a `{` to open the lambda block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LAMBDA_TERM_BRACE] = { "expected a lambda block beginning with `{` to end with `}`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LAMBDA_TERM_END] = { "expected a lambda block beginning with `do` to end with `end`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_LOWER_ELEMENT] = { "expected a symbol in a `%i` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_LOWER_TERM] = { "expected a closing delimiter for the `%i` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_UPPER_ELEMENT] = { "expected a symbol in a `%I` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_I_UPPER_TERM] = { "expected a closing delimiter for the `%I` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_LOWER_ELEMENT] = { "expected a string in a `%w` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_LOWER_TERM] = { "expected a closing delimiter for the `%w` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_UPPER_ELEMENT] = { "expected a string in a `%W` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_LIST_W_UPPER_TERM] = { "expected a closing delimiter for the `%W` list", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MALLOC_FAILED] = { "failed to allocate memory", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MIXED_ENCODING] = { "UTF-8 mixed within %s source", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_IN_METHOD] = { "unexpected module definition in method body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_NAME] = { "unexpected constant path after `module`; class/module name must be CONSTANT", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MODULE_TERM] = { "expected an `end` to close the `module` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS] = { "multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST] = { "unexpected '%.*s' resulting in multiple splats in multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NOT_EXPRESSION] = { "expected an expression after `not`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NO_LOCAL_VARIABLE] = { "%.*s: no such local variable", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBER_LITERAL_UNDERSCORE] = { "number literal ending with a `_`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_IT] = { "numbered parameters are not allowed when an 'it' parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_ORDINARY] = { "numbered parameters are not allowed when an ordinary parameter is defined", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE] = { "numbered parameter is already used in outer scope", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_OPERATOR_MULTI_ASSIGN] = { "unexpected operator for a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_OPERATOR_WRITE_ARGUMENTS] = { "unexpected operator after a call with arguments", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_OPERATOR_WRITE_BLOCK] = { "unexpected operator after a call with a block", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_ASSOC_SPLAT_MULTI] = { "unexpected multiple `**` splat parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_BLOCK_MULTI] = { "multiple block parameters; only one block is allowed", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_CIRCULAR] = { "parameter default value references itself", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_METHOD_NAME] = { "unexpected name for a parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NAME_DUPLICATED] = { "duplicated argument name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NO_DEFAULT] = { "expected a default value for the parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NO_DEFAULT_KW] = { "expected a default value for the keyword parameter", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_NUMBERED_RESERVED] = { "%.2s is reserved for numbered parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_ORDER] = { "unexpected parameter order", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_SPLAT_MULTI] = { "unexpected multiple `*` splat parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_STAR] = { "unexpected parameter `*`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_UNEXPECTED_FWD] = { "unexpected `...` in parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PARAMETER_WILD_LOOSE_COMMA] = { "unexpected `,` in parameters", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_CAPTURE_DUPLICATE] = { "duplicated variable name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_BRACKET] = { "expected a pattern expression after the `[` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_COMMA] = { "expected a pattern expression after `,`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_HROCKET] = { "expected a pattern expression after `=>`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_IN] = { "expected a pattern expression after the `in` keyword", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_KEY] = { "expected a pattern expression after the key", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_PAREN] = { "expected a pattern expression after the `(` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_PIN] = { "expected a pattern expression after the `^` pin operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_PIPE] = { "expected a pattern expression after the `|` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_RANGE] = { "expected a pattern expression after the range operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_EXPRESSION_AFTER_REST] = { "unexpected pattern expression after the `**` expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY] = { "expected a key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY_DUPLICATE] = { "duplicated key name", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_HASH_KEY_LABEL] = { "expected a label as the key in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, // TODO // THIS // AND // ABOVE // IS WEIRD + [PM_ERR_PATTERN_IDENT_AFTER_HROCKET] = { "expected an identifier after the `=>` operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_LABEL_AFTER_COMMA] = { "expected a label after the `,` in the hash pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_REST] = { "unexpected rest pattern", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_TERM_BRACE] = { "expected a `}` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_TERM_BRACKET] = { "expected a `]` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PATTERN_TERM_PAREN] = { "expected a `)` to close the pattern expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_PIPEPIPEEQ_MULTI_ASSIGN] = { "unexpected `||=` in a multiple assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH] = { "regexp encoding option '%c' differs from source encoding '%s'", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING] = { "incompatible character encoding: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_NON_ESCAPED_MBC] = { "/.../n has a non escaped non ASCII character in non ASCII-8BIT script: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_INVALID_UNICODE_RANGE] = { "invalid Unicode range: /%.*s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_UNKNOWN_OPTIONS] = { "unknown regexp %s: %.*s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_TERM] = { "expected a closing delimiter for the regular expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP] = { "UTF-8 character in non UTF-8 regexp: /%s/", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_EXPRESSION] = { "expected a rescued expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_MODIFIER_VALUE] = { "expected a value after the `rescue` modifier", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_TERM] = { "expected a closing delimiter for the `rescue` clause", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RESCUE_VARIABLE] = { "expected an exception variable after `=>` in a rescue statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_RETURN_INVALID] = { "invalid `return` in a class or module body", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SINGLETON_FOR_LITERALS] = { "cannot define singleton method for literals", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_ALIAS] = { "unexpected an `alias` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_POSTEXE_END] = { "unexpected an `END` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_PREEXE_BEGIN] = { "unexpected a `BEGIN` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STATEMENT_UNDEF] = { "unexpected an `undef` at a non-statement position", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_CONCATENATION] = { "expected a string for concatenation", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_INTERPOLATED_TERM] = { "expected a closing delimiter for the interpolated string", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_LITERAL_EOF] = { "unterminated string meets end of file", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_STRING_LITERAL_TERM] = { "unexpected %s, expected a string literal terminator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_INVALID] = { "invalid symbol", PM_ERROR_LEVEL_SYNTAX }, // TODO expected symbol? prism.c ~9719 + [PM_ERR_SYMBOL_TERM_DYNAMIC] = { "expected a closing delimiter for the dynamic symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_SYMBOL_TERM_INTERPOLATED] = { "expected a closing delimiter for the interpolated symbol", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_TERNARY_COLON] = { "expected a `:` after the true expression of a ternary operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_TERNARY_EXPRESSION_FALSE] = { "expected an expression after `:` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_TERNARY_EXPRESSION_TRUE] = { "expected an expression after `?` in the ternary operator", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNDEF_ARGUMENT] = { "invalid argument being passed to `undef`; expected a bare word, constant, or symbol argument", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNARY_RECEIVER] = { "unexpected %s, expected a receiver for unary `%c`", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT] = { "unexpected %s, assuming it is closing the parent %s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNEXPECTED_TOKEN_IGNORE] = { "unexpected %s, ignoring it", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_UNTIL_TERM] = { "expected an `end` to close the `until` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_VOID_EXPRESSION] = { "unexpected void value expression", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WHILE_TERM] = { "expected an `end` to close the `while` statement", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WRITE_TARGET_IN_METHOD] = { "dynamic constant assignment", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WRITE_TARGET_READONLY] = { "Can't set variable %.*s", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_WRITE_TARGET_UNEXPECTED] = { "unexpected write target", PM_ERROR_LEVEL_SYNTAX }, + [PM_ERR_XSTRING_TERM] = { "expected a closing delimiter for the `%x` or backtick string", PM_ERROR_LEVEL_SYNTAX }, // Warnings [PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS] = { "ambiguous first argument; put parentheses or a space even after `-` operator", PM_WARNING_LEVEL_VERBOSE }, @@ -335,6 +345,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = { [PM_WARN_KEYWORD_EOL] = { "`%.*s` at the end of line without an expression", PM_WARNING_LEVEL_VERBOSE }, [PM_WARN_LITERAL_IN_CONDITION_DEFAULT] = { "%sliteral in %s", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_LITERAL_IN_CONDITION_VERBOSE] = { "%sliteral in %s", PM_WARNING_LEVEL_VERBOSE }, + [PM_WARN_SHEBANG_CARRIAGE_RETURN] = { "shebang line ending with \\r may cause problems", PM_WARNING_LEVEL_DEFAULT }, [PM_WARN_UNEXPECTED_CARRIAGE_RETURN] = { "encountered \\r in middle of line, treated as a mere space", PM_WARNING_LEVEL_DEFAULT } }; @@ -378,7 +389,7 @@ pm_diagnostic_level(pm_diagnostic_id_t diag_id) { */ bool pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) { - pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(sizeof(pm_diagnostic_t), 1); + pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); if (diagnostic == NULL) return false; *diagnostic = (pm_diagnostic_t) { @@ -410,7 +421,7 @@ pm_diagnostic_list_append_format(pm_list_t *list, const uint8_t *start, const ui return false; } - pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(sizeof(pm_diagnostic_t), 1); + pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) xcalloc(1, sizeof(pm_diagnostic_t)); if (diagnostic == NULL) { return false; } diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb index 94b976645dd1d1..97101e36d52d91 100644 --- a/prism/templates/src/serialize.c.erb +++ b/prism/templates/src/serialize.c.erb @@ -220,7 +220,7 @@ pm_serialize_data_loc(const pm_parser_t *parser, pm_buffer_t *buffer) { static void pm_serialize_diagnostic(pm_parser_t *parser, pm_diagnostic_t *diagnostic, pm_buffer_t *buffer) { // serialize the type - pm_buffer_append_byte(buffer, (uint8_t) diagnostic->diag_id); + pm_buffer_append_varuint(buffer, (uint32_t) diagnostic->diag_id); // serialize message size_t message_length = strlen(diagnostic->message); diff --git a/prism/util/pm_newline_list.c b/prism/util/pm_newline_list.c index f9dff4c1666187..ce07ce8c8e4644 100644 --- a/prism/util/pm_newline_list.c +++ b/prism/util/pm_newline_list.c @@ -19,6 +19,14 @@ pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capac return true; } +/** + * Clear out the newlines that have been appended to the list. + */ +void +pm_newline_list_clear(pm_newline_list_t *list) { + list->size = 1; +} + /** * Append a new offset to the newline list. Returns true if the reallocation of * the offsets succeeds (if one was necessary), otherwise returns false. diff --git a/prism/util/pm_newline_list.h b/prism/util/pm_newline_list.h index 612ee35d3f8fd6..7ae9b6b3da0ac5 100644 --- a/prism/util/pm_newline_list.h +++ b/prism/util/pm_newline_list.h @@ -61,6 +61,14 @@ typedef struct { */ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity); +/** + * Clear out the newlines that have been appended to the list. + * + * @param list The list to clear. + */ +void +pm_newline_list_clear(pm_newline_list_t *list); + /** * Append a new offset to the newline list. Returns true if the reallocation of * the offsets succeeds (if one was necessary), otherwise returns false. diff --git a/prism_compile.c b/prism_compile.c index aec2f80f4102dc..5aeef191f780a7 100644 --- a/prism_compile.c +++ b/prism_compile.c @@ -528,6 +528,38 @@ pm_compile_regexp_dynamic(rb_iseq_t *iseq, const pm_node_t *node, const pm_node_ PUSH_INSN2(ret, *node_location, toregexp, INT2FIX(parse_regexp_flags(node) & 0xFF), INT2FIX(length)); } +static VALUE +pm_source_file_value(const pm_source_file_node_t *node, const pm_scope_node_t *scope_node) +{ + const pm_string_t *filepath = &node->filepath; + size_t length = pm_string_length(filepath); + + if (length > 0) { + rb_encoding *filepath_encoding = scope_node->filepath_encoding != NULL ? scope_node->filepath_encoding : rb_utf8_encoding(); + return rb_fstring(rb_enc_str_new((const char *) pm_string_source(filepath), length, filepath_encoding)); + } + else { + return rb_fstring_lit(""); + } +} + +/** + * Return a static literal string, optionally with attached debugging + * information. + */ +static VALUE +pm_static_literal_string(rb_iseq_t *iseq, VALUE string, int line_number) +{ + if (ISEQ_COMPILE_DATA(iseq)->option->debug_frozen_string_literal || RTEST(ruby_debug)) { + VALUE debug_info = rb_ary_new_from_args(2, rb_iseq_path(iseq), INT2FIX(line_number)); + rb_ivar_set(string, id_debug_created_info, rb_obj_freeze(debug_info)); + return rb_str_freeze(string); + } + else { + return rb_fstring(string); + } +} + /** * Certain nodes can be compiled literally. This function returns the literal * value described by the given node. For example, an array node with all static @@ -588,8 +620,11 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_n const pm_interpolated_regular_expression_node_t *cast = (const pm_interpolated_regular_expression_node_t *) node; return parse_regexp_concat(iseq, scope_node, (const pm_node_t *) cast, &cast->parts); } - case PM_INTERPOLATED_STRING_NODE: - return pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, true); + case PM_INTERPOLATED_STRING_NODE: { + VALUE string = pm_static_literal_concat(&((const pm_interpolated_string_node_t *) node)->parts, scope_node, false); + int line_number = pm_node_line_number(scope_node->parser, node); + return pm_static_literal_string(iseq, string, line_number); + } case PM_INTERPOLATED_SYMBOL_NODE: { const pm_interpolated_symbol_node_t *cast = (const pm_interpolated_symbol_node_t *) node; VALUE string = pm_static_literal_concat(&cast->parts, scope_node, true); @@ -612,19 +647,15 @@ pm_static_literal_value(rb_iseq_t *iseq, const pm_node_t *node, const pm_scope_n return rb_enc_from_encoding(scope_node->encoding); case PM_SOURCE_FILE_NODE: { const pm_source_file_node_t *cast = (const pm_source_file_node_t *) node; - size_t length = pm_string_length(&cast->filepath); - - if (length > 0) { - return rb_enc_str_new((const char *) pm_string_source(&cast->filepath), length, scope_node->encoding); - } - else { - return rb_fstring_lit(""); - } + return pm_source_file_value(cast, scope_node); } case PM_SOURCE_LINE_NODE: return INT2FIX(pm_node_line_number(scope_node->parser, node)); - case PM_STRING_NODE: - return rb_fstring(parse_string_encoded(scope_node, node, &((pm_string_node_t *)node)->unescaped)); + case PM_STRING_NODE: { + VALUE string = parse_string_encoded(scope_node, node, &((const pm_string_node_t *) node)->unescaped); + int line_number = pm_node_line_number(scope_node->parser, node); + return pm_static_literal_string(iseq, string, line_number); + } case PM_SYMBOL_NODE: return ID2SYM(parse_string_symbol(scope_node, (const pm_symbol_node_t *) node)); case PM_TRUE_NODE: @@ -2669,6 +2700,7 @@ pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_ if (previous) { scope->parser = previous->parser; scope->encoding = previous->encoding; + scope->filepath_encoding = previous->filepath_encoding; scope->constants = previous->constants; } @@ -4299,7 +4331,10 @@ pm_compile_case_node_dispatch(rb_iseq_t *iseq, VALUE dispatch, const pm_node_t * break; case PM_STRING_NODE: { const pm_string_node_t *cast = (const pm_string_node_t *) node; - key = rb_fstring(parse_string_encoded(scope_node, node, &cast->unescaped)); + VALUE string = parse_string_encoded(scope_node, node, &cast->unescaped); + + int line_number = pm_node_line_number(scope_node->parser, node); + key = pm_static_literal_string(iseq, string, line_number); break; } default: @@ -6773,27 +6808,38 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^ const pm_pre_execution_node_t *cast = (const pm_pre_execution_node_t *) node; - DECL_ANCHOR(pre_ex); - INIT_ANCHOR(pre_ex); + LINK_ANCHOR *outer_pre = scope_node->pre_execution_anchor; + RUBY_ASSERT(outer_pre != NULL); + + // BEGIN{} nodes can be nested, so here we're going to do the same thing + // that we did for the top-level compilation where we create two + // anchors and then join them in the correct order into the resulting + // anchor. + DECL_ANCHOR(inner_pre); + INIT_ANCHOR(inner_pre); + scope_node->pre_execution_anchor = inner_pre; + + DECL_ANCHOR(inner_body); + INIT_ANCHOR(inner_body); if (cast->statements != NULL) { const pm_node_list_t *body = &cast->statements->body; + for (size_t index = 0; index < body->size; index++) { - pm_compile_node(iseq, body->nodes[index], pre_ex, true, scope_node); + pm_compile_node(iseq, body->nodes[index], inner_body, true, scope_node); } } if (!popped) { - PUSH_INSN(pre_ex, location, putnil); + PUSH_INSN(inner_body, location, putnil); } - pre_ex->last->next = ret->anchor.next; - ret->anchor.next = pre_ex->anchor.next; - ret->anchor.next->prev = pre_ex->anchor.next; - - if (ret->last == (LINK_ELEMENT *)ret) { - ret->last = pre_ex->last; - } + // Now that everything has been compiled, join both anchors together + // into the correct outer pre execution anchor, and reset the value so + // that subsequent BEGIN{} nodes can be compiled correctly. + ADD_SEQ(outer_pre, inner_pre); + ADD_SEQ(outer_pre, inner_body); + scope_node->pre_execution_anchor = outer_pre; return; } @@ -8074,7 +8120,7 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^^^^ if (!popped) { const pm_source_file_node_t *cast = (const pm_source_file_node_t *) node; - VALUE string = rb_fstring(parse_string(scope_node, &cast->filepath)); + VALUE string = pm_source_file_value(cast, scope_node); if (PM_NODE_FLAG_P(cast, PM_STRING_FLAGS_FROZEN)) { PUSH_INSN1(ret, location, putobject, string); @@ -8131,7 +8177,8 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, // ^^^^^ if (!popped) { const pm_string_node_t *cast = (const pm_string_node_t *) node; - VALUE value = rb_fstring(parse_string_encoded(scope_node, node, &cast->unescaped)); + VALUE value = parse_string_encoded(scope_node, node, &cast->unescaped); + value = pm_static_literal_string(iseq, value, location.line); if (PM_NODE_FLAG_P(node, PM_STRING_FLAGS_FROZEN)) { PUSH_INSN1(ret, location, putobject, value); @@ -8306,6 +8353,20 @@ pm_compile_node(rb_iseq_t *iseq, const pm_node_t *node, LINK_ANCHOR *const ret, } } +/** True if the given iseq can have pre execution blocks. */ +static inline bool +pm_iseq_pre_execution_p(rb_iseq_t *iseq) +{ + switch (ISEQ_BODY(iseq)->type) { + case ISEQ_TYPE_TOP: + case ISEQ_TYPE_EVAL: + case ISEQ_TYPE_MAIN: + return true; + default: + return false; + } +} + /** * This is the main entry-point into the prism compiler. It accepts the iseq * that it should be compiling instruction into and a pointer to the scope node @@ -8319,7 +8380,32 @@ pm_iseq_compile_node(rb_iseq_t *iseq, pm_scope_node_t *node) DECL_ANCHOR(ret); INIT_ANCHOR(ret); - pm_compile_node(iseq, (const pm_node_t *) node, ret, false, node); + if (pm_iseq_pre_execution_p(iseq)) { + // Because these ISEQs can have BEGIN{}, we're going to create two + // anchors to compile them, a "pre" and a "body". We'll mark the "pre" + // on the scope node so that when BEGIN{} is found, its contents will be + // added to the "pre" anchor. + DECL_ANCHOR(pre); + INIT_ANCHOR(pre); + node->pre_execution_anchor = pre; + + // Now we'll compile the body as normal. We won't compile directly into + // the "ret" anchor yet because we want to add the "pre" anchor to the + // beginning of the "ret" anchor first. + DECL_ANCHOR(body); + INIT_ANCHOR(body); + pm_compile_node(iseq, (const pm_node_t *) node, body, false, node); + + // Now we'll join both anchors together so that the content is in the + // correct order. + ADD_SEQ(ret, pre); + ADD_SEQ(ret, body); + } + else { + // In other circumstances, we can just compile the node directly into + // the "ret" anchor. + pm_compile_node(iseq, (const pm_node_t *) node, ret, false, node); + } CHECK(iseq_setup_insn(iseq, ret)); return iseq_setup(iseq, ret); @@ -8373,43 +8459,91 @@ pm_parse_process_error_utf8_p(const pm_parser_t *parser, const pm_location_t *lo static VALUE pm_parse_process_error(const pm_parse_result_t *result) { - const pm_diagnostic_t *head = (const pm_diagnostic_t *) result->parser.error_list.head; + const pm_parser_t *parser = &result->parser; + const pm_diagnostic_t *head = (const pm_diagnostic_t *) parser->error_list.head; bool valid_utf8 = true; + pm_buffer_t buffer = { 0 }; + const pm_string_t *filepath = &parser->filepath; + for (const pm_diagnostic_t *error = head; error != NULL; error = (const pm_diagnostic_t *) error->node.next) { - // Any errors with the level PM_ERROR_LEVEL_ARGUMENT effectively take - // over as the only argument that gets raised. This is to allow priority - // messages that should be handled before anything else. - if (error->level == PM_ERROR_LEVEL_ARGUMENT) { - return rb_exc_new(rb_eArgError, error->message, strlen(error->message)); - } + switch (error->level) { + case PM_ERROR_LEVEL_SYNTAX: + // It is implicitly assumed that the error messages will be + // encodeable as UTF-8. Because of this, we can't include source + // examples that contain invalid byte sequences. So if any source + // examples include invalid UTF-8 byte sequences, we will skip + // showing source examples entirely. + if (valid_utf8 && !pm_parse_process_error_utf8_p(parser, &error->location)) { + valid_utf8 = false; + } + break; + case PM_ERROR_LEVEL_ARGUMENT: { + // Any errors with the level PM_ERROR_LEVEL_ARGUMENT take over as + // the only argument that gets raised. This is to allow priority + // messages that should be handled before anything else. + int32_t line_number = (int32_t) pm_location_line_number(parser, &error->location); + + pm_buffer_append_format( + &buffer, + "%.*s:%" PRIi32 ": %s", + (int) pm_string_length(filepath), + pm_string_source(filepath), + line_number, + error->message + ); + + if (pm_parse_process_error_utf8_p(parser, &error->location)) { + pm_buffer_append_byte(&buffer, '\n'); + + pm_list_node_t *list_node = (pm_list_node_t *) error; + pm_list_t error_list = { .size = 1, .head = list_node, .tail = list_node }; - // It is implicitly assumed that the error messages will be encodeable - // as UTF-8. Because of this, we can't include source examples that - // contain invalid byte sequences. So if any source examples include - // invalid UTF-8 byte sequences, we will skip showing source examples - // entirely. - if (valid_utf8 && !pm_parse_process_error_utf8_p(&result->parser, &error->location)) { - valid_utf8 = false; + pm_parser_errors_format(parser, &error_list, &buffer, rb_stderr_tty_p(), false); + } + + VALUE value = rb_exc_new(rb_eArgError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + pm_buffer_free(&buffer); + + return value; + } + case PM_ERROR_LEVEL_LOAD: { + // Load errors are much simpler, because they don't include any of + // the source in them. We create the error directly from the + // message. + VALUE message = rb_enc_str_new_cstr(error->message, rb_locale_encoding()); + VALUE value = rb_exc_new3(rb_eLoadError, message); + rb_ivar_set(value, rb_intern_const("@path"), Qnil); + return value; + } } } - pm_buffer_t buffer = { 0 }; - pm_buffer_append_string(&buffer, "syntax errors found\n", 20); + pm_buffer_append_format( + &buffer, + "%.*s:%" PRIi32 ": syntax error%s found\n", + (int) pm_string_length(filepath), + pm_string_source(filepath), + (int32_t) pm_location_line_number(parser, &head->location), + (parser->error_list.size > 1) ? "s" : "" + ); if (valid_utf8) { - pm_parser_errors_format(&result->parser, &buffer, rb_stderr_tty_p()); + pm_parser_errors_format(parser, &parser->error_list, &buffer, rb_stderr_tty_p(), true); } else { - const pm_string_t *filepath = &result->parser.filepath; - for (const pm_diagnostic_t *error = head; error != NULL; error = (pm_diagnostic_t *) error->node.next) { if (error != head) pm_buffer_append_byte(&buffer, '\n'); - pm_buffer_append_format(&buffer, "%.*s:%" PRIi32 ": %s", (int) pm_string_length(filepath), pm_string_source(filepath), (int32_t) pm_location_line_number(&result->parser, &error->location), error->message); + pm_buffer_append_format(&buffer, "%.*s:%" PRIi32 ": %s", (int) pm_string_length(filepath), pm_string_source(filepath), (int32_t) pm_location_line_number(parser, &error->location), error->message); } } VALUE error = rb_exc_new(rb_eSyntaxError, pm_buffer_value(&buffer), pm_buffer_length(&buffer)); + + rb_encoding *filepath_encoding = result->node.filepath_encoding != NULL ? result->node.filepath_encoding : rb_utf8_encoding(); + VALUE path = rb_enc_str_new((const char *) pm_string_source(filepath), pm_string_length(filepath), filepath_encoding); + + rb_ivar_set(error, rb_intern_const("@path"), path); pm_buffer_free(&buffer); return error; @@ -8428,7 +8562,10 @@ pm_parse_process(pm_parse_result_t *result, pm_node_t *node) // First, set up the scope node so that the AST node is attached and can be // freed regardless of whether or we return an error. pm_scope_node_t *scope_node = &result->node; + rb_encoding *filepath_encoding = scope_node->filepath_encoding; + pm_scope_node_init(node, scope_node, NULL); + scope_node->filepath_encoding = filepath_encoding; // If there are errors, raise an appropriate error and free the result. if (parser->error_list.size > 0) { @@ -8627,6 +8764,7 @@ pm_parse_string(pm_parse_result_t *result, VALUE source, VALUE filepath) pm_string_constant_init(&result->input, RSTRING_PTR(source), RSTRING_LEN(source)); pm_options_encoding_set(&result->options, rb_enc_name(encoding)); + result->node.filepath_encoding = rb_enc_get(filepath); pm_options_filepath_set(&result->options, RSTRING_PTR(filepath)); RB_GC_GUARD(filepath); diff --git a/prism_compile.h b/prism_compile.h index 427fa54b516013..e58bed271f2b66 100644 --- a/prism_compile.h +++ b/prism_compile.h @@ -13,6 +13,9 @@ typedef struct pm_local_index_struct { int index, level; } pm_local_index_t; +// A declaration for the struct that lives in compile.c. +struct iseq_link_anchor; + // ScopeNodes are helper nodes, and will never be part of the AST. We manually // declare them here to avoid generating them. typedef struct pm_scope_node { @@ -26,6 +29,13 @@ typedef struct pm_scope_node { const pm_parser_t *parser; rb_encoding *encoding; + /** + * This is the encoding of the actual filepath object that will be used when + * a __FILE__ node is compiled or when the path has to be set on a syntax + * error. + */ + rb_encoding *filepath_encoding; + // The size of the local table // on the iseq which includes // locals and hidden variables @@ -33,6 +43,12 @@ typedef struct pm_scope_node { ID *constants; st_table *index_lookup_table; + + /** + * This will only be set on the top-level scope node. It will contain all of + * the instructions pertaining to BEGIN{} nodes. + */ + struct iseq_link_anchor *pre_execution_anchor; } pm_scope_node_t; void pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_t *previous); @@ -40,10 +56,19 @@ void pm_scope_node_destroy(pm_scope_node_t *scope_node); bool *rb_ruby_prism_ptr(void); typedef struct { + /** The parser that will do the actual parsing. */ pm_parser_t parser; + + /** The options that will be passed to the parser. */ pm_options_t options; + + /** The input that represents the source to be parsed. */ pm_string_t input; + + /** The resulting scope node that will hold the generated AST. */ pm_scope_node_t node; + + /** Whether or not this parse result has performed its parsing yet. */ bool parsed; } pm_parse_result_t; diff --git a/process.c b/process.c index 8d08da76505635..72483815d68f1b 100644 --- a/process.c +++ b/process.c @@ -1452,7 +1452,7 @@ proc_wait(int argc, VALUE *argv) * or as the logical OR of both: * * - Process::WNOHANG: Does not block if no child process is available. - * - Process:WUNTRACED: May return a stopped child process, even if not yet reported. + * - Process::WUNTRACED: May return a stopped child process, even if not yet reported. * * Not all flags are available on all platforms. * diff --git a/ruby_parser.c b/ruby_parser.c index 6d85a72c5b0c18..b674474f8c319b 100644 --- a/ruby_parser.c +++ b/ruby_parser.c @@ -93,32 +93,6 @@ dvar_defined(ID id, const void *p) return rb_dvar_defined(id, (const rb_iseq_t *)p); } -static bool -hash_literal_key_p(VALUE k) -{ - switch (OBJ_BUILTIN_TYPE(k)) { - case T_NODE: - return false; - default: - return true; - } -} - -static int -literal_cmp(VALUE val, VALUE lit) -{ - if (val == lit) return 0; - if (!hash_literal_key_p(val) || !hash_literal_key_p(lit)) return -1; - return rb_iseq_cdhash_cmp(val, lit); -} - -static st_index_t -literal_hash(VALUE a) -{ - if (!hash_literal_key_p(a)) return (st_index_t)a; - return rb_iseq_cdhash_hash(a); -} - static int is_usascii_enc(void *enc) { @@ -263,18 +237,6 @@ enc_from_encoding(void *enc) return rb_enc_from_encoding((rb_encoding *)enc); } -static int -encoding_get(VALUE obj) -{ - return ENCODING_GET(obj); -} - -static void -encoding_set(VALUE obj, int encindex) -{ - ENCODING_SET(obj, encindex); -} - static int encoding_is_ascii8bit(VALUE obj) { @@ -425,12 +387,6 @@ gc_guard(VALUE obj) RB_GC_GUARD(obj); } -static rb_imemo_tmpbuf_t * -tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt) -{ - return rb_imemo_tmpbuf_parser_heap(buf, old_heap, cnt); -} - static VALUE arg_error(void) { @@ -500,7 +456,6 @@ static const rb_parser_config_t rb_global_parser_config = { .nonempty_memcpy = nonempty_memcpy, .xmalloc_mul_add = rb_xmalloc_mul_add, - .tmpbuf_parser_heap = tmpbuf_parser_heap, .ast_new = ast_new, .compile_callback = rb_suppress_tracing, @@ -603,8 +558,6 @@ static const rb_parser_config_t rb_global_parser_config = { .ascii8bit_encoding = ascii8bit_encoding, .enc_codelen = enc_codelen, .enc_mbcput = enc_mbcput, - .char_to_option_kcode = rb_char_to_option_kcode, - .ascii8bit_encindex = rb_ascii8bit_encindex, .enc_find_index = rb_enc_find_index, .enc_from_index = enc_from_index, .enc_associate_index = rb_enc_associate_index, @@ -613,8 +566,6 @@ static const rb_parser_config_t rb_global_parser_config = { .enc_coderange_unknown = ENC_CODERANGE_UNKNOWN, .enc_compatible = enc_compatible, .enc_from_encoding = enc_from_encoding, - .encoding_get = encoding_get, - .encoding_set = encoding_set, .encoding_is_ascii8bit = encoding_is_ascii8bit, .usascii_encoding = usascii_encoding, .enc_coderange_broken = ENC_CODERANGE_BROKEN, @@ -627,9 +578,6 @@ static const rb_parser_config_t rb_global_parser_config = { .local_defined = local_defined, .dvar_defined = dvar_defined, - .literal_cmp = literal_cmp, - .literal_hash = literal_hash, - .syntax_error_append = syntax_error_append, .raise = rb_raise, .syntax_error_new = syntax_error_new, diff --git a/rubyparser.h b/rubyparser.h index 9a809aa059f220..138de47f0dd48b 100644 --- a/rubyparser.h +++ b/rubyparser.h @@ -200,6 +200,8 @@ typedef struct rb_code_location_struct { rb_code_position_t beg_pos; rb_code_position_t end_pos; } rb_code_location_t; +#define YYLTYPE rb_code_location_t +#define YYLTYPE_IS_DECLARED 1 typedef struct rb_parser_ast_token { int id; @@ -673,7 +675,7 @@ typedef struct RNode_LIT { typedef struct RNode_INTEGER { NODE node; - char* val; + char *val; int minus; int base; } rb_node_integer_t; @@ -681,14 +683,14 @@ typedef struct RNode_INTEGER { typedef struct RNode_FLOAT { NODE node; - char* val; + char *val; int minus; } rb_node_float_t; typedef struct RNode_RATIONAL { NODE node; - char* val; + char *val; int minus; int base; int seen_point; @@ -703,7 +705,7 @@ enum rb_numeric_type { typedef struct RNode_IMAGINARY { NODE node; - char* val; + char *val; int minus; int base; int seen_point; @@ -1239,7 +1241,6 @@ typedef struct rb_parser_config_struct { void *(*xmalloc_mul_add)(size_t x, size_t y, size_t z); /* imemo */ - rb_imemo_tmpbuf_t *(*tmpbuf_parser_heap)(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt); rb_ast_t *(*ast_new)(VALUE nb); // VALUE rb_suppress_tracing(VALUE (*func)(VALUE), VALUE arg); @@ -1356,16 +1357,12 @@ typedef struct rb_parser_config_struct { rb_encoding *(*ascii8bit_encoding)(void); int (*enc_codelen)(int c, rb_encoding *enc); int (*enc_mbcput)(unsigned int c, void *buf, rb_encoding *enc); - int (*char_to_option_kcode)(int c, int *option, int *kcode); - int (*ascii8bit_encindex)(void); int (*enc_find_index)(const char *name); rb_encoding *(*enc_from_index)(int idx); VALUE (*enc_associate_index)(VALUE obj, int encindex); int (*enc_isspace)(OnigCodePoint c, rb_encoding *enc); rb_encoding *(*enc_compatible)(VALUE str1, VALUE str2); VALUE (*enc_from_encoding)(rb_encoding *enc); - int (*encoding_get)(VALUE obj); - void (*encoding_set)(VALUE obj, int encindex); int (*encoding_is_ascii8bit)(VALUE obj); rb_encoding *(*usascii_encoding)(void); int enc_coderange_broken; @@ -1382,10 +1379,6 @@ typedef struct rb_parser_config_struct { // int rb_dvar_defined(ID id, const rb_iseq_t *iseq); int (*dvar_defined)(ID, const void*); - /* Compile (parse.y) */ - int (*literal_cmp)(VALUE val, VALUE lit); - parser_st_index_t (*literal_hash)(VALUE a); - /* Error (Exception) */ RBIMPL_ATTR_FORMAT(RBIMPL_PRINTF_FORMAT, 6, 0) VALUE (*syntax_error_append)(VALUE, VALUE, int, int, rb_encoding*, const char*, va_list); diff --git a/shape.c b/shape.c index 8241f67d6ab5d6..1158aad52c5f11 100644 --- a/shape.c +++ b/shape.c @@ -863,7 +863,13 @@ rb_shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) RUBY_ASSERT(rb_shape_id(shape) != OBJ_TOO_COMPLEX_SHAPE_ID); if (!shape_cache_get_iv_index(shape, id, value)) { - return shape_get_iv_index(shape, id, value); + // If it wasn't in the ancestor cache, then don't do a linear search + if (shape->ancestor_index && shape->next_iv_index >= ANCESTOR_CACHE_THRESHOLD) { + return false; + } + else { + return shape_get_iv_index(shape, id, value); + } } return true; diff --git a/spec/bundler/bundler/digest_spec.rb b/spec/bundler/bundler/digest_spec.rb index fd7b0c968e007c..f876827964a3a0 100644 --- a/spec/bundler/bundler/digest_spec.rb +++ b/spec/bundler/bundler/digest_spec.rb @@ -11,7 +11,7 @@ it "is compatible with stdlib" do random_strings = ["foo", "skfjsdlkfjsdf", "3924m", "ldskfj"] - # https://datatracker.ietf.org/doc/html/rfc3174#section-7.3 + # https://www.rfc-editor.org/rfc/rfc3174#section-7.3 rfc3174_test_cases = ["abc", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq", "a", "01234567" * 8] (random_strings + rfc3174_test_cases).each do |payload| diff --git a/spec/bundler/bundler/gem_version_promoter_spec.rb b/spec/bundler/bundler/gem_version_promoter_spec.rb index fccbb58fea181e..917daba95de5e5 100644 --- a/spec/bundler/bundler/gem_version_promoter_spec.rb +++ b/spec/bundler/bundler/gem_version_promoter_spec.rb @@ -33,13 +33,13 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "numerically sorts versions" do versions = sorted_versions(candidates: %w[1.7.7 1.7.8 1.7.9 1.7.15 1.8.0], current: "1.7.8") - expect(versions).to eq %w[1.7.7 1.7.8 1.7.9 1.7.15 1.8.0] + expect(versions).to eq %w[1.8.0 1.7.15 1.7.9 1.7.8 1.7.7] end context "with no options" do it "defaults to level=:major, strict=false, pre=false" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0] + expect(versions).to eq %w[2.1.0 2.0.1 1.0.0 0.9.0 0.3.1 0.3.0 0.2.0] end end @@ -51,25 +51,25 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "keeps downgrades" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0] + expect(versions).to eq %w[2.1.0 2.0.1 1.0.0 0.9.0 0.3.1 0.3.0 0.2.0] end end context "when level is minor" do before { gvp.level = :minor } - it "removes downgrades and major upgrades" do + it "sorts highest minor within same major in first position" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.3.0 0.3.1 0.9.0] + expect(versions).to eq %w[0.9.0 0.3.1 0.3.0 1.0.0 2.1.0 2.0.1 0.2.0] end end context "when level is patch" do before { gvp.level = :patch } - it "removes downgrades and major and minor upgrades" do + it "sorts highest patch within same minor in first position" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.3.0 0.3.1] + expect(versions).to eq %w[0.3.1 0.3.0 0.9.0 1.0.0 2.0.1 2.1.0 0.2.0] end end end @@ -82,25 +82,25 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "orders by version" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0] + expect(versions).to eq %w[2.1.0 2.0.1 1.0.0 0.9.0 0.3.1 0.3.0 0.2.0] end end context "when level is minor" do before { gvp.level = :minor } - it "favors downgrades, then upgrades by major descending, minor ascending, patch ascending" do + it "favors minor upgrades, then patch upgrades, then major upgrades, then downgrades" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 2.0.1 2.1.0 1.0.0 0.3.0 0.3.1 0.9.0] + expect(versions).to eq %w[0.9.0 0.3.1 0.3.0 1.0.0 2.1.0 2.0.1 0.2.0] end end context "when level is patch" do before { gvp.level = :patch } - it "favors downgrades, then upgrades by major descending, minor descending, patch ascending" do + it "favors patch upgrades, then minor upgrades, then major upgrades, then downgrades" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.0.1 2.1.0], current: "0.3.0") - expect(versions).to eq %w[0.2.0 2.1.0 2.0.1 1.0.0 0.9.0 0.3.0 0.3.1] + expect(versions).to eq %w[0.3.1 0.3.0 0.9.0 1.0.0 2.0.1 2.1.0 0.2.0] end end end @@ -110,7 +110,7 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "sorts regardless of prerelease status" do versions = sorted_versions(candidates: %w[1.7.7.pre 1.8.0 1.8.1.pre 1.8.1 2.0.0.pre 2.0.0], current: "1.8.0") - expect(versions).to eq %w[1.7.7.pre 1.8.0 1.8.1.pre 1.8.1 2.0.0.pre 2.0.0] + expect(versions).to eq %w[2.0.0 2.0.0.pre 1.8.1 1.8.1.pre 1.8.0 1.7.7.pre] end end @@ -119,16 +119,16 @@ def sorted_versions(candidates:, current:, name: "foo", locked: []) it "deprioritizes prerelease gems" do versions = sorted_versions(candidates: %w[1.7.7.pre 1.8.0 1.8.1.pre 1.8.1 2.0.0.pre 2.0.0], current: "1.8.0") - expect(versions).to eq %w[1.7.7.pre 1.8.1.pre 2.0.0.pre 1.8.0 1.8.1 2.0.0] + expect(versions).to eq %w[2.0.0 1.8.1 1.8.0 2.0.0.pre 1.8.1.pre 1.7.7.pre] end end context "when locking and not major" do before { gvp.level = :minor } - it "keeps the current version last" do + it "keeps the current version first" do versions = sorted_versions(candidates: %w[0.2.0 0.3.0 0.3.1 0.9.0 1.0.0 2.1.0 2.0.1], current: "0.3.0", locked: ["bar"]) - expect(versions.last).to eq("0.3.0") + expect(versions.first).to eq("0.3.0") end end end diff --git a/spec/bundler/commands/lock_spec.rb b/spec/bundler/commands/lock_spec.rb index 0f1aeef910650e..f6793d393ba828 100644 --- a/spec/bundler/commands/lock_spec.rb +++ b/spec/bundler/commands/lock_spec.rb @@ -392,6 +392,22 @@ expect(the_bundle.locked_gems.specs.map(&:full_name)).to eq(%w[foo-1.5.0 bar-2.1.1 qux-1.1.0].sort) end + it "shows proper error when Gemfile changes forbid patch upgrades, and --patch --strict is given" do + # force next minor via Gemfile + gemfile <<-G + source "#{file_uri_for(gem_repo4)}" + gem 'foo', '1.5.0' + gem 'qux' + G + + bundle "lock --update foo --patch --strict", raise_on_error: false + + expect(err).to include( + "foo is locked to 1.4.3, while Gemfile is requesting foo (= 1.5.0). " \ + "--strict --patch was specified, but there are no patch level upgrades from 1.4.3 satisfying foo (= 1.5.0), so version solving has failed" + ) + end + context "pre" do it "defaults to major" do bundle "lock --update --pre" diff --git a/spec/bundler/commands/update_spec.rb b/spec/bundler/commands/update_spec.rb index 2a09e0531ba7ea..cfb86ebb54832f 100644 --- a/spec/bundler/commands/update_spec.rb +++ b/spec/bundler/commands/update_spec.rb @@ -864,6 +864,90 @@ expect(exitstatus).to eq(22) end + context "with multiple sources and caching enabled" do + before do + build_repo2 do + build_gem "rack", "1.0.0" + + build_gem "request_store", "1.0.0" do |s| + s.add_dependency "rack", "1.0.0" + end + end + + build_repo4 do + # set up repo with no gems + end + + gemfile <<~G + source "#{file_uri_for(gem_repo2)}" + + gem "request_store" + + source "#{file_uri_for(gem_repo4)}" do + end + G + + lockfile <<~L + GEM + remote: #{file_uri_for(gem_repo2)}/ + specs: + rack (1.0.0) + request_store (1.0.0) + rack (= 1.0.0) + + GEM + remote: #{file_uri_for(gem_repo4)}/ + specs: + + PLATFORMS + #{local_platform} + + DEPENDENCIES + request_store + + BUNDLED WITH + #{Bundler::VERSION} + L + end + + it "works" do + bundle :install + bundle :cache + + update_repo2 do + build_gem "request_store", "1.1.0" do |s| + s.add_dependency "rack", "1.0.0" + end + end + + bundle "update request_store" + + expect(out).to include("Bundle updated!") + + expect(lockfile).to eq <<~L + GEM + remote: #{file_uri_for(gem_repo2)}/ + specs: + rack (1.0.0) + request_store (1.1.0) + rack (= 1.0.0) + + GEM + remote: #{file_uri_for(gem_repo4)}/ + specs: + + PLATFORMS + #{local_platform} + + DEPENDENCIES + request_store + + BUNDLED WITH + #{Bundler::VERSION} + L + end + end + context "with multiple, duplicated sources, with lockfile in old format", bundler: "< 3" do before do build_repo2 do diff --git a/spec/bundler/support/builders.rb b/spec/bundler/support/builders.rb index 46931e9ca530ba..91e6e6e574b931 100644 --- a/spec/bundler/support/builders.rb +++ b/spec/bundler/support/builders.rb @@ -18,7 +18,7 @@ def pl(platform) end def rake_version - "13.1.0" + "13.2.0" end def build_repo1 diff --git a/spec/prism.mspec b/spec/prism.mspec index bd427392e2f4e3..9f5e2b2a5fdaa5 100644 --- a/spec/prism.mspec +++ b/spec/prism.mspec @@ -1,14 +1,6 @@ # frozen_string_literal: true -## Command line -MSpec.register(:exclude, "The -S command line option runs launcher found in PATH, but only code after the first /#!.*ruby.*/-ish line in target file") -MSpec.register(:exclude, "The -x command line option runs code after the first /#!.*ruby.*/-ish line in target file") -MSpec.register(:exclude, "The -x command line option fails when /#!.*ruby.*/-ish line in target file is not found") -MSpec.register(:exclude, "The -x command line option behaves as -x was set when non-ruby shebang is encountered on first line") -MSpec.register(:exclude, "The --debug flag produces debugging info on attempted frozen string modification") - ## Language -MSpec.register(:exclude, "The BEGIN keyword runs multiple begins in FIFO order") MSpec.register(:exclude, "Executing break from within a block works when passing through a super call") MSpec.register(:exclude, "The defined? keyword when called with a method name in a void context warns about the void context when parsing it") MSpec.register(:exclude, "Hash literal expands an '**{}' or '**obj' element with the last key/value pair taking precedence") @@ -17,8 +9,6 @@ MSpec.register(:exclude, "Hash literal merges multiple nested '**obj' in Hash li MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes") MSpec.register(:exclude, "Hash literal raises a SyntaxError at parse time when Symbol key with invalid bytes and 'key: value' syntax used") MSpec.register(:exclude, "The next statement in a method is invalid and raises a SyntaxError") -MSpec.register(:exclude, "Pattern matching variable pattern does not support using variable name (except _) several times") -MSpec.register(:exclude, "Pattern matching Hash pattern raise SyntaxError when keys duplicate in pattern") MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation") MSpec.register(:exclude, "Regexps with encoding modifiers supports /e (EUC encoding) with interpolation /o") MSpec.register(:exclude, "Regexps with encoding modifiers preserves EUC-JP as /e encoding through interpolation") @@ -33,10 +23,6 @@ MSpec.register(:exclude, "A Symbol literal raises an SyntaxError at parse time w ## Core MSpec.register(:exclude, "IO.popen with a leading Array argument accepts a trailing Hash of Process.exec options") MSpec.register(:exclude, "IO.popen with a leading Array argument accepts an IO mode argument following the Array") -MSpec.register(:exclude, "Kernel#eval includes file and line information in syntax error") -MSpec.register(:exclude, "Kernel#eval evaluates string with given filename and negative linenumber") -MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows spaces before the magic encoding comment") -MSpec.register(:exclude, "Kernel#eval with a magic encoding comment allows a shebang line and some spaces before the magic encoding comment") MSpec.register(:exclude, "TracePoint#eval_script is the evald source code") MSpec.register(:exclude, "TracePoint#event returns the type of event") MSpec.register(:exclude, "TracePoint#inspect returns a String showing the event, method, path and line for a :return event") @@ -60,6 +46,4 @@ MSpec.register(:exclude, "Coverage.result does not clear counters when stop: fal MSpec.register(:exclude, "Coverage.result clears counters (sets 0 values) when stop: false and clear: true specified") MSpec.register(:exclude, "Coverage.result does not clear counters when stop: false and clear: false specified") MSpec.register(:exclude, "Coverage.start measures coverage within eval") -MSpec.register(:exclude, "ERB#filename raises an exception if there are errors processing content") -MSpec.register(:exclude, "ERB#filename uses '(erb)' as filename when filename is not set") MSpec.register(:exclude, "Socket.gethostbyaddr using an IPv6 address with an explicit address family raises SocketError when the address is not supported by the family") diff --git a/spec/ruby/command_line/frozen_strings_spec.rb b/spec/ruby/command_line/frozen_strings_spec.rb index 24b979b68ba1f8..334b98273b7827 100644 --- a/spec/ruby/command_line/frozen_strings_spec.rb +++ b/spec/ruby/command_line/frozen_strings_spec.rb @@ -28,6 +28,15 @@ end describe "With neither --enable-frozen-string-literal nor --disable-frozen-string-literal flag set" do + before do + # disable --enable-frozen-string-literal and --disable-frozen-string-literal passed in $RUBYOPT + @rubyopt = ENV["RUBYOPT"] + ENV["RUBYOPT"] = "" + end + + after do + ENV["RUBYOPT"] = @rubyopt + end it "produce a different object each time" do ruby_exe(fixture(__FILE__, "freeze_flag_one_literal.rb")).chomp.should == "false" diff --git a/spec/ruby/core/class/subclasses_spec.rb b/spec/ruby/core/class/subclasses_spec.rb index a16b934d4fc5d6..50eb5358d97419 100644 --- a/spec/ruby/core/class/subclasses_spec.rb +++ b/spec/ruby/core/class/subclasses_spec.rb @@ -7,7 +7,7 @@ assert_subclasses(ModuleSpecs::Parent, [ModuleSpecs::Child, ModuleSpecs::Child2]) end - it "does not return included modules" do + it "does not return included modules from the parent" do parent = Class.new child = Class.new(parent) mod = Module.new @@ -16,6 +16,33 @@ assert_subclasses(parent, [child]) end + it "does not return included modules from the child" do + parent = Class.new + child = Class.new(parent) + mod = Module.new + parent.include(mod) + + assert_subclasses(parent, [child]) + end + + it "does not return prepended modules from the parent" do + parent = Class.new + child = Class.new(parent) + mod = Module.new + parent.prepend(mod) + + assert_subclasses(parent, [child]) + end + + it "does not return prepended modules from the child" do + parent = Class.new + child = Class.new(parent) + mod = Module.new + child.prepend(mod) + + assert_subclasses(parent, [child]) + end + it "does not return singleton classes" do a = Class.new diff --git a/spec/ruby/core/dir/children_spec.rb b/spec/ruby/core/dir/children_spec.rb index 92c593e5ba2b4e..0ad3df4669e36d 100644 --- a/spec/ruby/core/dir/children_spec.rb +++ b/spec/ruby/core/dir/children_spec.rb @@ -131,4 +131,17 @@ children = @dir.children.sort children.first.encoding.should equal(Encoding::EUC_KR) end + + it "returns the same result when called repeatedly" do + @dir = Dir.open DirSpecs.mock_dir + + a = [] + @dir.each {|dir| a << dir} + + b = [] + @dir.each {|dir| b << dir} + + a.sort.should == b.sort + a.sort.should == DirSpecs.expected_paths + end end diff --git a/spec/ruby/core/dir/each_child_spec.rb b/spec/ruby/core/dir/each_child_spec.rb index 520186e79ef310..7194273b9529c4 100644 --- a/spec/ruby/core/dir/each_child_spec.rb +++ b/spec/ruby/core/dir/each_child_spec.rb @@ -86,6 +86,19 @@ @dir.each_child { |f| f }.should == @dir end + it "returns the same result when called repeatedly" do + @dir = Dir.open DirSpecs.mock_dir + + a = [] + @dir.each {|dir| a << dir} + + b = [] + @dir.each {|dir| b << dir} + + a.sort.should == b.sort + a.sort.should == DirSpecs.expected_paths + end + describe "when no block is given" do it "returns an Enumerator" do @dir = Dir.new(DirSpecs.mock_dir) diff --git a/spec/ruby/core/dir/each_spec.rb b/spec/ruby/core/dir/each_spec.rb index 8c69a7212be3c2..7674663d82b0e4 100644 --- a/spec/ruby/core/dir/each_spec.rb +++ b/spec/ruby/core/dir/each_spec.rb @@ -35,6 +35,17 @@ ls.should include(@dir.read) end + it "returns the same result when called repeatedly" do + a = [] + @dir.each {|dir| a << dir} + + b = [] + @dir.each {|dir| b << dir} + + a.sort.should == b.sort + a.sort.should == DirSpecs.expected_paths + end + describe "when no block is given" do it "returns an Enumerator" do @dir.each.should be_an_instance_of(Enumerator) diff --git a/spec/ruby/core/exception/fixtures/syntax_error.rb b/spec/ruby/core/exception/fixtures/syntax_error.rb new file mode 100644 index 00000000000000..ccec62f7a1a8f5 --- /dev/null +++ b/spec/ruby/core/exception/fixtures/syntax_error.rb @@ -0,0 +1,3 @@ +# rubocop:disable Lint/Syntax +1+1=2 +# rubocop:enable Lint/Syntax diff --git a/spec/ruby/core/exception/syntax_error_spec.rb b/spec/ruby/core/exception/syntax_error_spec.rb new file mode 100644 index 00000000000000..6cc8522de388ed --- /dev/null +++ b/spec/ruby/core/exception/syntax_error_spec.rb @@ -0,0 +1,27 @@ +require_relative '../../spec_helper' + +ruby_version_is "3.2" do + describe "SyntaxError#path" do + it "returns the file path provided to eval" do + filename = "speccing.rb" + + -> { + eval("if true", TOPLEVEL_BINDING, filename) + }.should raise_error(SyntaxError) { |e| + e.path.should == filename + } + end + + it "returns the file path that raised an exception" do + expected_path = fixture(__FILE__, "syntax_error.rb") + + -> { + require_relative "fixtures/syntax_error" + }.should raise_error(SyntaxError) { |e| e.path.should == expected_path } + end + + it "returns nil when constructed directly" do + SyntaxError.new.path.should == nil + end + end +end diff --git a/spec/ruby/core/hash/compare_by_identity_spec.rb b/spec/ruby/core/hash/compare_by_identity_spec.rb index 3804f04bf643e9..2975526a97a5f0 100644 --- a/spec/ruby/core/hash/compare_by_identity_spec.rb +++ b/spec/ruby/core/hash/compare_by_identity_spec.rb @@ -110,6 +110,10 @@ def o.hash; 123; end @idh.keys.first.should equal foo end + # Check `#[]=` call with a String literal. + # Don't use `#+` because with `#+` it's no longer a String literal. + # + # See https://bugs.ruby-lang.org/issues/12855 it "gives different identity for string literals" do eval <<~RUBY # frozen_string_literal: false diff --git a/spec/ruby/core/kernel/eval_spec.rb b/spec/ruby/core/kernel/eval_spec.rb index 5d82f57e447181..454bc4a58e5c34 100644 --- a/spec/ruby/core/kernel/eval_spec.rb +++ b/spec/ruby/core/kernel/eval_spec.rb @@ -135,7 +135,7 @@ it "includes file and line information in syntax error" do expected = 'speccing.rb' -> { - eval('if true',TOPLEVEL_BINDING, expected) + eval('if true', TOPLEVEL_BINDING, expected) }.should raise_error(SyntaxError) { |e| e.message.should =~ /#{expected}:1:.+/ } @@ -144,7 +144,7 @@ it "evaluates string with given filename and negative linenumber" do expected_file = 'speccing.rb' -> { - eval('if true',TOPLEVEL_BINDING, expected_file, -100) + eval('if true', TOPLEVEL_BINDING, expected_file, -100) }.should raise_error(SyntaxError) { |e| e.message.should =~ /#{expected_file}:-100:.+/ } diff --git a/spec/ruby/core/marshal/shared/load.rb b/spec/ruby/core/marshal/shared/load.rb index 1c8fa4c38c4178..f599042529341a 100644 --- a/spec/ruby/core/marshal/shared/load.rb +++ b/spec/ruby/core/marshal/shared/load.rb @@ -647,7 +647,7 @@ end it "sets binmode if it is loading through StringIO stream" do - io = StringIO.new(+"\004\b:\vsymbol") + io = StringIO.new("\004\b:\vsymbol") def io.binmode; raise "binmode"; end -> { Marshal.load(io) }.should raise_error(RuntimeError, "binmode") end diff --git a/spec/ruby/language/assignments_spec.rb b/spec/ruby/language/assignments_spec.rb index 005c1f0dc3f928..2773508d8d44ed 100644 --- a/spec/ruby/language/assignments_spec.rb +++ b/spec/ruby/language/assignments_spec.rb @@ -1,7 +1,61 @@ require_relative '../spec_helper' # Should be synchronized with spec/ruby/language/optional_assignments_spec.rb +# Some specs for assignments are located in language/variables_spec.rb describe 'Assignments' do + describe 'using =' do + describe 'evaluation order' do + it 'evaluates expressions left to right when assignment with an accessor' do + object = Object.new + def object.a=(value) end + ScratchPad.record [] + + (ScratchPad << :receiver; object).a = (ScratchPad << :rhs; :value) + ScratchPad.recorded.should == [:receiver, :rhs] + end + + it 'evaluates expressions left to right when assignment with a #[]=' do + object = Object.new + def object.[]=(_, _) end + ScratchPad.record [] + + (ScratchPad << :receiver; object)[(ScratchPad << :argument; :a)] = (ScratchPad << :rhs; :value) + ScratchPad.recorded.should == [:receiver, :argument, :rhs] + end + + # similar tests for evaluation order are located in language/constants_spec.rb + ruby_version_is ''...'3.2' do + it 'evaluates expressions right to left when assignment with compounded constant' do + m = Module.new + ScratchPad.record [] + + (ScratchPad << :module; m)::A = (ScratchPad << :rhs; :value) + ScratchPad.recorded.should == [:rhs, :module] + end + end + + ruby_version_is '3.2' do + it 'evaluates expressions left to right when assignment with compounded constant' do + m = Module.new + ScratchPad.record [] + + (ScratchPad << :module; m)::A = (ScratchPad << :rhs; :value) + ScratchPad.recorded.should == [:module, :rhs] + end + end + + it 'raises TypeError after evaluation of right-hand-side when compounded constant module is not a module' do + ScratchPad.record [] + + -> { + (:not_a_module)::A = (ScratchPad << :rhs; :value) + }.should raise_error(TypeError) + + ScratchPad.recorded.should == [:rhs] + end + end + end + describe 'using +=' do describe 'using an accessor' do before do @@ -148,3 +202,328 @@ module ConstantSpecs end end end + +# generic cases +describe 'Multiple assignments' do + it 'assigns multiple targets when assignment with an accessor' do + object = Object.new + class << object + attr_accessor :a, :b + end + + object.a, object.b = :a, :b + + object.a.should == :a + object.b.should == :b + end + + it 'assigns multiple targets when assignment with a nested accessor' do + object = Object.new + class << object + attr_accessor :a, :b + end + + (object.a, object.b), c = [:a, :b], nil + + object.a.should == :a + object.b.should == :b + end + + it 'assigns multiple targets when assignment with a #[]=' do + object = Object.new + class << object + def []=(k, v) (@h ||= {})[k] = v; end + def [](k) (@h ||= {})[k]; end + end + + object[:a], object[:b] = :a, :b + + object[:a].should == :a + object[:b].should == :b + end + + it 'assigns multiple targets when assignment with a nested #[]=' do + object = Object.new + class << object + def []=(k, v) (@h ||= {})[k] = v; end + def [](k) (@h ||= {})[k]; end + end + + (object[:a], object[:b]), c = [:v1, :v2], nil + + object[:a].should == :v1 + object[:b].should == :v2 + end + + it 'assigns multiple targets when assignment with compounded constant' do + m = Module.new + + m::A, m::B = :a, :b + + m::A.should == :a + m::B.should == :b + end + + it 'assigns multiple targets when assignment with a nested compounded constant' do + m = Module.new + + (m::A, m::B), c = [:a, :b], nil + + m::A.should == :a + m::B.should == :b + end +end + +describe 'Multiple assignments' do + describe 'evaluation order' do + ruby_version_is ''...'3.1' do + it 'evaluates expressions right to left when assignment with an accessor' do + object = Object.new + def object.a=(value) end + ScratchPad.record [] + + (ScratchPad << :a; object).a, (ScratchPad << :b; object).a = (ScratchPad << :c; :c), (ScratchPad << :d; :d) + ScratchPad.recorded.should == [:c, :d, :a, :b] + end + + it 'evaluates expressions right to left when assignment with a nested accessor' do + object = Object.new + def object.a=(value) end + ScratchPad.record [] + + ((ScratchPad << :a; object).a, foo), bar = [(ScratchPad << :b; :b)] + ScratchPad.recorded.should == [:b, :a] + end + end + + ruby_version_is '3.1' do + it 'evaluates expressions left to right when assignment with an accessor' do + object = Object.new + def object.a=(value) end + ScratchPad.record [] + + (ScratchPad << :a; object).a, (ScratchPad << :b; object).a = (ScratchPad << :c; :c), (ScratchPad << :d; :d) + ScratchPad.recorded.should == [:a, :b, :c, :d] + end + + it 'evaluates expressions left to right when assignment with a nested accessor' do + object = Object.new + def object.a=(value) end + ScratchPad.record [] + + ((ScratchPad << :a; object).a, foo), bar = [(ScratchPad << :b; :b)] + ScratchPad.recorded.should == [:a, :b] + end + + it 'evaluates expressions left to right when assignment with a deeply nested accessor' do + o = Object.new + def o.a=(value) end + def o.b=(value) end + def o.c=(value) end + def o.d=(value) end + def o.e=(value) end + def o.f=(value) end + ScratchPad.record [] + + (ScratchPad << :a; o).a, + ((ScratchPad << :b; o).b, + ((ScratchPad << :c; o).c, (ScratchPad << :d; o).d), + (ScratchPad << :e; o).e), + (ScratchPad << :f; o).f = (ScratchPad << :value; :value) + + ScratchPad.recorded.should == [:a, :b, :c, :d, :e, :f, :value] + end + end + + ruby_version_is ''...'3.1' do + it 'evaluates expressions right to left when assignment with a #[]=' do + object = Object.new + def object.[]=(_, _) end + ScratchPad.record [] + + (ScratchPad << :a; object)[(ScratchPad << :b; :b)], (ScratchPad << :c; object)[(ScratchPad << :d; :d)] = (ScratchPad << :e; :e), (ScratchPad << :f; :f) + ScratchPad.recorded.should == [:e, :f, :a, :b, :c, :d] + end + + it 'evaluates expressions right to left when assignment with a nested #[]=' do + object = Object.new + def object.[]=(_, _) end + ScratchPad.record [] + + ((ScratchPad << :a; object)[(ScratchPad << :b; :b)], foo), bar = [(ScratchPad << :c; :c)] + ScratchPad.recorded.should == [:c, :a, :b] + end + end + + ruby_version_is '3.1' do + it 'evaluates expressions left to right when assignment with a #[]=' do + object = Object.new + def object.[]=(_, _) end + ScratchPad.record [] + + (ScratchPad << :a; object)[(ScratchPad << :b; :b)], (ScratchPad << :c; object)[(ScratchPad << :d; :d)] = (ScratchPad << :e; :e), (ScratchPad << :f; :f) + ScratchPad.recorded.should == [:a, :b, :c, :d, :e, :f] + end + + it 'evaluates expressions left to right when assignment with a nested #[]=' do + object = Object.new + def object.[]=(_, _) end + ScratchPad.record [] + + ((ScratchPad << :a; object)[(ScratchPad << :b; :b)], foo), bar = [(ScratchPad << :c; :c)] + ScratchPad.recorded.should == [:a, :b, :c] + end + + it 'evaluates expressions left to right when assignment with a deeply nested #[]=' do + o = Object.new + def o.[]=(_, _) end + ScratchPad.record [] + + (ScratchPad << :ra; o)[(ScratchPad << :aa; :aa)], + ((ScratchPad << :rb; o)[(ScratchPad << :ab; :ab)], + ((ScratchPad << :rc; o)[(ScratchPad << :ac; :ac)], (ScratchPad << :rd; o)[(ScratchPad << :ad; :ad)]), + (ScratchPad << :re; o)[(ScratchPad << :ae; :ae)]), + (ScratchPad << :rf; o)[(ScratchPad << :af; :af)] = (ScratchPad << :value; :value) + + ScratchPad.recorded.should == [:ra, :aa, :rb, :ab, :rc, :ac, :rd, :ad, :re, :ae, :rf, :af, :value] + end + end + + ruby_version_is ''...'3.2' do + it 'evaluates expressions right to left when assignment with compounded constant' do + m = Module.new + ScratchPad.record [] + + (ScratchPad << :a; m)::A, (ScratchPad << :b; m)::B = (ScratchPad << :c; :c), (ScratchPad << :d; :d) + ScratchPad.recorded.should == [:c, :d, :a, :b] + end + end + + ruby_version_is '3.2' do + it 'evaluates expressions left to right when assignment with compounded constant' do + m = Module.new + ScratchPad.record [] + + (ScratchPad << :a; m)::A, (ScratchPad << :b; m)::B = (ScratchPad << :c; :c), (ScratchPad << :d; :d) + ScratchPad.recorded.should == [:a, :b, :c, :d] + end + + it 'evaluates expressions left to right when assignment with a nested compounded constant' do + m = Module.new + ScratchPad.record [] + + ((ScratchPad << :a; m)::A, foo), bar = [(ScratchPad << :b; :b)] + ScratchPad.recorded.should == [:a, :b] + end + + it 'evaluates expressions left to right when assignment with deeply nested compounded constants' do + m = Module.new + ScratchPad.record [] + + (ScratchPad << :a; m)::A, + ((ScratchPad << :b; m)::B, + ((ScratchPad << :c; m)::C, (ScratchPad << :d; m)::D), + (ScratchPad << :e; m)::E), + (ScratchPad << :f; m)::F = (ScratchPad << :value; :value) + + ScratchPad.recorded.should == [:a, :b, :c, :d, :e, :f, :value] + end + end + end + + context 'when assignment with method call and receiver is self' do + it 'assigns values correctly when assignment with accessor' do + object = Object.new + class << object + attr_accessor :a, :b + + def assign(v1, v2) + self.a, self.b = v1, v2 + end + end + + object.assign :v1, :v2 + object.a.should == :v1 + object.b.should == :v2 + end + + it 'evaluates expressions right to left when assignment with a nested accessor' do + object = Object.new + class << object + attr_accessor :a, :b + + def assign(v1, v2) + (self.a, self.b), c = [v1, v2], nil + end + end + + object.assign :v1, :v2 + object.a.should == :v1 + object.b.should == :v2 + end + + it 'assigns values correctly when assignment with a #[]=' do + object = Object.new + class << object + def []=(key, v) + @h ||= {} + @h[key] = v + end + + def [](key) + (@h || {})[key] + end + + def assign(k1, v1, k2, v2) + self[k1], self[k2] = v1, v2 + end + end + + object.assign :k1, :v1, :k2, :v2 + object[:k1].should == :v1 + object[:k2].should == :v2 + end + + it 'assigns values correctly when assignment with a nested #[]=' do + object = Object.new + class << object + def []=(key, v) + @h ||= {} + @h[key] = v + end + + def [](key) + (@h || {})[key] + end + + def assign(k1, v1, k2, v2) + (self[k1], self[k2]), c = [v1, v2], nil + end + end + + object.assign :k1, :v1, :k2, :v2 + object[:k1].should == :v1 + object[:k2].should == :v2 + end + + it 'assigns values correctly when assignment with compounded constant' do + m = Module.new + m.module_exec do + self::A, self::B = :v1, :v2 + end + + m::A.should == :v1 + m::B.should == :v2 + end + + it 'assigns values correctly when assignment with a nested compounded constant' do + m = Module.new + m.module_exec do + (self::A, self::B), c = [:v1, :v2], nil + end + + m::A.should == :v1 + m::B.should == :v2 + end + end +end diff --git a/spec/ruby/language/case_spec.rb b/spec/ruby/language/case_spec.rb index cfa612b93abb78..3262f09dd590f5 100644 --- a/spec/ruby/language/case_spec.rb +++ b/spec/ruby/language/case_spec.rb @@ -415,6 +415,19 @@ def test(v) self.test("bar").should == false self.test(true).should == true end + + it "warns if there are identical when clauses" do + -> { + eval <<~RUBY + case 1 + when 2 + :foo + when 2 + :bar + end + RUBY + }.should complain(/warning: duplicated .when' clause with line \d+ is ignored/, verbose: true) + end end describe "The 'case'-construct with no target expression" do diff --git a/spec/ruby/language/ensure_spec.rb b/spec/ruby/language/ensure_spec.rb index 175222c86e01dc..16e626b4d0be5e 100644 --- a/spec/ruby/language/ensure_spec.rb +++ b/spec/ruby/language/ensure_spec.rb @@ -335,7 +335,7 @@ def foo begin raise "oops" ensure - return caller(0, 2) + return caller(0, 2) # rubocop:disable Lint/EnsureReturn end end line = __LINE__ diff --git a/spec/ruby/language/if_spec.rb b/spec/ruby/language/if_spec.rb index a5da69600049c4..70c847d830d0a6 100644 --- a/spec/ruby/language/if_spec.rb +++ b/spec/ruby/language/if_spec.rb @@ -305,6 +305,15 @@ 6.times(&b) ScratchPad.recorded.should == [4, 5, 4, 5] end + + it "warns when Integer literals are used instead of predicates" do + -> { + eval <<~RUBY + 10.times { |i| ScratchPad << i if 4..5 } + RUBY + }.should complain(/warning: integer literal in flip-flop/, verbose: true) + ScratchPad.recorded.should == [] + end end describe "when a branch syntactically does not return a value" do diff --git a/spec/ruby/language/string_spec.rb b/spec/ruby/language/string_spec.rb index 1a1cd35850563d..083a7f5db50de2 100644 --- a/spec/ruby/language/string_spec.rb +++ b/spec/ruby/language/string_spec.rb @@ -231,9 +231,16 @@ def long_string_literals ruby_exe(fixture(__FILE__, "freeze_magic_comment_across_files.rb")).chomp.should == "true" end - it "produce different objects for literals with the same content in different files if the other file doesn't have the comment" do - frozen_string_literal = "test".frozen? && "test".equal?("test") - ruby_exe(fixture(__FILE__, "freeze_magic_comment_across_files_no_comment.rb")).chomp.should == (!frozen_string_literal).to_s + guard -> { !(eval("'test'").frozen? && "test".equal?("test")) } do + it "produces different objects for literals with the same content in different files if the other file doesn't have the comment and String literals aren't frozen by default" do + ruby_exe(fixture(__FILE__, "freeze_magic_comment_across_files_no_comment.rb")).chomp.should == "true" + end + end + + guard -> { eval("'test'").frozen? && "test".equal?("test") } do + it "produces the same objects for literals with the same content in different files if the other file doesn't have the comment and String literals are frozen by default" do + ruby_exe(fixture(__FILE__, "freeze_magic_comment_across_files_no_comment.rb")).chomp.should == "false" + end end it "produce different objects for literals with the same content in different files if they have different encodings" do diff --git a/spec/ruby/library/io-wait/wait_spec.rb b/spec/ruby/library/io-wait/wait_spec.rb index 3861281277bc12..fc07c6a8d9831f 100644 --- a/spec/ruby/library/io-wait/wait_spec.rb +++ b/spec/ruby/library/io-wait/wait_spec.rb @@ -48,25 +48,18 @@ end it "waits for the READABLE event to be ready" do - queue = Queue.new - thread = Thread.new { queue.pop; sleep 1; @w.write('data to read') }; + @r.wait(IO::READABLE, 0).should == nil - queue.push('signal'); - @r.wait(IO::READABLE, 2).should_not == nil - - thread.join + @w.write('data to read') + @r.wait(IO::READABLE, 0).should_not == nil end it "waits for the WRITABLE event to be ready" do written_bytes = IOWaitSpec.exhaust_write_buffer(@w) + @w.wait(IO::WRITABLE, 0).should == nil - queue = Queue.new - thread = Thread.new { queue.pop; sleep 1; @r.read(written_bytes) }; - - queue.push('signal'); - @w.wait(IO::WRITABLE, 2).should_not == nil - - thread.join + @r.read(written_bytes) + @w.wait(IO::WRITABLE, 0).should_not == nil end it "returns nil when the READABLE event is not ready during the timeout" do @@ -89,6 +82,24 @@ -> { @w.wait(-1, 0) }.should raise_error(ArgumentError, "Events must be positive integer!") end end + + it "changes thread status to 'sleep' when waits for READABLE event" do + t = Thread.new { @r.wait(IO::READABLE, 10) } + sleep 1 + t.status.should == 'sleep' + t.kill + t.join # Thread#kill doesn't wait for the thread to end + end + + it "changes thread status to 'sleep' when waits for WRITABLE event" do + written_bytes = IOWaitSpec.exhaust_write_buffer(@w) + + t = Thread.new { @w.wait(IO::WRITABLE, 10) } + sleep 1 + t.status.should == 'sleep' + t.kill + t.join # Thread#kill doesn't wait for the thread to end + end end context "[timeout, mode] passed" do diff --git a/spec/ruby/library/net-http/httpresponse/inspect_spec.rb b/spec/ruby/library/net-http/httpresponse/inspect_spec.rb index 23b6bff58190dd..43071ec8cdfb8c 100644 --- a/spec/ruby/library/net-http/httpresponse/inspect_spec.rb +++ b/spec/ruby/library/net-http/httpresponse/inspect_spec.rb @@ -8,7 +8,7 @@ res.inspect.should == "#" res = Net::HTTPUnknownResponse.new("1.0", "???", "test response") - socket = Net::BufferedIO.new(StringIO.new(+"test body")) + socket = Net::BufferedIO.new(StringIO.new("test body")) res.reading_body(socket, true) {} res.inspect.should == "#" end diff --git a/spec/ruby/library/net-http/httpresponse/read_body_spec.rb b/spec/ruby/library/net-http/httpresponse/read_body_spec.rb index 61a576d81263e5..4530a26bfc264b 100644 --- a/spec/ruby/library/net-http/httpresponse/read_body_spec.rb +++ b/spec/ruby/library/net-http/httpresponse/read_body_spec.rb @@ -5,7 +5,7 @@ describe "Net::HTTPResponse#read_body" do before :each do @res = Net::HTTPUnknownResponse.new("1.0", "???", "test response") - @socket = Net::BufferedIO.new(StringIO.new(+"test body")) + @socket = Net::BufferedIO.new(StringIO.new("test body")) end describe "when passed no arguments" do diff --git a/spec/ruby/library/net-http/httpresponse/reading_body_spec.rb b/spec/ruby/library/net-http/httpresponse/reading_body_spec.rb index b9ab112c964504..637a2806f89fa0 100644 --- a/spec/ruby/library/net-http/httpresponse/reading_body_spec.rb +++ b/spec/ruby/library/net-http/httpresponse/reading_body_spec.rb @@ -5,7 +5,7 @@ describe "Net::HTTPResponse#reading_body" do before :each do @res = Net::HTTPUnknownResponse.new("1.0", "???", "test response") - @socket = Net::BufferedIO.new(StringIO.new(+"test body")) + @socket = Net::BufferedIO.new(StringIO.new("test body")) end describe "when body_allowed is true" do diff --git a/spec/ruby/library/net-http/httpresponse/shared/body.rb b/spec/ruby/library/net-http/httpresponse/shared/body.rb index f35ca3200cbb2c..618e3936fb5fe2 100644 --- a/spec/ruby/library/net-http/httpresponse/shared/body.rb +++ b/spec/ruby/library/net-http/httpresponse/shared/body.rb @@ -3,7 +3,7 @@ describe :net_httpresponse_body, shared: true do before :each do @res = Net::HTTPUnknownResponse.new("1.0", "???", "test response") - @socket = Net::BufferedIO.new(StringIO.new(+"test body")) + @socket = Net::BufferedIO.new(StringIO.new("test body")) end it "returns the read body" do diff --git a/spec/ruby/library/socket/basicsocket/read_nonblock_spec.rb b/spec/ruby/library/socket/basicsocket/read_nonblock_spec.rb index df44a50afa9b77..ea5e65da5c93f7 100644 --- a/spec/ruby/library/socket/basicsocket/read_nonblock_spec.rb +++ b/spec/ruby/library/socket/basicsocket/read_nonblock_spec.rb @@ -18,7 +18,37 @@ it "receives data after it's ready" do IO.select([@r], nil, nil, 2) - @r.recv_nonblock(5).should == "aaa" + @r.read_nonblock(5).should == "aaa" + end + + platform_is_not :windows do + it 'returned data is binary encoded regardless of the external encoding' do + IO.select([@r], nil, nil, 2) + @r.read_nonblock(1).encoding.should == Encoding::BINARY + + @w.send("bbb", 0, @r.getsockname) + @r.set_encoding(Encoding::ISO_8859_1) + IO.select([@r], nil, nil, 2) + buffer = @r.read_nonblock(3) + buffer.should == "bbb" + buffer.encoding.should == Encoding::BINARY + end + end + + it 'replaces the content of the provided buffer without changing its encoding' do + buffer = "initial data".dup.force_encoding(Encoding::UTF_8) + + IO.select([@r], nil, nil, 2) + @r.read_nonblock(3, buffer) + buffer.should == "aaa" + buffer.encoding.should == Encoding::UTF_8 + + @w.send("bbb", 0, @r.getsockname) + @r.set_encoding(Encoding::ISO_8859_1) + IO.select([@r], nil, nil, 2) + @r.read_nonblock(3, buffer) + buffer.should == "bbb" + buffer.encoding.should == Encoding::UTF_8 end platform_is :linux do diff --git a/spec/ruby/library/socket/basicsocket/read_spec.rb b/spec/ruby/library/socket/basicsocket/read_spec.rb new file mode 100644 index 00000000000000..ba9de7d5cf81ba --- /dev/null +++ b/spec/ruby/library/socket/basicsocket/read_spec.rb @@ -0,0 +1,47 @@ +require_relative '../spec_helper' +require_relative '../fixtures/classes' + +describe "BasicSocket#read" do + SocketSpecs.each_ip_protocol do |family, ip_address| + before :each do + @r = Socket.new(family, :DGRAM) + @w = Socket.new(family, :DGRAM) + + @r.bind(Socket.pack_sockaddr_in(0, ip_address)) + @w.send("aaa", 0, @r.getsockname) + end + + after :each do + @r.close unless @r.closed? + @w.close unless @w.closed? + end + + it "receives data after it's ready" do + @r.read(3).should == "aaa" + end + + it 'returned data is binary encoded regardless of the external encoding' do + @r.read(3).encoding.should == Encoding::BINARY + + @w.send("bbb", 0, @r.getsockname) + @r.set_encoding(Encoding::UTF_8) + buffer = @r.read(3) + buffer.should == "bbb" + buffer.encoding.should == Encoding::BINARY + end + + it 'replaces the content of the provided buffer without changing its encoding' do + buffer = "initial data".dup.force_encoding(Encoding::UTF_8) + + @r.read(3, buffer) + buffer.should == "aaa" + buffer.encoding.should == Encoding::UTF_8 + + @w.send("bbb", 0, @r.getsockname) + @r.set_encoding(Encoding::ISO_8859_1) + @r.read(3, buffer) + buffer.should == "bbb" + buffer.encoding.should == Encoding::UTF_8 + end + end +end diff --git a/spec/ruby/library/stringio/binmode_spec.rb b/spec/ruby/library/stringio/binmode_spec.rb index 853d9c9bd66542..9e92c63814e84a 100644 --- a/spec/ruby/library/stringio/binmode_spec.rb +++ b/spec/ruby/library/stringio/binmode_spec.rb @@ -3,7 +3,7 @@ describe "StringIO#binmode" do it "returns self" do - io = StringIO.new("example") + io = StringIO.new(+"example") io.binmode.should equal(io) end diff --git a/spec/ruby/library/stringio/fcntl_spec.rb b/spec/ruby/library/stringio/fcntl_spec.rb index f252d5e73890df..a78004d868dbc8 100644 --- a/spec/ruby/library/stringio/fcntl_spec.rb +++ b/spec/ruby/library/stringio/fcntl_spec.rb @@ -3,6 +3,6 @@ describe "StringIO#fcntl" do it "raises a NotImplementedError" do - -> { StringIO.new(+"boom").fcntl }.should raise_error(NotImplementedError) + -> { StringIO.new("boom").fcntl }.should raise_error(NotImplementedError) end end diff --git a/spec/ruby/library/stringio/initialize_spec.rb b/spec/ruby/library/stringio/initialize_spec.rb index d1dff590bb1b08..ad067a0be17ca8 100644 --- a/spec/ruby/library/stringio/initialize_spec.rb +++ b/spec/ruby/library/stringio/initialize_spec.rb @@ -172,7 +172,7 @@ # NOTE: Synchronise with core/io/new_spec.rb (core/io/shared/new.rb) describe "StringIO#initialize when passed keyword arguments" do it "sets the mode based on the passed :mode option" do - io = StringIO.new(+"example", "r") + io = StringIO.new("example", "r") io.closed_read?.should be_false io.closed_write?.should be_true end diff --git a/spec/ruby/library/stringio/read_nonblock_spec.rb b/spec/ruby/library/stringio/read_nonblock_spec.rb index 8b78b1b0e4b44c..74736f7792dc86 100644 --- a/spec/ruby/library/stringio/read_nonblock_spec.rb +++ b/spec/ruby/library/stringio/read_nonblock_spec.rb @@ -33,7 +33,7 @@ describe "StringIO#read_nonblock" do it "accepts an exception option" do - stringio = StringIO.new(+'foo') + stringio = StringIO.new('foo') stringio.read_nonblock(3, exception: false).should == 'foo' end diff --git a/spec/ruby/library/stringio/reopen_spec.rb b/spec/ruby/library/stringio/reopen_spec.rb index 151bb58c6f3053..7021ff17e5cd9d 100644 --- a/spec/ruby/library/stringio/reopen_spec.rb +++ b/spec/ruby/library/stringio/reopen_spec.rb @@ -3,11 +3,11 @@ describe "StringIO#reopen when passed [Object, Integer]" do before :each do - @io = StringIO.new(+"example") + @io = StringIO.new("example") end it "reopens self with the passed Object in the passed mode" do - @io.reopen(+"reopened", IO::RDONLY) + @io.reopen("reopened", IO::RDONLY) @io.closed_read?.should be_false @io.closed_write?.should be_true @io.string.should == "reopened" @@ -51,11 +51,11 @@ describe "StringIO#reopen when passed [Object, Object]" do before :each do - @io = StringIO.new(+"example") + @io = StringIO.new("example") end it "reopens self with the passed Object in the passed mode" do - @io.reopen(+"reopened", "r") + @io.reopen("reopened", "r") @io.closed_read?.should be_false @io.closed_write?.should be_true @io.string.should == "reopened" @@ -83,7 +83,7 @@ it "tries to convert the passed Object to a String using #to_str" do obj = mock("to_str") - obj.should_receive(:to_str).and_return(+"to_str") + obj.should_receive(:to_str).and_return("to_str") @io.reopen(obj, "r") @io.string.should == "to_str" end @@ -107,7 +107,7 @@ it "tries to convert the passed mode Object to an Integer using #to_str" do obj = mock("to_str") obj.should_receive(:to_str).and_return("r") - @io.reopen(+"reopened", obj) + @io.reopen("reopened", obj) @io.closed_read?.should be_false @io.closed_write?.should be_true @io.string.should == "reopened" @@ -128,7 +128,7 @@ describe "StringIO#reopen when passed [String]" do before :each do - @io = StringIO.new(+"example") + @io = StringIO.new("example") end it "reopens self with the passed String in read-write mode" do @@ -157,7 +157,7 @@ describe "StringIO#reopen when passed [Object]" do before :each do - @io = StringIO.new(+"example") + @io = StringIO.new("example") end it "raises a TypeError when passed an Object that can't be converted to a StringIO" do @@ -180,7 +180,7 @@ describe "StringIO#reopen when passed no arguments" do before :each do - @io = StringIO.new(+"example\nsecond line") + @io = StringIO.new("example\nsecond line") end it "resets self's mode to read-write" do @@ -208,7 +208,7 @@ # for details. describe "StringIO#reopen" do before :each do - @io = StringIO.new(+'hello','a') + @io = StringIO.new(+'hello', 'a') end # TODO: find out if this is really a bug diff --git a/spec/ruby/optional/capi/ext/gc_spec.c b/spec/ruby/optional/capi/ext/gc_spec.c index 7dd0b876550ec4..1392bc6ee668da 100644 --- a/spec/ruby/optional/capi/ext/gc_spec.c +++ b/spec/ruby/optional/capi/ext/gc_spec.c @@ -8,7 +8,12 @@ extern "C" { VALUE registered_tagged_value; VALUE registered_reference_value; VALUE registered_before_rb_gc_register_address; -VALUE registered_before_rb_global_variable; +VALUE registered_before_rb_global_variable_string; +VALUE registered_before_rb_global_variable_bignum; +VALUE registered_before_rb_global_variable_float; +VALUE registered_after_rb_global_variable_string; +VALUE registered_after_rb_global_variable_bignum; +VALUE registered_after_rb_global_variable_float; VALUE rb_gc_register_address_outside_init; static VALUE registered_tagged_address(VALUE self) { @@ -23,8 +28,28 @@ static VALUE get_registered_before_rb_gc_register_address(VALUE self) { return registered_before_rb_gc_register_address; } -static VALUE get_registered_before_rb_global_variable(VALUE self) { - return registered_before_rb_global_variable; +static VALUE get_registered_before_rb_global_variable_string(VALUE self) { + return registered_before_rb_global_variable_string; +} + +static VALUE get_registered_before_rb_global_variable_bignum(VALUE self) { + return registered_before_rb_global_variable_bignum; +} + +static VALUE get_registered_before_rb_global_variable_float(VALUE self) { + return registered_before_rb_global_variable_float; +} + +static VALUE get_registered_after_rb_global_variable_string(VALUE self) { + return registered_after_rb_global_variable_string; +} + +static VALUE get_registered_after_rb_global_variable_bignum(VALUE self) { + return registered_after_rb_global_variable_bignum; +} + +static VALUE get_registered_after_rb_global_variable_float(VALUE self) { + return registered_after_rb_global_variable_float; } static VALUE gc_spec_rb_gc_register_address(VALUE self) { @@ -71,17 +96,34 @@ void Init_gc_spec(void) { rb_gc_register_address(®istered_tagged_value); rb_gc_register_address(®istered_reference_value); rb_gc_register_address(®istered_before_rb_gc_register_address); - rb_global_variable(®istered_before_rb_global_variable); + rb_global_variable(®istered_before_rb_global_variable_string); + rb_global_variable(®istered_before_rb_global_variable_bignum); + rb_global_variable(®istered_before_rb_global_variable_float); registered_tagged_value = INT2NUM(10); registered_reference_value = rb_str_new2("Globally registered data"); registered_before_rb_gc_register_address = rb_str_new_cstr("registered before rb_gc_register_address()"); - registered_before_rb_global_variable = rb_str_new_cstr("registered before rb_global_variable()"); + + registered_before_rb_global_variable_string = rb_str_new_cstr("registered before rb_global_variable()"); + registered_before_rb_global_variable_bignum = LL2NUM(INT64_MAX); + registered_before_rb_global_variable_float = DBL2NUM(3.14); + + registered_after_rb_global_variable_string = rb_str_new_cstr("registered after rb_global_variable()"); + rb_global_variable(®istered_after_rb_global_variable_string); + registered_after_rb_global_variable_bignum = LL2NUM(INT64_MAX); + rb_global_variable(®istered_after_rb_global_variable_bignum); + registered_after_rb_global_variable_float = DBL2NUM(6.28); + rb_global_variable(®istered_after_rb_global_variable_float); rb_define_method(cls, "registered_tagged_address", registered_tagged_address, 0); rb_define_method(cls, "registered_reference_address", registered_reference_address, 0); rb_define_method(cls, "registered_before_rb_gc_register_address", get_registered_before_rb_gc_register_address, 0); - rb_define_method(cls, "registered_before_rb_global_variable", get_registered_before_rb_global_variable, 0); + rb_define_method(cls, "registered_before_rb_global_variable_string", get_registered_before_rb_global_variable_string, 0); + rb_define_method(cls, "registered_before_rb_global_variable_bignum", get_registered_before_rb_global_variable_bignum, 0); + rb_define_method(cls, "registered_before_rb_global_variable_float", get_registered_before_rb_global_variable_float, 0); + rb_define_method(cls, "registered_after_rb_global_variable_string", get_registered_after_rb_global_variable_string, 0); + rb_define_method(cls, "registered_after_rb_global_variable_bignum", get_registered_after_rb_global_variable_bignum, 0); + rb_define_method(cls, "registered_after_rb_global_variable_float", get_registered_after_rb_global_variable_float, 0); rb_define_method(cls, "rb_gc_register_address", gc_spec_rb_gc_register_address, 0); rb_define_method(cls, "rb_gc_unregister_address", gc_spec_rb_gc_unregister_address, 0); rb_define_method(cls, "rb_gc_enable", gc_spec_rb_gc_enable, 0); diff --git a/spec/ruby/optional/capi/gc_spec.rb b/spec/ruby/optional/capi/gc_spec.rb index d76ea7394f304e..aaced56483a0cc 100644 --- a/spec/ruby/optional/capi/gc_spec.rb +++ b/spec/ruby/optional/capi/gc_spec.rb @@ -27,9 +27,36 @@ end describe "rb_global_variable" do - it "keeps the value alive even if the value is assigned after rb_global_variable() is called" do + before :all do GC.start - @f.registered_before_rb_global_variable.should == "registered before rb_global_variable()" + end + + describe "keeps the value alive even if the value is assigned after rb_global_variable() is called" do + it "for a string" do + @f.registered_before_rb_global_variable_string.should == "registered before rb_global_variable()" + end + + it "for a bignum" do + @f.registered_before_rb_global_variable_bignum.should == 2**63 - 1 + end + + it "for a Float" do + @f.registered_before_rb_global_variable_float.should == 3.14 + end + end + + describe "keeps the value alive when the value is assigned before rb_global_variable() is called" do + it "for a string" do + @f.registered_after_rb_global_variable_string.should == "registered after rb_global_variable()" + end + + it "for a bignum" do + @f.registered_after_rb_global_variable_bignum.should == 2**63 - 1 + end + + it "for a Float" do + @f.registered_after_rb_global_variable_float.should == 6.28 + end end end diff --git a/string.c b/string.c index 49ab197834b716..b4c2d05e6ded75 100644 --- a/string.c +++ b/string.c @@ -1158,6 +1158,7 @@ str_cat_conv_enc_opts(VALUE newstr, long ofs, const char *ptr, long len, rb_str_resize(newstr, olen); } DATA_PTR(econv_wrapper) = 0; + RB_GC_GUARD(econv_wrapper); rb_econv_close(ec); switch (ret) { case econv_finished: diff --git a/test/.excludes-prism/TestAssignmentGen.rb b/test/.excludes-prism/TestAssignmentGen.rb index 7d9ffd235cb93f..5f3bb12ef7c22d 100644 --- a/test/.excludes-prism/TestAssignmentGen.rb +++ b/test/.excludes-prism/TestAssignmentGen.rb @@ -1 +1 @@ -exclude(:test_assignment, "unknown") +exclude(:test_assignment, "https://github.com/ruby/prism/issues/2370") diff --git a/test/.excludes-prism/TestCall.rb b/test/.excludes-prism/TestCall.rb new file mode 100644 index 00000000000000..969e32ea5a6054 --- /dev/null +++ b/test/.excludes-prism/TestCall.rb @@ -0,0 +1,3 @@ +exclude(:test_call_op_asgn_keywords, "https://github.com/ruby/prism/issues/2438") +exclude(:test_call_op_asgn_keywords_mutable, "https://github.com/ruby/prism/issues/2438") +exclude(:test_kwsplat_block_order_op_asgn, "https://github.com/ruby/prism/issues/2438") diff --git a/test/.excludes-prism/TestClass.rb b/test/.excludes-prism/TestClass.rb new file mode 100644 index 00000000000000..9681f78f4233bb --- /dev/null +++ b/test/.excludes-prism/TestClass.rb @@ -0,0 +1,3 @@ +exclude(:test_invalid_break_from_class_definition, "https://github.com/ruby/prism/issues/1604") +exclude(:test_invalid_next_from_class_definition, "https://github.com/ruby/prism/issues/1604") +exclude(:test_invalid_return_from_class_definition, "https://github.com/ruby/prism/issues/1604") diff --git a/test/.excludes-prism/TestEval.rb b/test/.excludes-prism/TestEval.rb deleted file mode 100644 index 6cc6bdfb1d65e6..00000000000000 --- a/test/.excludes-prism/TestEval.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_file_encoding, "incorrect encoding") diff --git a/test/.excludes-prism/TestException.rb b/test/.excludes-prism/TestException.rb deleted file mode 100644 index 3f6d0e3b812f7e..00000000000000 --- a/test/.excludes-prism/TestException.rb +++ /dev/null @@ -1 +0,0 @@ -exclude(:test_syntax_error_path, "unknown") diff --git a/test/.excludes-prism/TestIRB/RubyLexTest.rb b/test/.excludes-prism/TestIRB/RubyLexTest.rb new file mode 100644 index 00000000000000..2274ae62cfd589 --- /dev/null +++ b/test/.excludes-prism/TestIRB/RubyLexTest.rb @@ -0,0 +1 @@ +exclude(:test_code_block_open_with_should_continue, "symbol encoding") diff --git a/test/.excludes-prism/TestISeq.rb b/test/.excludes-prism/TestISeq.rb index de7052cb112d1f..a40c0e49a12417 100644 --- a/test/.excludes-prism/TestISeq.rb +++ b/test/.excludes-prism/TestISeq.rb @@ -1 +1,7 @@ -exclude(:test_syntax_error_message, "unknown") +exclude(:test_each_child, "https://github.com/ruby/prism/issues/2660") +exclude(:test_frozen_string_literal_compile_option, "https://github.com/ruby/prism/issues/2661") +exclude(:test_syntax_error_message, "Assertion checks against specific error format") +exclude(:test_to_binary_class_tracepoint, "https://github.com/ruby/prism/issues/2662") +exclude(:test_to_binary_end_tracepoint, "https://github.com/ruby/prism/issues/2663") +exclude(:test_trace_points, "https://github.com/ruby/prism/issues/2660") +exclude(:test_unreachable_syntax_error, "https://github.com/ruby/prism/issues/1604") diff --git a/test/.excludes-prism/TestM17N.rb b/test/.excludes-prism/TestM17N.rb index e38425f63dd215..e9e0623689f379 100644 --- a/test/.excludes-prism/TestM17N.rb +++ b/test/.excludes-prism/TestM17N.rb @@ -1,8 +1,8 @@ -exclude(:test_dynamic_eucjp_regexp, "https://github.com/ruby/prism/issues/1997") -exclude(:test_dynamic_sjis_regexp, "https://github.com/ruby/prism/issues/1997") -exclude(:test_dynamic_utf8_regexp, "https://github.com/ruby/prism/issues/1997") -exclude(:test_regexp_ascii, "https://github.com/ruby/prism/issues/1997") -exclude(:test_regexp_embed, "https://github.com/ruby/prism/issues/1997") +exclude(:test_dynamic_eucjp_regexp, "https://github.com/ruby/prism/issues/2664") +exclude(:test_dynamic_sjis_regexp, "https://github.com/ruby/prism/issues/2664") +exclude(:test_dynamic_utf8_regexp, "https://github.com/ruby/prism/issues/2664") +exclude(:test_regexp_ascii, "https://github.com/ruby/prism/issues/2664") +exclude(:test_regexp_embed, "https://github.com/ruby/prism/issues/2664") exclude(:test_regexp_mixed_unicode, "unknown") exclude(:test_regexp_too_short_multibyte_character, "unknown") exclude(:test_regexp_unicode, "unknown") diff --git a/test/.excludes-prism/TestParse.rb b/test/.excludes-prism/TestParse.rb index 87694ac15a9851..9c60ab15cf6ecc 100644 --- a/test/.excludes-prism/TestParse.rb +++ b/test/.excludes-prism/TestParse.rb @@ -1,10 +1,35 @@ -exclude(:test_assign_in_conditional, "missing warning") -exclude(:test_magic_comment, "incorrect encoding") +exclude(:test_disallowed_gloal_variable, "unknown") +exclude(:test_dynamic_constant_assignment, "unknown") +exclude(:test_else_without_rescue, "unknown") +exclude(:test_embedded_rd_error, "unknown") +exclude(:test_embedded_rd, "unknown") +exclude(:test_error_def_in_argument, "unknown") +exclude(:test_float, "unknown") +exclude(:test_global_variable, "unknown") +exclude(:test_here_document, "unknown") +exclude(:test_heredoc_unterminated_interpolation, "unknown") +exclude(:test_invalid_break_from_class_definition, "unknown") +exclude(:test_invalid_char, "unknown") +exclude(:test_invalid_next_from_class_definition, "unknown") +exclude(:test_location_of_invalid_token, "unknown") +exclude(:test_no_blockarg, "unknown") +exclude(:test_op_asgn1_with_block, "unknown") +exclude(:test_parse_string, "unknown") +exclude(:test_percent, "unknown") +exclude(:test_question, "unknown") +exclude(:test_shareable_constant_value_ignored, "unknown") exclude(:test_shareable_constant_value_nested, "ractor support") exclude(:test_shareable_constant_value_nonliteral, "ractor support") exclude(:test_shareable_constant_value_simple, "ractor support") exclude(:test_shareable_constant_value_unfrozen, "ractor support") exclude(:test_shareable_constant_value_unshareable_literal, "ractor support") +exclude(:test_string, "unknown") +exclude(:test_truncated_source_line, "unknown") +exclude(:test_unassignable, "unknown") +exclude(:test_unexpected_eof, "unknown") +exclude(:test_unexpected_token_after_numeric, "unknown") +exclude(:test_unterminated_regexp_error, "unknown") exclude(:test_unused_variable, "missing warning") -exclude(:test_utf8_bom, "incorrect error") exclude(:test_void_expr_stmts_value, "missing warning") +exclude(:test_void_value_in_rhs, "unknown") +exclude(:test_words, "unknown") diff --git a/test/.excludes-prism/TestPatternMatching.rb b/test/.excludes-prism/TestPatternMatching.rb index cb64a6d818787c..cfd0c6bed97058 100644 --- a/test/.excludes-prism/TestPatternMatching.rb +++ b/test/.excludes-prism/TestPatternMatching.rb @@ -1 +1,2 @@ exclude(:test_hash_pattern, "useless literal warning missing") +exclude(:test_invalid_syntax, "[a:] is disallowed") diff --git a/test/.excludes-prism/TestRegexp.rb b/test/.excludes-prism/TestRegexp.rb index f2b817d79a7266..68ad1414a9836b 100644 --- a/test/.excludes-prism/TestRegexp.rb +++ b/test/.excludes-prism/TestRegexp.rb @@ -1,7 +1,6 @@ -exclude(:test_unicode_age_14_0, "unknown") -exclude(:test_invalid_fragment, "unknown") -exclude(:test_assign_named_capture_to_reserved_word, "unknown") -exclude(:test_unicode_age_15_0, "unknown") -exclude(:test_unescape, "unknown") exclude(:test_invalid_escape_error, "unknown") -exclude(:test_unicode_age, "unknown") +exclude(:test_invalid_fragment, "https://github.com/ruby/prism/issues/2664") +exclude(:test_unescape, "unknown") +exclude(:test_unicode_age_14_0, "https://github.com/ruby/prism/issues/2664") +exclude(:test_unicode_age_15_0, "https://github.com/ruby/prism/issues/2664") +exclude(:test_unicode_age, "https://github.com/ruby/prism/issues/2664") diff --git a/test/.excludes-prism/TestRubyLiteral.rb b/test/.excludes-prism/TestRubyLiteral.rb index 4cfe25215e1190..bf5dbcd36a4909 100644 --- a/test/.excludes-prism/TestRubyLiteral.rb +++ b/test/.excludes-prism/TestRubyLiteral.rb @@ -1,3 +1,6 @@ -exclude(:test_dregexp, "unknown") -exclude(:test_hash_duplicated_key, "parser implementation dependent") +exclude(:test_debug_frozen_string_in_array_literal, "unknown") +exclude(:test_debug_frozen_string, "unknown") +exclude(:test_dregexp, "https://github.com/ruby/prism/issues/2664") +exclude(:test_hash_value_omission, "unknown") +exclude(:test_integer, "unknown") exclude(:test_string, "https://github.com/ruby/prism/issues/2331") diff --git a/test/.excludes-prism/TestRubyOptimization.rb b/test/.excludes-prism/TestRubyOptimization.rb new file mode 100644 index 00000000000000..df22ca4f714860 --- /dev/null +++ b/test/.excludes-prism/TestRubyOptimization.rb @@ -0,0 +1 @@ +exclude(:test_peephole_string_literal_range, "unknown") diff --git a/test/.excludes-prism/TestRubyVM.rb b/test/.excludes-prism/TestRubyVM.rb new file mode 100644 index 00000000000000..6d4c3ca6fe4513 --- /dev/null +++ b/test/.excludes-prism/TestRubyVM.rb @@ -0,0 +1 @@ +exclude(:test_keep_script_lines, "unknown") diff --git a/test/.excludes-prism/TestSyntax.rb b/test/.excludes-prism/TestSyntax.rb index 680974906bcb6f..f2f13e3161c674 100644 --- a/test/.excludes-prism/TestSyntax.rb +++ b/test/.excludes-prism/TestSyntax.rb @@ -1,4 +1,36 @@ +exclude(:test__END___cr, "unknown") +exclude(:test_anonymous_block_forwarding, "unknown") +exclude(:test_anonymous_keyword_rest_forwarding, "unknown") +exclude(:test_anonymous_rest_forwarding, "unknown") +exclude(:test_argument_forwarding_with_anon_rest_kwrest_and_block, "unknown") +exclude(:test_argument_forwarding_with_super, "unknown") +exclude(:test_argument_forwarding, "unknown") +exclude(:test_brace_after_literal_argument, "unknown") exclude(:test_dedented_heredoc_concatenation, "unknown") exclude(:test_dedented_heredoc_continued_line, "unknown") +exclude(:test_dedented_heredoc_invalid_identifer, "unknown") exclude(:test_duplicated_when, "unknown") +exclude(:test_error_message_encoding, "unknown") +exclude(:test_heredoc_cr, "unknown") +exclude(:test_heredoc_no_terminator, "unknown") +exclude(:test_invalid_break, "unknown") +exclude(:test_invalid_encoding_symbol, "unknown") +exclude(:test_invalid_literal_message, "unknown") +exclude(:test_invalid_next, "unknown") exclude(:test_it, "https://github.com/ruby/prism/issues/2323") +exclude(:test_keyword_invalid_name, "unknown") +exclude(:test_keyword_self_reference, "unknown") +exclude(:test_keywords_specified_and_not_accepted, "unknown") +exclude(:test_methoddef_endless_command, "unknown") +exclude(:test_numbered_parameter, "unknown") +exclude(:test_optional_self_reference, "unknown") +exclude(:test_parenthesised_statement_argument, "unknown") +exclude(:test_range_at_eol, "unknown") +exclude(:test_safe_call_in_massign_lhs, "unknown") +exclude(:test_syntax_error_at_newline, "unknown") +exclude(:test_unassignable, "unknown") +exclude(:test_unexpected_fraction, "unknown") +exclude(:test_unterminated_heredoc_cr, "unknown") +exclude(:test_unterminated_heredoc, "unknown") +exclude(:test_warn_balanced, "unknown") +exclude(:test_warn_unreachable, "unknown") diff --git a/test/mkmf/test_config.rb b/test/mkmf/test_config.rb index 7211c61d53ef8a..0d2cb3751ccc65 100644 --- a/test/mkmf/test_config.rb +++ b/test/mkmf/test_config.rb @@ -1,5 +1,6 @@ # frozen_string_literal: false require_relative 'base' +require 'tempfile' class TestMkmfConfig < TestMkmf def test_dir_config @@ -27,4 +28,30 @@ def test_with_config_without_arg assert_equal(false, with_config("foo")) end; end + + def test_with_target_rbconfig + Tempfile.create(%w"rbconfig .rb", ".") do |tmp| + tmp.puts <<~'end;' + module RbConfig + CONFIG = {} + MAKEFILE_CONFIG = {} + + def self.fire_update!(key, value); end + def self.expand(val, config = CONFIG); val; end + end; + ::RbConfig::CONFIG.each do |k, v| + tmp.puts " CONFIG[#{k.dump}] = #{v.dump}" + end + ::RbConfig::MAKEFILE_CONFIG.each do |k, v| + tmp.puts " MAKEFILE_CONFIG[#{k.dump}] = #{v.dump}" + end + tmp.puts " CONFIG['testing-only'] = 'ok'" + tmp.puts "end" + tmp.close + assert_separately([], ["--target-rbconfig=#{tmp.path}"], <<-'end;') + assert_equal("ok", MakeMakefile::RbConfig::CONFIG["testing-only"]) + assert_not_equal(::RbConfig, MakeMakefile::RbConfig) + end; + end + end end diff --git a/test/openssl/test_ssl.rb b/test/openssl/test_ssl.rb index dcb7757add02be..66d63a981d08c2 100644 --- a/test/openssl/test_ssl.rb +++ b/test/openssl/test_ssl.rb @@ -691,7 +691,7 @@ def test_verify_wildcard assert_equal(true, OpenSSL::SSL.verify_wildcard("xn--qdk4b9b", "xn--qdk4b9b")) end - # Comments in this test is excerpted from http://tools.ietf.org/html/rfc6125#page-27 + # Comments in this test is excerpted from https://www.rfc-editor.org/rfc/rfc6125#page-27 def test_post_connection_check_wildcard_san # case-insensitive ASCII comparison # RFC 6125, section 6.4.1 diff --git a/test/prism/command_line_test.rb b/test/prism/command_line_test.rb index 57ab0dee45e944..4b04c36f3aa5e2 100644 --- a/test/prism/command_line_test.rb +++ b/test/prism/command_line_test.rb @@ -65,5 +65,47 @@ def test_command_line_e result = Prism.parse("1 if 2..3", command_line: "e") assert_equal 0, result.warnings.length end + + def test_command_line_x_implicit + result = Prism.parse(<<~RUBY) + #!/bin/bash + exit 1 + + #!/usr/bin/env ruby + 1 + RUBY + + assert_kind_of IntegerNode, result.value.statements.body.first + end + + def test_command_line_x_explicit + result = Prism.parse(<<~RUBY, command_line: "x") + exit 1 + + #!/usr/bin/env ruby + 1 + RUBY + + assert_kind_of IntegerNode, result.value.statements.body.first + end + + def test_command_line_x_implicit_fail + result = Prism.parse(<<~RUBY) + #!/bin/bash + exit 1 + RUBY + + assert_equal 1, result.errors.length + assert_equal :load, result.errors.first.level + end + + def test_command_line_x_explicit_fail + result = Prism.parse(<<~RUBY, command_line: "x") + exit 1 + RUBY + + assert_equal 1, result.errors.length + assert_equal :load, result.errors.first.level + end end end diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb index 6e6e74ee5d5a1d..dbc4a8924d8565 100644 --- a/test/prism/errors_test.rb +++ b/test/prism/errors_test.rb @@ -26,7 +26,7 @@ def test_module_name_recoverable ) assert_errors expected, "module Parent module end", [ - ["expected a constant name after `module`", 14..20], + ["unexpected constant path after `module`; class/module name must be CONSTANT", 14..20], ["unexpected 'end', assuming it is closing the parent module definition", 21..24] ] end @@ -177,7 +177,7 @@ def test_unterminated_empty_string def test_incomplete_instance_var_string assert_errors expression('%@#@@#'), '%@#@@#', [ - ["'@#' is not allowed as an instance variable name", 4..5], + ["'@' without identifiers is not allowed as an instance variable name", 4..5], ["unexpected instance variable, expecting end-of-input", 4..5] ] end @@ -325,7 +325,7 @@ def test_aliasing_non_global_variable_with_global_variable def test_aliasing_global_variable_with_global_number_variable assert_errors expression("alias $a $1"), "alias $a $1", [ - ["invalid argument being passed to `alias`; expected a bare word, symbol, constant, or global variable", 9..11] + ["invalid argument being passed to `alias`; can't make alias for the number variables", 9..11] ] end @@ -452,7 +452,7 @@ def test_module_definition_in_method_body ) assert_errors expected, "def foo;module A;end;end", [ - ["unexpected module definition in a method definition", 8..14] + ["unexpected module definition in method body", 8..14] ] end @@ -490,7 +490,7 @@ def test_module_definition_in_method_body_within_block Location() ) - assert_errors expected, <<~RUBY, [["unexpected module definition in a method definition", 21..27]] + assert_errors expected, <<~RUBY, [["unexpected module definition in method body", 21..27]] def foo bar do module Foo;end @@ -505,7 +505,7 @@ def foo(bar = module A;end);end def foo;rescue;module A;end;end def foo;ensure;module A;end;end RUBY - message = "unexpected module definition in a method definition" + message = "unexpected module definition in method body" assert_errors expression(source), source, [ [message, 14..20], [message, 47..53], @@ -541,7 +541,7 @@ def test_class_definition_in_method_body ) assert_errors expected, "def foo;class A;end;end", [ - ["unexpected class definition in a method definition", 8..13] + ["unexpected class definition in method body", 8..13] ] end @@ -551,7 +551,7 @@ def foo(bar = class A;end);end def foo;rescue;class A;end;end def foo;ensure;class A;end;end RUBY - message = "unexpected class definition in a method definition" + message = "unexpected class definition in method body" assert_errors expression(source), source, [ [message, 14..19], [message, 46..51], @@ -1130,28 +1130,24 @@ def test_dont_allow_setting_to_back_and_nth_reference end def test_duplicated_parameter_names - # For some reason, Ripper reports no error for Ruby 3.0 when you have - # duplicated parameter names for positional parameters. - unless RUBY_VERSION < "3.1.0" - expected = DefNode( - :foo, - Location(), - nil, - ParametersNode([RequiredParameterNode(0, :a), RequiredParameterNode(0, :b), RequiredParameterNode(ParameterFlags::REPEATED_PARAMETER, :a)], [], nil, [], [], nil, nil), - nil, - [:a, :b], - Location(), - nil, - Location(), - Location(), - nil, - Location() - ) + expected = DefNode( + :foo, + Location(), + nil, + ParametersNode([RequiredParameterNode(0, :a), RequiredParameterNode(0, :b), RequiredParameterNode(ParameterFlags::REPEATED_PARAMETER, :a)], [], nil, [], [], nil, nil), + nil, + [:a, :b], + Location(), + nil, + Location(), + Location(), + nil, + Location() + ) - assert_errors expected, "def foo(a,b,a);end", [ - ["repeated parameter name", 12..13] - ] - end + assert_errors expected, "def foo(a,b,a);end", [ + ["duplicated argument name", 12..13] + ] expected = DefNode( :foo, @@ -1169,7 +1165,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,*a);end", [ - ["repeated parameter name", 13..14] + ["duplicated argument name", 13..14] ] expected = DefNode( @@ -1188,7 +1184,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,**a);end", [ - ["repeated parameter name", 14..15] + ["duplicated argument name", 14..15] ] expected = DefNode( @@ -1207,7 +1203,7 @@ def test_duplicated_parameter_names ) assert_errors expected, "def foo(a,b,&a);end", [ - ["repeated parameter name", 13..14] + ["duplicated argument name", 13..14] ] expected = DefNode( @@ -1258,7 +1254,7 @@ def test_invalid_operator_write_dot def test_unterminated_global_variable assert_errors expression("$"), "$", [ - ["'$' is not allowed as a global variable name", 0..1] + ["'$' without identifiers is not allowed as a global variable name", 0..1] ] end @@ -1370,7 +1366,7 @@ def test_double_scope_numbered_parameters source = "-> { _1 + -> { _2 } }" errors = [["numbered parameter is already used in outer scope", 15..17]] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_invalid_number_underscores @@ -1436,13 +1432,13 @@ def test_parameter_name_ending_with_bang_or_question_mark ["unexpected name for a parameter", 8..10], ["unexpected name for a parameter", 11..13] ] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_class_name source = "class 0.X end" assert_errors expression(source), source, [ - ["expected a constant name after `class`", 6..9], + ["unexpected constant path after `class`; class/module name must be CONSTANT", 6..9], ] end @@ -1482,17 +1478,16 @@ def test_shadow_args_in_lambda def test_shadow_args_in_block source = "tap{|a;a|}" assert_errors expression(source), source, [ - ["repeated parameter name", 7..8], + ["duplicated argument name", 7..8], ] end def test_repeated_parameter_name_in_destructured_params source = "def f(a, (b, (a))); end" - # In Ruby 3.0.x, `Ripper.sexp_raw` does not return `nil` for this case. - compare_ripper = RUBY_ENGINE == "ruby" && (RUBY_VERSION.split('.').map { |x| x.to_i } <=> [3, 1]) >= 1 + assert_errors expression(source), source, [ - ["repeated parameter name", 14..15], - ], compare_ripper: compare_ripper + ["duplicated argument name", 14..15], + ] end def test_assign_to_numbered_parameter @@ -1574,6 +1569,7 @@ def test_check_value_expression 1 => ^(if 1; (return) else (return) end) 1 => ^(unless 1; (return) else (return) end) RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 7..13], @@ -1584,7 +1580,7 @@ def test_check_value_expression [message, 97..103], [message, 123..129], [message, 168..174], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_statement @@ -1607,6 +1603,7 @@ class << (return) for x in (return) end RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], @@ -1617,7 +1614,7 @@ class << (return) [message, 110..116], [message, 132..138], [message, 154..160], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_def @@ -1629,12 +1626,13 @@ def x(a = return) def x(a: return) end RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 5..11], [message, 29..35], [message, 50..56], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_assignment @@ -1644,13 +1642,14 @@ def test_void_value_expression_in_assignment a, b = return, 1 a, b = 1, *return RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], [message, 18..24], [message, 32..38], [message, 53..59], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_modifier @@ -1662,6 +1661,7 @@ def test_void_value_expression_in_modifier (return) => a (return) in a RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 6..12], @@ -1670,7 +1670,7 @@ def test_void_value_expression_in_modifier [message, 58..64], [message, 67..73], [message, 81..87] - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_expression @@ -1685,6 +1685,7 @@ def test_void_value_expression_in_expression ((return)..) ((return)...) RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], @@ -1696,7 +1697,7 @@ def test_void_value_expression_in_expression [message, 85..91], [message, 96..102], [message, 109..115] - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_array @@ -1709,6 +1710,7 @@ def test_void_value_expression_in_array [ *return ] [ **return ] RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], @@ -1718,7 +1720,7 @@ def test_void_value_expression_in_array [message, 58..64], [message, 70..76], [message, 83..89], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_hash @@ -1728,13 +1730,14 @@ def test_void_value_expression_in_hash { a: return } { **return } RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 2..8], [message, 23..29], [message, 37..43], [message, 50..56], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_call @@ -1745,6 +1748,7 @@ def test_void_value_expression_in_call (return)[1] = 2 (return)::foo RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], @@ -1752,7 +1756,7 @@ def test_void_value_expression_in_call [message, 27..33], [message, 39..45], [message, 55..61], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_constant_path @@ -1760,11 +1764,12 @@ def test_void_value_expression_in_constant_path (return)::A class (return)::A; end RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 1..7], [message, 19..25], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_arguments @@ -1778,6 +1783,7 @@ def test_void_value_expression_in_arguments foo(:a => return) foo(a: return) RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 4..10], @@ -1788,7 +1794,7 @@ def test_void_value_expression_in_arguments [message, 71..77], [message, 94..100], [message, 109..115], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_unary_call @@ -1796,11 +1802,12 @@ def test_void_value_expression_in_unary_call +(return) not return RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 2..8], [message, 14..20], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_void_value_expression_in_binary_call @@ -1812,13 +1819,14 @@ def test_void_value_expression_in_binary_call 1 or (return) (return) or 1 RUBY + message = 'unexpected void value expression' assert_errors expression(source), source, [ [message, 5..11], [message, 14..20], [message, 42..48], [message, 71..77], - ], compare_ripper: false # Ripper does not check 'void value expression'. + ] end def test_trailing_comma_in_calls @@ -1934,13 +1942,14 @@ def foo(bar: bar) = 42 proc { |foo = foo| } proc { |foo: foo| } RUBY + message = 'parameter default value references itself' assert_errors expression(source), source, [ [message, 14..17], [message, 37..40], [message, 61..64], [message, 81..84], - ], compare_ripper: false # Ripper does not check 'circular reference'. + ] end def test_command_calls @@ -1976,8 +1985,9 @@ def a = b rescue c d begin; rescue a b; end begin; rescue a b => c; end RUBY + sources.each do |source| - assert_nil Ripper.sexp_raw(source) + refute_valid_syntax(source) assert_false(Prism.parse(source).success?) end end @@ -1993,8 +2003,9 @@ def test_range_and_bin_op 1.. % 2 1.. ** 2 RUBY + sources.each do |source| - assert_nil Ripper.sexp_raw(source) + refute_valid_syntax(source) assert_false(Prism.parse(source).success?) end end @@ -2049,22 +2060,22 @@ def test_non_assoc_equality def test_block_arg_and_block source = 'foo(&1) { }' assert_errors expression(source), source, [ - ['multiple block arguments; only one block is allowed', 8..11] - ], compare_ripper: false # Ripper does not check 'both block arg and actual block given'. + ["both block arg and actual block given; only one block is allowed", 8..11] + ] end def test_forwarding_arg_and_block source = 'def foo(...) = foo(...) { }' assert_errors expression(source), source, [ ['both a block argument and a forwarding argument; only one block is allowed', 24..27] - ], compare_ripper: false # Ripper does not check 'both block arg and actual block given'. + ] end def test_it_with_ordinary_parameter source = "proc { || it }" errors = [["`it` is not allowed when an ordinary parameter is defined", 10..12]] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors, check_valid_syntax: RUBY_VERSION >= "3.4.0" end def test_regular_expression_with_unknown_regexp_options @@ -2124,7 +2135,7 @@ def ([1]).foo; end ["cannot define singleton method for literals", 380..388], ["cannot define singleton method for literals", 404..407] ] - assert_errors expression(source), source, errors, compare_ripper: false + assert_errors expression(source), source, errors end def test_assignment_to_literal_in_conditionals @@ -2157,11 +2168,58 @@ def test_assignment_to_literal_in_conditionals ] * source.lines.count end + def test_duplicate_pattern_capture + source = <<~RUBY + case (); in [a, a]; end + case (); in [a, *a]; end + case (); in {a: a, b: a}; end + case (); in {a: a, **a}; end + case (); in [a, {a:}]; end + case (); in [a, {a: {a: {a: [a]}}}]; end + case (); in a => a; end + case (); in [A => a, {a: b => a}]; end + RUBY + + assert_error_messages source, Array.new(source.lines.length, "duplicated variable name") + refute_error_messages "case (); in [_a, _a]; end" + end + + def test_duplicate_pattern_hash_key + assert_error_messages "case (); in {a:, a:}; end", ["duplicated key name", "duplicated variable name"] + assert_error_messages "case (); in {a:1, a:2}; end", ["duplicated key name"] + refute_error_messages "case (); in [{a:1}, {a:2}]; end" + end + private - def assert_errors(expected, source, errors, compare_ripper: RUBY_ENGINE == "ruby") - # Ripper behaves differently on JRuby/TruffleRuby, so only check this on CRuby - assert_nil Ripper.sexp_raw(source) if compare_ripper + if RUBY_ENGINE == "ruby" + def check_syntax(source) + $VERBOSE, previous = nil, $VERBOSE + + begin + RubyVM::InstructionSequence.compile(source) + ensure + $VERBOSE = previous + end + end + + def assert_valid_syntax(source) + check_syntax(source) + end + + def refute_valid_syntax(source) + assert_raise(SyntaxError) { check_syntax(source) } + end + else + def assert_valid_syntax(source) + end + + def refute_valid_syntax(source) + end + end + + def assert_errors(expected, source, errors, check_valid_syntax: true) + refute_valid_syntax(source) if check_valid_syntax result = Prism.parse(source) node = result.value.statements.body.last @@ -2170,12 +2228,17 @@ def assert_errors(expected, source, errors, compare_ripper: RUBY_ENGINE == "ruby assert_equal(errors, result.errors.map { |e| [e.message, e.location.start_offset..e.location.end_offset] }) end - def assert_error_messages(source, errors, compare_ripper: RUBY_ENGINE == "ruby") - assert_nil Ripper.sexp_raw(source) if compare_ripper + def assert_error_messages(source, errors) + refute_valid_syntax(source) result = Prism.parse(source) assert_equal(errors, result.errors.map(&:message)) end + def refute_error_messages(source) + assert_valid_syntax(source) + assert Prism.parse_success?(source) + end + def assert_warning_messages(source, warnings) result = Prism.parse(source) assert_equal(warnings, result.warnings.map(&:message)) diff --git a/test/prism/fixtures/regex.txt b/test/prism/fixtures/regex.txt index ef2f6d45a32d8d..18200e5cbd230a 100644 --- a/test/prism/fixtures/regex.txt +++ b/test/prism/fixtures/regex.txt @@ -38,3 +38,9 @@ b>)/ =~ ""; ab a = 1 tap { /(?)/ =~ to_s } + +/(?)/ =~ "" +/(?)/ =~ "" + +/(?)/ =~ "" +def foo(nil:) = /(?)/ =~ "" diff --git a/test/prism/ruby_api_test.rb b/test/prism/ruby_api_test.rb index 4153a69ad75644..49296117bf5488 100644 --- a/test/prism/ruby_api_test.rb +++ b/test/prism/ruby_api_test.rb @@ -233,21 +233,6 @@ def test_integer_base_flags assert_equal 16, base[parse_expression("0x1")] end - def test_offset - source = <<~RUBY - #!/bin/sh - - echo "foo" - exit 0 - - #!/usr/bin/env ruby - - puts "bar" - RUBY - - assert Prism.parse_success?(source, offset: 30) - end - private def parse_expression(source) diff --git a/test/prism/snapshots/regex.txt b/test/prism/snapshots/regex.txt index d07ab8c5e71900..ef576b8bc0affe 100644 --- a/test/prism/snapshots/regex.txt +++ b/test/prism/snapshots/regex.txt @@ -1,8 +1,8 @@ -@ ProgramNode (location: (1,0)-(40,24)) +@ ProgramNode (location: (1,0)-(46,32)) ├── locals: [:foo, :ab, :abc, :a] └── statements: - @ StatementsNode (location: (1,0)-(40,24)) - └── body: (length: 21) + @ StatementsNode (location: (1,0)-(46,32)) + └── body: (length: 25) ├── @ CallNode (location: (1,0)-(1,9)) │ ├── flags: ignore_visibility │ ├── receiver: ∅ @@ -316,56 +316,195 @@ │ │ ├── flags: decimal │ │ └── value: 1 │ └── operator_loc: (39,2)-(39,3) = "=" - └── @ CallNode (location: (40,0)-(40,24)) - ├── flags: ignore_visibility + ├── @ CallNode (location: (40,0)-(40,24)) + │ ├── flags: ignore_visibility + │ ├── receiver: ∅ + │ ├── call_operator_loc: ∅ + │ ├── name: :tap + │ ├── message_loc: (40,0)-(40,3) = "tap" + │ ├── opening_loc: ∅ + │ ├── arguments: ∅ + │ ├── closing_loc: ∅ + │ └── block: + │ @ BlockNode (location: (40,4)-(40,24)) + │ ├── locals: [] + │ ├── parameters: ∅ + │ ├── body: + │ │ @ StatementsNode (location: (40,6)-(40,22)) + │ │ └── body: (length: 1) + │ │ └── @ MatchWriteNode (location: (40,6)-(40,22)) + │ │ ├── call: + │ │ │ @ CallNode (location: (40,6)-(40,22)) + │ │ │ ├── flags: ∅ + │ │ │ ├── receiver: + │ │ │ │ @ RegularExpressionNode (location: (40,6)-(40,14)) + │ │ │ │ ├── flags: forced_us_ascii_encoding + │ │ │ │ ├── opening_loc: (40,6)-(40,7) = "/" + │ │ │ │ ├── content_loc: (40,7)-(40,13) = "(?)" + │ │ │ │ ├── closing_loc: (40,13)-(40,14) = "/" + │ │ │ │ └── unescaped: "(?)" + │ │ │ ├── call_operator_loc: ∅ + │ │ │ ├── name: :=~ + │ │ │ ├── message_loc: (40,15)-(40,17) = "=~" + │ │ │ ├── opening_loc: ∅ + │ │ │ ├── arguments: + │ │ │ │ @ ArgumentsNode (location: (40,18)-(40,22)) + │ │ │ │ ├── flags: ∅ + │ │ │ │ └── arguments: (length: 1) + │ │ │ │ └── @ CallNode (location: (40,18)-(40,22)) + │ │ │ │ ├── flags: variable_call, ignore_visibility + │ │ │ │ ├── receiver: ∅ + │ │ │ │ ├── call_operator_loc: ∅ + │ │ │ │ ├── name: :to_s + │ │ │ │ ├── message_loc: (40,18)-(40,22) = "to_s" + │ │ │ │ ├── opening_loc: ∅ + │ │ │ │ ├── arguments: ∅ + │ │ │ │ ├── closing_loc: ∅ + │ │ │ │ └── block: ∅ + │ │ │ ├── closing_loc: ∅ + │ │ │ └── block: ∅ + │ │ └── targets: (length: 1) + │ │ └── @ LocalVariableTargetNode (location: (40,10)-(40,11)) + │ │ ├── name: :a + │ │ └── depth: 1 + │ ├── opening_loc: (40,4)-(40,5) = "{" + │ └── closing_loc: (40,23)-(40,24) = "}" + ├── @ MatchWriteNode (location: (42,0)-(42,16)) + │ ├── call: + │ │ @ CallNode (location: (42,0)-(42,16)) + │ │ ├── flags: ∅ + │ │ ├── receiver: + │ │ │ @ RegularExpressionNode (location: (42,0)-(42,10)) + │ │ │ ├── flags: forced_us_ascii_encoding + │ │ │ ├── opening_loc: (42,0)-(42,1) = "/" + │ │ │ ├── content_loc: (42,1)-(42,9) = "(?)" + │ │ │ ├── closing_loc: (42,9)-(42,10) = "/" + │ │ │ └── unescaped: "(?)" + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :=~ + │ │ ├── message_loc: (42,11)-(42,13) = "=~" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: + │ │ │ @ ArgumentsNode (location: (42,14)-(42,16)) + │ │ │ ├── flags: ∅ + │ │ │ └── arguments: (length: 1) + │ │ │ └── @ StringNode (location: (42,14)-(42,16)) + │ │ │ ├── flags: ∅ + │ │ │ ├── opening_loc: (42,14)-(42,15) = "\"" + │ │ │ ├── content_loc: (42,15)-(42,15) = "" + │ │ │ ├── closing_loc: (42,15)-(42,16) = "\"" + │ │ │ └── unescaped: "" + │ │ ├── closing_loc: ∅ + │ │ └── block: ∅ + │ └── targets: (length: 1) + │ └── @ LocalVariableTargetNode (location: (42,4)-(42,7)) + │ ├── name: :foo + │ └── depth: 0 + ├── @ CallNode (location: (43,0)-(43,16)) + │ ├── flags: ∅ + │ ├── receiver: + │ │ @ RegularExpressionNode (location: (43,0)-(43,10)) + │ │ ├── flags: forced_us_ascii_encoding + │ │ ├── opening_loc: (43,0)-(43,1) = "/" + │ │ ├── content_loc: (43,1)-(43,9) = "(?)" + │ │ ├── closing_loc: (43,9)-(43,10) = "/" + │ │ └── unescaped: "(?)" + │ ├── call_operator_loc: ∅ + │ ├── name: :=~ + │ ├── message_loc: (43,11)-(43,13) = "=~" + │ ├── opening_loc: ∅ + │ ├── arguments: + │ │ @ ArgumentsNode (location: (43,14)-(43,16)) + │ │ ├── flags: ∅ + │ │ └── arguments: (length: 1) + │ │ └── @ StringNode (location: (43,14)-(43,16)) + │ │ ├── flags: ∅ + │ │ ├── opening_loc: (43,14)-(43,15) = "\"" + │ │ ├── content_loc: (43,15)-(43,15) = "" + │ │ ├── closing_loc: (43,15)-(43,16) = "\"" + │ │ └── unescaped: "" + │ ├── closing_loc: ∅ + │ └── block: ∅ + ├── @ CallNode (location: (45,0)-(45,16)) + │ ├── flags: ∅ + │ ├── receiver: + │ │ @ RegularExpressionNode (location: (45,0)-(45,10)) + │ │ ├── flags: forced_us_ascii_encoding + │ │ ├── opening_loc: (45,0)-(45,1) = "/" + │ │ ├── content_loc: (45,1)-(45,9) = "(?)" + │ │ ├── closing_loc: (45,9)-(45,10) = "/" + │ │ └── unescaped: "(?)" + │ ├── call_operator_loc: ∅ + │ ├── name: :=~ + │ ├── message_loc: (45,11)-(45,13) = "=~" + │ ├── opening_loc: ∅ + │ ├── arguments: + │ │ @ ArgumentsNode (location: (45,14)-(45,16)) + │ │ ├── flags: ∅ + │ │ └── arguments: (length: 1) + │ │ └── @ StringNode (location: (45,14)-(45,16)) + │ │ ├── flags: ∅ + │ │ ├── opening_loc: (45,14)-(45,15) = "\"" + │ │ ├── content_loc: (45,15)-(45,15) = "" + │ │ ├── closing_loc: (45,15)-(45,16) = "\"" + │ │ └── unescaped: "" + │ ├── closing_loc: ∅ + │ └── block: ∅ + └── @ DefNode (location: (46,0)-(46,32)) + ├── name: :foo + ├── name_loc: (46,4)-(46,7) = "foo" ├── receiver: ∅ - ├── call_operator_loc: ∅ - ├── name: :tap - ├── message_loc: (40,0)-(40,3) = "tap" - ├── opening_loc: ∅ - ├── arguments: ∅ - ├── closing_loc: ∅ - └── block: - @ BlockNode (location: (40,4)-(40,24)) - ├── locals: [] - ├── parameters: ∅ - ├── body: - │ @ StatementsNode (location: (40,6)-(40,22)) - │ └── body: (length: 1) - │ └── @ MatchWriteNode (location: (40,6)-(40,22)) - │ ├── call: - │ │ @ CallNode (location: (40,6)-(40,22)) - │ │ ├── flags: ∅ - │ │ ├── receiver: - │ │ │ @ RegularExpressionNode (location: (40,6)-(40,14)) - │ │ │ ├── flags: forced_us_ascii_encoding - │ │ │ ├── opening_loc: (40,6)-(40,7) = "/" - │ │ │ ├── content_loc: (40,7)-(40,13) = "(?)" - │ │ │ ├── closing_loc: (40,13)-(40,14) = "/" - │ │ │ └── unescaped: "(?)" - │ │ ├── call_operator_loc: ∅ - │ │ ├── name: :=~ - │ │ ├── message_loc: (40,15)-(40,17) = "=~" - │ │ ├── opening_loc: ∅ - │ │ ├── arguments: - │ │ │ @ ArgumentsNode (location: (40,18)-(40,22)) - │ │ │ ├── flags: ∅ - │ │ │ └── arguments: (length: 1) - │ │ │ └── @ CallNode (location: (40,18)-(40,22)) - │ │ │ ├── flags: variable_call, ignore_visibility - │ │ │ ├── receiver: ∅ - │ │ │ ├── call_operator_loc: ∅ - │ │ │ ├── name: :to_s - │ │ │ ├── message_loc: (40,18)-(40,22) = "to_s" - │ │ │ ├── opening_loc: ∅ - │ │ │ ├── arguments: ∅ - │ │ │ ├── closing_loc: ∅ - │ │ │ └── block: ∅ - │ │ ├── closing_loc: ∅ - │ │ └── block: ∅ - │ └── targets: (length: 1) - │ └── @ LocalVariableTargetNode (location: (40,10)-(40,11)) - │ ├── name: :a - │ └── depth: 1 - ├── opening_loc: (40,4)-(40,5) = "{" - └── closing_loc: (40,23)-(40,24) = "}" + ├── parameters: + │ @ ParametersNode (location: (46,8)-(46,12)) + │ ├── requireds: (length: 0) + │ ├── optionals: (length: 0) + │ ├── rest: ∅ + │ ├── posts: (length: 0) + │ ├── keywords: (length: 1) + │ │ └── @ RequiredKeywordParameterNode (location: (46,8)-(46,12)) + │ │ ├── flags: ∅ + │ │ ├── name: :nil + │ │ └── name_loc: (46,8)-(46,12) = "nil:" + │ ├── keyword_rest: ∅ + │ └── block: ∅ + ├── body: + │ @ StatementsNode (location: (46,16)-(46,32)) + │ └── body: (length: 1) + │ └── @ MatchWriteNode (location: (46,16)-(46,32)) + │ ├── call: + │ │ @ CallNode (location: (46,16)-(46,32)) + │ │ ├── flags: ∅ + │ │ ├── receiver: + │ │ │ @ RegularExpressionNode (location: (46,16)-(46,26)) + │ │ │ ├── flags: forced_us_ascii_encoding + │ │ │ ├── opening_loc: (46,16)-(46,17) = "/" + │ │ │ ├── content_loc: (46,17)-(46,25) = "(?)" + │ │ │ ├── closing_loc: (46,25)-(46,26) = "/" + │ │ │ └── unescaped: "(?)" + │ │ ├── call_operator_loc: ∅ + │ │ ├── name: :=~ + │ │ ├── message_loc: (46,27)-(46,29) = "=~" + │ │ ├── opening_loc: ∅ + │ │ ├── arguments: + │ │ │ @ ArgumentsNode (location: (46,30)-(46,32)) + │ │ │ ├── flags: ∅ + │ │ │ └── arguments: (length: 1) + │ │ │ └── @ StringNode (location: (46,30)-(46,32)) + │ │ │ ├── flags: ∅ + │ │ │ ├── opening_loc: (46,30)-(46,31) = "\"" + │ │ │ ├── content_loc: (46,31)-(46,31) = "" + │ │ │ ├── closing_loc: (46,31)-(46,32) = "\"" + │ │ │ └── unescaped: "" + │ │ ├── closing_loc: ∅ + │ │ └── block: ∅ + │ └── targets: (length: 1) + │ └── @ LocalVariableTargetNode (location: (46,20)-(46,23)) + │ ├── name: :nil + │ └── depth: 0 + ├── locals: [:nil] + ├── def_keyword_loc: (46,0)-(46,3) = "def" + ├── operator_loc: ∅ + ├── lparen_loc: (46,7)-(46,8) = "(" + ├── rparen_loc: (46,12)-(46,13) = ")" + ├── equal_loc: (46,14)-(46,15) = "=" + └── end_keyword_loc: ∅ diff --git a/test/prism/static_literals_test.rb b/test/prism/static_literals_test.rb index 26699319741999..31c802bf90d188 100644 --- a/test/prism/static_literals_test.rb +++ b/test/prism/static_literals_test.rb @@ -32,6 +32,7 @@ def test_static_literals assert_warning("1ri", "1ri", "(0+(1/1)*i)") assert_warning("1.0ri", "1.0ri", "(0+(1/1)*i)") + assert_warning("__FILE__", "\"#{__FILE__}\"", __FILE__) assert_warning("\"#{__FILE__}\"") assert_warning("\"foo\"") @@ -50,17 +51,23 @@ def test_static_literals private + class NullWarning + def message + "" + end + end + def parse_warnings(left, right) warnings = [] - warnings << Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first + warnings << (Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first || NullWarning.new) { #{left} => 1, #{right} => 2 } RUBY - warnings << Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first + warnings << (Prism.parse(<<~RUBY, filepath: __FILE__).warnings.first || NullWarning.new) case foo when #{left} when #{right} @@ -79,7 +86,7 @@ def assert_warning(left, right = left, message = left) end def refute_warning(left, right) - assert_empty parse_warnings(left, right).compact + assert_empty parse_warnings(left, right).grep_v(NullWarning) end end end diff --git a/test/reline/test_line_editor.rb b/test/reline/test_line_editor.rb index 0963d2c8fe835c..bf688ac3c6f193 100644 --- a/test/reline/test_line_editor.rb +++ b/test/reline/test_line_editor.rb @@ -127,4 +127,29 @@ def test_complicated end end end + + def test_menu_info_format + list = %w[aa b c d e f g hhh i j k] + col3 = [ + 'aa e i', + 'b f j', + 'c g k', + 'd hhh' + ] + col2 = [ + 'aa g', + 'b hhh', + 'c i', + 'd j', + 'e k', + 'f' + ] + assert_equal(col3, Reline::LineEditor::MenuInfo.new(list).lines(19)) + assert_equal(col3, Reline::LineEditor::MenuInfo.new(list).lines(15)) + assert_equal(col2, Reline::LineEditor::MenuInfo.new(list).lines(14)) + assert_equal(col2, Reline::LineEditor::MenuInfo.new(list).lines(10)) + assert_equal(list, Reline::LineEditor::MenuInfo.new(list).lines(9)) + assert_equal(list, Reline::LineEditor::MenuInfo.new(list).lines(0)) + assert_equal([], Reline::LineEditor::MenuInfo.new([]).lines(10)) + end end diff --git a/test/reline/yamatanooroti/test_rendering.rb b/test/reline/yamatanooroti/test_rendering.rb index efb1e01562b905..53c842a04a41c7 100644 --- a/test/reline/yamatanooroti/test_rendering.rb +++ b/test/reline/yamatanooroti/test_rendering.rb @@ -1108,6 +1108,19 @@ def test_autocomplete_target_is_wrapped EOC end + def test_force_insert_before_autocomplete + start_terminal(20, 20, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl --autocomplete}, startup_message: 'Multiline REPL.') + write('Sy') + write(";St\t\t") + close + assert_screen(<<~'EOC') + Multiline REPL. + prompt> Sy;Struct + String + Struct + EOC + end + def test_simple_dialog_with_scroll_key start_terminal(20, 50, %W{ruby -I#{@pwd}/lib #{@pwd}/test/reline/yamatanooroti/multiline_repl --dialog long,scrollkey}, startup_message: 'Multiline REPL.') write('a') diff --git a/test/resolv/test_dns.rb b/test/resolv/test_dns.rb index 20c3408cd6b9ca..40c5406db83525 100644 --- a/test/resolv/test_dns.rb +++ b/test/resolv/test_dns.rb @@ -64,6 +64,50 @@ def with_udp(host, port) end end + def with_udp_and_tcp(host, port) + if port == 0 + # Automatic port; we might need to retry until we find a port which is free on both UDP _and_ TCP. + retries_remaining = 5 + t = nil + u = nil + begin + begin + u = UDPSocket.new + u.bind(host, 0) + _, udp_port, _, _ = u.addr + t = TCPServer.new(host, udp_port) + t.listen(1) + rescue Errno::EADDRINUSE, Errno::EACCES + # ADDRINUSE is what should get thrown if we try and bind a port which is already bound on UNIXen, + # but windows can sometimes throw EACCESS. + # See: https://stackoverflow.com/questions/48478869/cannot-bind-to-some-ports-due-to-permission-denied + retries_remaining -= 1 + if retries_remaining > 0 + t&.close + t = nil + u&.close + u = nil + retry + end + raise + end + + # If we get to this point, we have a valid t & u socket + yield u, t + ensure + t&.close + u&.close + end + else + # Explicitly specified port, don't retry the bind. + with_udp(host, port) do |u| + with_tcp(host, port) do |t| + yield u, t + end + end + end + end + # [ruby-core:65836] def test_resolve_with_2_ndots conf = Resolv::DNS::Config.new :nameserver => ['127.0.0.1'], :ndots => 2 @@ -176,156 +220,154 @@ def test_query_ipv4_address_truncated_tcp_fallback num_records = 50 - with_udp('127.0.0.1', 0) {|u| + with_udp_and_tcp('127.0.0.1', 0) {|u, t| _, server_port, _, server_address = u.addr - with_tcp('127.0.0.1', server_port) {|t| - client_thread = Thread.new { - Resolv::DNS.open(:nameserver_port => [[server_address, server_port]]) {|dns| - dns.getresources("foo.example.org", Resolv::DNS::Resource::IN::A) - } - } - udp_server_thread = Thread.new { - msg, (_, client_port, _, client_address) = Timeout.timeout(5) {u.recvfrom(4096)} - id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") - qr = (word2 & 0x8000) >> 15 - opcode = (word2 & 0x7800) >> 11 - aa = (word2 & 0x0400) >> 10 - tc = (word2 & 0x0200) >> 9 - rd = (word2 & 0x0100) >> 8 - ra = (word2 & 0x0080) >> 7 - z = (word2 & 0x0070) >> 4 - rcode = word2 & 0x000f - rest = msg[12..-1] - assert_equal(0, qr) # 0:query 1:response - assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS - assert_equal(0, aa) # Authoritative Answer - assert_equal(0, tc) # TrunCation - assert_equal(1, rd) # Recursion Desired - assert_equal(0, ra) # Recursion Available - assert_equal(0, z) # Reserved for future use - assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused - assert_equal(1, qdcount) # number of entries in the question section. - assert_equal(0, ancount) # number of entries in the answer section. - assert_equal(0, nscount) # number of entries in the authority records section. - assert_equal(0, arcount) # number of entries in the additional records section. - name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") - assert_operator(rest, :start_with?, name) - rest = rest[name.length..-1] - assert_equal(4, rest.length) - qtype, _ = rest.unpack("nn") - assert_equal(1, qtype) # A - assert_equal(1, qtype) # IN - id = id - qr = 1 - opcode = opcode - aa = 0 - tc = 1 - rd = rd - ra = 1 - z = 0 - rcode = 0 - qdcount = 0 - ancount = num_records - nscount = 0 - arcount = 0 - word2 = (qr << 15) | - (opcode << 11) | - (aa << 10) | - (tc << 9) | - (rd << 8) | - (ra << 7) | - (z << 4) | - rcode - msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") - type = 1 - klass = 1 - ttl = 3600 - rdlength = 4 - num_records.times do |i| - rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 - rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") - msg << rr - end - u.send(msg[0...512], 0, client_address, client_port) - } - tcp_server_thread = Thread.new { - ct = t.accept - msg = ct.recv(512) - msg.slice!(0..1) # Size (only for TCP) - id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") - qr = (word2 & 0x8000) >> 15 - opcode = (word2 & 0x7800) >> 11 - aa = (word2 & 0x0400) >> 10 - tc = (word2 & 0x0200) >> 9 - rd = (word2 & 0x0100) >> 8 - ra = (word2 & 0x0080) >> 7 - z = (word2 & 0x0070) >> 4 - rcode = word2 & 0x000f - rest = msg[12..-1] - assert_equal(0, qr) # 0:query 1:response - assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS - assert_equal(0, aa) # Authoritative Answer - assert_equal(0, tc) # TrunCation - assert_equal(1, rd) # Recursion Desired - assert_equal(0, ra) # Recursion Available - assert_equal(0, z) # Reserved for future use - assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused - assert_equal(1, qdcount) # number of entries in the question section. - assert_equal(0, ancount) # number of entries in the answer section. - assert_equal(0, nscount) # number of entries in the authority records section. - assert_equal(0, arcount) # number of entries in the additional records section. - name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") - assert_operator(rest, :start_with?, name) - rest = rest[name.length..-1] - assert_equal(4, rest.length) - qtype, _ = rest.unpack("nn") - assert_equal(1, qtype) # A - assert_equal(1, qtype) # IN - id = id - qr = 1 - opcode = opcode - aa = 0 - tc = 0 - rd = rd - ra = 1 - z = 0 - rcode = 0 - qdcount = 0 - ancount = num_records - nscount = 0 - arcount = 0 - word2 = (qr << 15) | - (opcode << 11) | - (aa << 10) | - (tc << 9) | - (rd << 8) | - (ra << 7) | - (z << 4) | - rcode - msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") - type = 1 - klass = 1 - ttl = 3600 - rdlength = 4 - num_records.times do |i| - rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 - rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") - msg << rr - end - msg = "#{[msg.bytesize].pack("n")}#{msg}" # Prefix with size - ct.send(msg, 0) - ct.close + client_thread = Thread.new { + Resolv::DNS.open(:nameserver_port => [[server_address, server_port]]) {|dns| + dns.getresources("foo.example.org", Resolv::DNS::Resource::IN::A) } - result, _ = assert_join_threads([client_thread, udp_server_thread, tcp_server_thread]) - assert_instance_of(Array, result) - assert_equal(50, result.length) - result.each_with_index do |rr, i| - assert_instance_of(Resolv::DNS::Resource::IN::A, rr) - assert_instance_of(Resolv::IPv4, rr.address) - assert_equal("192.0.2.#{i}", rr.address.to_s) - assert_equal(3600, rr.ttl) + } + udp_server_thread = Thread.new { + msg, (_, client_port, _, client_address) = Timeout.timeout(5) {u.recvfrom(4096)} + id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") + qr = (word2 & 0x8000) >> 15 + opcode = (word2 & 0x7800) >> 11 + aa = (word2 & 0x0400) >> 10 + tc = (word2 & 0x0200) >> 9 + rd = (word2 & 0x0100) >> 8 + ra = (word2 & 0x0080) >> 7 + z = (word2 & 0x0070) >> 4 + rcode = word2 & 0x000f + rest = msg[12..-1] + assert_equal(0, qr) # 0:query 1:response + assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS + assert_equal(0, aa) # Authoritative Answer + assert_equal(0, tc) # TrunCation + assert_equal(1, rd) # Recursion Desired + assert_equal(0, ra) # Recursion Available + assert_equal(0, z) # Reserved for future use + assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused + assert_equal(1, qdcount) # number of entries in the question section. + assert_equal(0, ancount) # number of entries in the answer section. + assert_equal(0, nscount) # number of entries in the authority records section. + assert_equal(0, arcount) # number of entries in the additional records section. + name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") + assert_operator(rest, :start_with?, name) + rest = rest[name.length..-1] + assert_equal(4, rest.length) + qtype, _ = rest.unpack("nn") + assert_equal(1, qtype) # A + assert_equal(1, qtype) # IN + id = id + qr = 1 + opcode = opcode + aa = 0 + tc = 1 + rd = rd + ra = 1 + z = 0 + rcode = 0 + qdcount = 0 + ancount = num_records + nscount = 0 + arcount = 0 + word2 = (qr << 15) | + (opcode << 11) | + (aa << 10) | + (tc << 9) | + (rd << 8) | + (ra << 7) | + (z << 4) | + rcode + msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") + type = 1 + klass = 1 + ttl = 3600 + rdlength = 4 + num_records.times do |i| + rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 + rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") + msg << rr end + u.send(msg[0...512], 0, client_address, client_port) } + tcp_server_thread = Thread.new { + ct = t.accept + msg = ct.recv(512) + msg.slice!(0..1) # Size (only for TCP) + id, word2, qdcount, ancount, nscount, arcount = msg.unpack("nnnnnn") + qr = (word2 & 0x8000) >> 15 + opcode = (word2 & 0x7800) >> 11 + aa = (word2 & 0x0400) >> 10 + tc = (word2 & 0x0200) >> 9 + rd = (word2 & 0x0100) >> 8 + ra = (word2 & 0x0080) >> 7 + z = (word2 & 0x0070) >> 4 + rcode = word2 & 0x000f + rest = msg[12..-1] + assert_equal(0, qr) # 0:query 1:response + assert_equal(0, opcode) # 0:QUERY 1:IQUERY 2:STATUS + assert_equal(0, aa) # Authoritative Answer + assert_equal(0, tc) # TrunCation + assert_equal(1, rd) # Recursion Desired + assert_equal(0, ra) # Recursion Available + assert_equal(0, z) # Reserved for future use + assert_equal(0, rcode) # 0:No-error 1:Format-error 2:Server-failure 3:Name-Error 4:Not-Implemented 5:Refused + assert_equal(1, qdcount) # number of entries in the question section. + assert_equal(0, ancount) # number of entries in the answer section. + assert_equal(0, nscount) # number of entries in the authority records section. + assert_equal(0, arcount) # number of entries in the additional records section. + name = [3, "foo", 7, "example", 3, "org", 0].pack("Ca*Ca*Ca*C") + assert_operator(rest, :start_with?, name) + rest = rest[name.length..-1] + assert_equal(4, rest.length) + qtype, _ = rest.unpack("nn") + assert_equal(1, qtype) # A + assert_equal(1, qtype) # IN + id = id + qr = 1 + opcode = opcode + aa = 0 + tc = 0 + rd = rd + ra = 1 + z = 0 + rcode = 0 + qdcount = 0 + ancount = num_records + nscount = 0 + arcount = 0 + word2 = (qr << 15) | + (opcode << 11) | + (aa << 10) | + (tc << 9) | + (rd << 8) | + (ra << 7) | + (z << 4) | + rcode + msg = [id, word2, qdcount, ancount, nscount, arcount].pack("nnnnnn") + type = 1 + klass = 1 + ttl = 3600 + rdlength = 4 + num_records.times do |i| + rdata = [192,0,2,i].pack("CCCC") # 192.0.2.x (TEST-NET address) RFC 3330 + rr = [name, type, klass, ttl, rdlength, rdata].pack("a*nnNna*") + msg << rr + end + msg = "#{[msg.bytesize].pack("n")}#{msg}" # Prefix with size + ct.send(msg, 0) + ct.close + } + result, _ = assert_join_threads([client_thread, udp_server_thread, tcp_server_thread]) + assert_instance_of(Array, result) + assert_equal(50, result.length) + result.each_with_index do |rr, i| + assert_instance_of(Resolv::DNS::Resource::IN::A, rr) + assert_instance_of(Resolv::IPv4, rr.address) + assert_equal("192.0.2.#{i}", rr.address.to_s) + assert_equal(3600, rr.ttl) + end } end diff --git a/test/ruby/test_literal.rb b/test/ruby/test_literal.rb index 61e1e096df8d08..c6154af1f61161 100644 --- a/test/ruby/test_literal.rb +++ b/test/ruby/test_literal.rb @@ -509,9 +509,6 @@ def test_hash_duplicated_key ) do |key| assert_warning(/key #{Regexp.quote(eval(key).inspect)} is duplicated/) { eval("{#{key} => :bar, #{key} => :foo}") } end - - assert_warning(/key 1 is duplicated/) { eval("{__LINE__ => :bar, 1 => :foo}") } - assert_warning(/key \"FILENAME\" is duplicated/) { eval("{__FILE__ => :bar, 'FILENAME' => :foo}", binding, "FILENAME") } end def test_hash_frozen_key_id diff --git a/test/ruby/test_parse.rb b/test/ruby/test_parse.rb index 62498170fd5242..e365c39def583b 100644 --- a/test/ruby/test_parse.rb +++ b/test/ruby/test_parse.rb @@ -803,14 +803,20 @@ def test_magic_comment # coding: foo END end - assert_include(e.message, "# coding: foo\n ^~~") + + message = e.message.gsub(/\033\[.*?m/, "") + assert_include(message, "# coding: foo\n") + assert_include(message, " ^") e = assert_raise(ArgumentError) do eval <<-END, nil, __FILE__, __LINE__+1 # coding = foo END end - assert_include(e.message, "# coding = foo\n ^~~") + + message = e.message.gsub(/\033\[.*?m/, "") + assert_include(message, "# coding = foo\n") + assert_include(message, " ^") end def test_utf8_bom diff --git a/test/ruby/test_process.rb b/test/ruby/test_process.rb index 59140ba664541c..7ef184d63964ab 100644 --- a/test/ruby/test_process.rb +++ b/test/ruby/test_process.rb @@ -199,58 +199,67 @@ def test_execopts_rlimit max = Process.getrlimit(:CORE).last + # When running under ASAN, we need to set disable_coredump=0 for this test; by default + # the ASAN runtime library sets RLIMIT_CORE to 0, "to avoid dumping a 16T+ core file", and + # that inteferes with this test. + asan_options = ENV['ASAN_OPTIONS'] || '' + asan_options << ':' unless asan_options.empty? + env = { + 'ASAN_OPTIONS' => "#{asan_options}disable_coredump=0" + } + n = max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>n]) {|io| assert_equal("#{n}\n#{n}\n", io.read) } n = 0 - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>n]) {|io| assert_equal("#{n}\n#{n}\n", io.read) } n = max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>[n]]) {|io| assert_equal("#{n}\n#{n}\n", io.read) } m, n = 0, max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>[m,n]]) {|io| assert_equal("#{m}\n#{n}\n", io.read) } m, n = 0, 0 - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE)", :rlimit_core=>[m,n]]) {|io| assert_equal("#{m}\n#{n}\n", io.read) } n = max - IO.popen([RUBY, "-e", + IO.popen([env, RUBY, "-e", "puts Process.getrlimit(:CORE), Process.getrlimit(:CPU)", :rlimit_core=>n, :rlimit_cpu=>3600]) {|io| assert_equal("#{n}\n#{n}\n""3600\n3600\n", io.read) } assert_raise(ArgumentError) do - system(RUBY, '-e', 'exit', 'rlimit_bogus'.to_sym => 123) + system(env, RUBY, '-e', 'exit', 'rlimit_bogus'.to_sym => 123) end - assert_separately([],"#{<<~"begin;"}\n#{<<~'end;'}", 'rlimit_cpu'.to_sym => 3600) + assert_separately([env],"#{<<~"begin;"}\n#{<<~'end;'}", 'rlimit_cpu'.to_sym => 3600) BUG = "[ruby-core:82033] [Bug #13744]" begin; assert_equal([3600,3600], Process.getrlimit(:CPU), BUG) end; assert_raise_with_message(ArgumentError, /bogus/) do - system(RUBY, '-e', 'exit', :rlimit_bogus => 123) + system(env, RUBY, '-e', 'exit', :rlimit_bogus => 123) end assert_raise_with_message(ArgumentError, /rlimit_cpu/) { - system(RUBY, '-e', 'exit', "rlimit_cpu\0".to_sym => 3600) + system(env, RUBY, '-e', 'exit', "rlimit_cpu\0".to_sym => 3600) } end diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb index 446548d6d9a395..8396066dc1ba19 100644 --- a/test/ruby/test_rubyoptions.rb +++ b/test/ruby/test_rubyoptions.rb @@ -865,6 +865,9 @@ def assert_segv(args, message=nil, list: SEGVTest::ExpectedStderrList, **opt, &b env = Hash === args.first ? args.shift : {} args.unshift("--yjit") if self.class.yjit_enabled? env.update({'RUBY_ON_BUG' => nil}) + # ASAN registers a segv handler which prints out "AddressSanitizer: DEADLYSIGNAL" when + # catching sigsegv; we don't expect that output, so suppress it. + env.update({'ASAN_OPTIONS' => 'handle_segv=0'}) args.unshift(env) test_stdin = "" diff --git a/test/ruby/test_syntax.rb b/test/ruby/test_syntax.rb index 371c41fe37beda..7cc5e542a7b7bd 100644 --- a/test/ruby/test_syntax.rb +++ b/test/ruby/test_syntax.rb @@ -734,7 +734,7 @@ def test_duplicated_when end } } - assert_warning(/3: #{w}.+4: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m) { + assert_warning(/3: #{w}/m) { eval %q{ case 1 when __LINE__, __LINE__ @@ -743,7 +743,7 @@ def test_duplicated_when end } } - assert_warning(/3: #{w}.+4: #{w}.+4: #{w}.+5: #{w}.+5: #{w}/m) { + assert_warning(/3: #{w}/m) { eval %q{ case 1 when __FILE__, __FILE__ diff --git a/test/rubygems/test_gem_commands_rebuild_command.rb b/test/rubygems/test_gem_commands_rebuild_command.rb new file mode 100644 index 00000000000000..5e8c797e2d11ad --- /dev/null +++ b/test/rubygems/test_gem_commands_rebuild_command.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require_relative "helper" +require "rubygems/commands/build_command" +require "rubygems/commands/rebuild_command" +require "rubygems/package" + +class TestGemCommandsRebuildCommand < Gem::TestCase + def setup + super + + readme_file = File.join(@tempdir, "README.md") + + begin + umask_orig = File.umask(2) + File.open readme_file, "w" do |f| + f.write "My awesome gem" + end + ensure + File.umask(umask_orig) + end + + @gem_name = "rebuild_test_gem" + @gem_version = "1.0.0" + @gem = util_spec @gem_name do |s| + s.version = @gem_version + s.license = "AGPL-3.0" + s.files = ["README.md"] + end + end + + def util_test_build_gem(gem, args) + @ui = Gem::MockGemUi.new + + cmd = Gem::Commands::BuildCommand.new + + cmd.options[:args] = args + cmd.options[:build_path] = @tempdir + use_ui @ui do + cmd.execute + end + gem_file = "#{@gem_name}-#{@gem_version}.gem" + output = @ui.output.split "\n" + assert_equal " Successfully built RubyGem", output.shift + assert_equal " Name: #{@gem_name}", output.shift + assert_equal " Version: #{@gem_version}", output.shift + assert_equal " File: #{gem_file}", output.shift + assert_equal [], output + + gem_file = File.join(@tempdir, gem_file) + assert File.exist?(gem_file) + + spec = Gem::Package.new(gem_file).spec + + assert_equal @gem_name, spec.name + assert_equal "this is a summary", spec.summary + gem_file + end + + def util_test_rebuild_gem(gem, args, original_gem_file, gemspec_file, timestamp) + @ui = Gem::MockGemUi.new + + cmd = Gem::Commands::RebuildCommand.new + + cmd.options[:args] = args + cmd.options[:original_gem_file] = original_gem_file + cmd.options[:build_path] = @tempdir + cmd.options[:gemspec_file] = gemspec_file + use_ui @ui do + cmd.execute + end + gem_file = "#{@gem_name}-#{@gem_version}.gem" + output = @ui.output.split "\n" + + assert_equal " Successfully built RubyGem", output.shift + assert_equal " Name: #{@gem_name}", output.shift + assert_equal " Version: #{@gem_version}", output.shift + assert_equal " File: #{gem_file}", output.shift + assert_empty output.shift + assert_match(/^Built at: .+ \(#{timestamp}\)/, output.shift) + original_line = output.shift + original = original_line.split(" ")[-1] + assert_match(/^Original build saved to: /, original_line) + reproduced_line = output.shift + reproduced = reproduced_line.split(" ")[-1] + assert_match(/^Reproduced build saved to: /, reproduced_line) + assert_equal "Working directory: #{@tempdir}", output.shift + assert_equal "", output.shift + assert_equal "Hash comparison:", output.shift + output.shift # " #{old_hash}\t#{old_file}" + output.shift # " #{new_hash}\t#{new_file}" + assert_empty output.shift + assert_equal "SUCCESS - original and rebuild hashes matched", output.shift + assert_equal [], output + + assert File.exist?(original) + assert File.exist?(reproduced) + + old_spec = Gem::Package.new(original).spec + new_spec = Gem::Package.new(reproduced).spec + + assert_equal @gem_name, old_spec.name + assert_equal "this is a summary", old_spec.summary + + assert_equal old_spec.name, new_spec.name + assert_equal old_spec.summary, new_spec.summary + + reproduced + end + + def test_build_is_reproducible + # Back up SOURCE_DATE_EPOCH to restore later. + epoch = ENV["SOURCE_DATE_EPOCH"] + + gemspec_file = File.join(@tempdir, @gem.spec_name) + + # Initial Build + + # Set SOURCE_DATE_EPOCH to 2001-02-03 04:05:06 -0500. + ENV["SOURCE_DATE_EPOCH"] = timestamp = Time.new(2001, 2, 3, 4, 5, 6).to_i.to_s + File.write(gemspec_file, @gem.to_ruby) + gem_file = util_test_build_gem @gem, [gemspec_file] + + build_contents = File.read(gem_file) + + gem_file_dir = File.dirname(gem_file) + gem_file_name = File.basename(gem_file) + original_gem_file = File.join(gem_file_dir, "original-" + gem_file_name) + File.rename(gem_file, original_gem_file) + + # Rebuild + + # Set SOURCE_DATE_EPOCH to a different value, meaning we are + # also testing that `gem rebuild` overrides the value. + ENV["SOURCE_DATE_EPOCH"] = Time.new(2007, 8, 9, 10, 11, 12).to_s + + rebuild_gem_file = util_test_rebuild_gem(@gem, [@gem_name, @gem_version], original_gem_file, gemspec_file, timestamp) + + rebuild_contents = File.read(rebuild_gem_file) + + assert_equal build_contents, rebuild_contents + ensure + ENV["SOURCE_DATE_EPOCH"] = epoch + end +end diff --git a/test/rubygems/test_gem_source.rb b/test/rubygems/test_gem_source.rb index 4d445f343765d6..6baa203dcb1829 100644 --- a/test/rubygems/test_gem_source.rb +++ b/test/rubygems/test_gem_source.rb @@ -54,8 +54,9 @@ def test_dependency_resolver_set_bundler_api end def test_dependency_resolver_set_file_uri - File.write(File.join(@tempdir, "prerelease_specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b)) - File.write(File.join(@tempdir, "specs.4.8.gz"), Gem::Util.gzip("\x04\x08[\x05".b)) + empty_dump = Gem::Util.gzip("\x04\x08[\x05".b) + File.binwrite(File.join(@tempdir, "prerelease_specs.4.8.gz"), empty_dump) + File.binwrite(File.join(@tempdir, "specs.4.8.gz"), empty_dump) source = Gem::Source.new "file://#{@tempdir}/" diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb index 2af69235744e09..e17cd0abb11a01 100644 --- a/test/stringio/test_stringio.rb +++ b/test/stringio/test_stringio.rb @@ -978,6 +978,27 @@ def test_coderange_after_overwrite assert_predicate(s.string, :ascii_only?) end + if eval(%{ "test".frozen? && !"test".equal?("test") }) # Ruby 3.4+ chilled strings + def test_chilled_string + chilled_string = eval(%{""}) + io = StringIO.new(chilled_string) + assert_warning(/literal string will be frozen/) { io << "test" } + assert_equal("test", io.string) + assert_same(chilled_string, io.string) + end + + def test_chilled_string_string_set + io = StringIO.new + chilled_string = eval(%{""}) + io.string = chilled_string + assert_warning(/literal string will be frozen/) { io << "test" } + assert_equal("test", io.string) + assert_same(chilled_string, io.string) + end + end + + private + def assert_string(content, encoding, str, mesg = nil) assert_equal([content, encoding], [str, str.encoding], mesg) end diff --git a/thread_pthread.c b/thread_pthread.c index 2981d57e37da03..725edb17177f14 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -305,7 +305,7 @@ event_name(rb_event_flag_t event) static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */ -#if defined(SIGVTALRM) && !defined(__CYGWIN__) && !defined(__EMSCRIPTEN__) +#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__) # define USE_UBF_LIST 1 #endif diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 01c53e929cfe31..a748090b2645ca 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -1409,14 +1409,16 @@ def record(suite, method, assertions, time, error, source_location = nil) if writer && test_path && status # Occasionally, the file writing operation may be paused, especially when `--repeat-count` is specified. # In such cases, we proceed to execute the operation here. - writer.write_object do - writer.write_key_value('testPath', test_path) - writer.write_key_value('status', status) - writer.write_key_value('duration', time) - writer.write_key_value('createdAt', Time.now.to_s) - writer.write_key_value('stderr', e) - writer.write_key_value('stdout', nil) - end + writer.write_object( + { + testPath: test_path, + status: status, + duration: time, + createdAt: Time.now.to_s, + stderr: e, + stdout: nil + } + ) end end @@ -1457,7 +1459,7 @@ def initialize(path) write_new_line end - def write_object + def write_object obj if @is_first_obj @is_first_obj = false else @@ -1465,15 +1467,7 @@ def write_object write_new_line end @indent_level += 1 - write_indent - @file.write("{") - write_new_line - @indent_level += 1 - yield - @indent_level -= 1 - write_new_line - write_indent - @file.write("}") + @file.write(to_json_str(obj)) @indent_level -= 1 @is_first_key_val = true # Occasionally, invalid JSON will be created as shown below, especially when `--repeat-count` is specified. @@ -1492,40 +1486,26 @@ def write_object def write_array(key) @indent_level += 1 - write_indent @file.write(to_json_str(key)) write_colon @file.write(" ", "[") write_new_line end - def write_key_value(key, value) - if @is_first_key_val - @is_first_key_val = false - else - write_comma - write_new_line - end - write_indent - @file.write(to_json_str(key)) - write_colon - @file.write(" ") - @file.write(to_json_str(value)) - end - def close return if @file.closed? close_array @indent_level -= 1 write_new_line - @file.write("}") + @file.write("}", "\n") @file.flush @file.close end private def to_json_str(obj) - JSON.dump(obj) + json = JSON.pretty_generate(obj) + json.gsub(/^/, ' ' * (2 * @indent_level)) end def write_indent diff --git a/tool/lib/webrick/httprequest.rb b/tool/lib/webrick/httprequest.rb index d34eac7ecf6ed0..258ee37a38efbe 100644 --- a/tool/lib/webrick/httprequest.rb +++ b/tool/lib/webrick/httprequest.rb @@ -402,7 +402,7 @@ def fixup() # :nodoc: # This method provides the metavariables defined by the revision 3 # of "The WWW Common Gateway Interface Version 1.1" # To browse the current document of CGI Version 1.1, see below: - # http://tools.ietf.org/html/rfc3875 + # https://www.rfc-editor.org/rfc/rfc3875 def meta_vars meta = Hash.new diff --git a/tool/m4/ruby_wasm_tools.m4 b/tool/m4/ruby_wasm_tools.m4 index c00b33586061a5..efc017e7712884 100644 --- a/tool/m4/ruby_wasm_tools.m4 +++ b/tool/m4/ruby_wasm_tools.m4 @@ -9,7 +9,7 @@ AC_DEFUN([RUBY_WASM_TOOLS], AC_SUBST(wasmoptflags) : ${wasmoptflags=-O3} - AC_MSG_CHECKING([wheather \$WASI_SDK_PATH is set]) + AC_MSG_CHECKING([whether \$WASI_SDK_PATH is set]) AS_IF([test x"${WASI_SDK_PATH}" = x], [ AC_MSG_RESULT([no]) AC_MSG_ERROR([WASI_SDK_PATH environment variable is required]) diff --git a/tool/missing-baseruby.bat b/tool/missing-baseruby.bat index 34b37361e5ef6f..87a9857e066fcc 100755 --- a/tool/missing-baseruby.bat +++ b/tool/missing-baseruby.bat @@ -1,12 +1,14 @@ -: " +:"" == " @echo off || ( :warn echo>&2.%~1 goto :eof :abort exit /b 1 +)||( +:)"||( + s = %^# ) -: " : ; call() { local call=${1#:}; shift; $call "$@"; } : ; warn() { echo "$1" >&2; } : ; abort () { exit 1; } @@ -14,3 +16,4 @@ call :warn "executable host ruby is required. use --with-baseruby option." call :warn "Note that BASERUBY must be Ruby 3.0.0 or later." call :abort +: || (:^; abort if RUBY_VERSION < s[%r"warn .*Ruby ([\d.]+)(?:\.0)?",1]) diff --git a/tool/sync_default_gems.rb b/tool/sync_default_gems.rb index df7e59c9563a4b..0307028eb7569b 100755 --- a/tool/sync_default_gems.rb +++ b/tool/sync_default_gems.rb @@ -461,7 +461,7 @@ def commits_in_ranges(gem, repo, default_branch, ranges) # \r? needed in the regex in case the commit has windows-style line endings (because e.g. we're running # tests on Windows) pattern = "https://github\.com/#{Regexp.quote(repo)}/commit/([0-9a-f]+)\r?$" - log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", &:read) + log = IO.popen(%W"git log -E --grep=#{pattern} -n1 --format=%B", "rb", &:read) ranges = ["#{log[%r[#{pattern}\n\s*(?i:co-authored-by:.*)*\s*\Z], 1]}..#{gem}/#{default_branch}"] end @@ -471,7 +471,7 @@ def commits_in_ranges(gem, repo, default_branch, ranges) range = "#{range}~1..#{range}" end - IO.popen(%W"git log --format=%H,%s #{range} --") do |f| + IO.popen(%W"git log --format=%H,%s #{range} --", "rb") do |f| f.read.split("\n").reverse.map{|commit| commit.split(',', 2)} end end @@ -581,7 +581,7 @@ def pickup_files(gem, changed, picked) def pickup_commit(gem, sha, edit) # Attempt to cherry-pick a commit - result = IO.popen(%W"git cherry-pick #{sha}", &:read) + result = IO.popen(%W"git cherry-pick #{sha}", "rb", &:read) picked = $?.success? if result =~ /nothing\ to\ commit/ `git reset` diff --git a/tool/test/testunit/test_launchable.rb b/tool/test/testunit/test_launchable.rb new file mode 100644 index 00000000000000..a91d44b1cea260 --- /dev/null +++ b/tool/test/testunit/test_launchable.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: false +require 'test/unit' +require 'tempfile' +require 'json' + +class TestLaunchable < Test::Unit::TestCase + def test_json_stream_writer + Tempfile.create(['launchable-test-', '.json']) do |f| + json_stream_writer = Test::Unit::LaunchableOption::JsonStreamWriter.new(f.path) + json_stream_writer.write_array('testCases') + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 42, + status: "TEST_FAILED", + stdout: nil, + stderr: nil, + createdAt: "2021-10-05T12:34:00" + } + ) + json_stream_writer.write_object( + { + testPath: "file=test/test_a.rb#class=class1#testcase=testcase899", + duration: 45, + status: "TEST_PASSED", + stdout: "This is stdout", + stderr: "This is stderr", + createdAt: "2021-10-05T12:36:00" + } + ) + json_stream_writer.close() + expected = <config->malloc #undef xcalloc @@ -91,8 +85,6 @@ struct rb_imemo_tmpbuf_struct { #undef MEMCPY #define MEMCPY(p1,p2,type,n) (p->config->nonempty_memcpy((p1), (p2), sizeof(type), (n))) -#define rb_imemo_tmpbuf_parser_heap p->config->tmpbuf_parser_heap - #define compile_callback p->config->compile_callback #define reg_named_capture_assign p->config->reg_named_capture_assign @@ -211,8 +203,6 @@ struct rb_imemo_tmpbuf_struct { #define rb_ascii8bit_encoding p->config->ascii8bit_encoding #define rb_enc_codelen p->config->enc_codelen #define rb_enc_mbcput p->config->enc_mbcput -#define rb_char_to_option_kcode p->config->char_to_option_kcode -#define rb_ascii8bit_encindex p->config->ascii8bit_encindex #define rb_enc_find_index p->config->enc_find_index #define rb_enc_from_index p->config->enc_from_index #define rb_enc_associate_index p->config->enc_associate_index @@ -221,8 +211,6 @@ struct rb_imemo_tmpbuf_struct { #define ENC_CODERANGE_UNKNOWN p->config->enc_coderange_unknown #define rb_enc_compatible p->config->enc_compatible #define rb_enc_from_encoding p->config->enc_from_encoding -#define ENCODING_GET p->config->encoding_get -#define ENCODING_SET p->config->encoding_set #define ENCODING_IS_ASCII8BIT p->config->encoding_is_ascii8bit #define rb_usascii_encoding p->config->usascii_encoding @@ -231,9 +219,6 @@ struct rb_imemo_tmpbuf_struct { #define rb_local_defined p->config->local_defined #define rb_dvar_defined p->config->dvar_defined -#define literal_cmp p->config->literal_cmp -#define literal_hash p->config->literal_hash - #define rb_syntax_error_append p->config->syntax_error_append #define rb_raise p->config->raise #define syntax_error_new p->config->syntax_error_new diff --git a/win32/Makefile.sub b/win32/Makefile.sub index 80a43d00dae209..7dbb020eee11fc 100644 --- a/win32/Makefile.sub +++ b/win32/Makefile.sub @@ -11,13 +11,15 @@ PATH_SEPARATOR = ; TZ = # skip timezone tests PWD = $(MAKEDIR) empty = +tooldir = $(srcdir)/tool !ifndef MFLAGS MFLAGS=-l !endif !if "$(BASERUBY)" == "" -! if [for %I in (ruby.exe) do @echo BASERUBY = %~s$$PATH:I > baseruby.mk] +! if [ruby $(tooldir)/missing-baseruby.bat 2> nul] +! else if [for %I in (ruby.exe) do @echo BASERUBY = %~s$$PATH:I > baseruby.mk] ! else ! include baseruby.mk ! endif @@ -27,7 +29,7 @@ MFLAGS=-l BASERUBY = !endif !if "$(BASERUBY)" == "" -BASERUBY = echo executable host ruby is required. use --with-baseruby option.^& exit 1 +BASERUBY = $(tooldir:/=\)\missing-baseruby.bat HAVE_BASERUBY = no !else HAVE_BASERUBY = yes @@ -485,7 +487,6 @@ EXTOBJS = dmyext.$(OBJEXT) arch_hdrdir = $(EXTOUT)/include/$(arch) top_srcdir = $(srcdir) hdrdir = $(srcdir)/include -tooldir = $(srcdir)/tool VPATH = $(arch_hdrdir)/ruby;$(hdrdir)/ruby;$(srcdir);$(srcdir)/missing;$(win_srcdir) !ifndef GIT diff --git a/win32/setup.mak b/win32/setup.mak index d4178127bf2282..8c279948215f9c 100644 --- a/win32/setup.mak +++ b/win32/setup.mak @@ -66,6 +66,7 @@ RJIT_SUPPORT = $(RJIT_SUPPORT) # TOOLS << !if defined(BASERUBY) + $(BASERUBY:/=\) "$(srcdir)/tool/missing-baseruby.bat" @echo BASERUBY = $(BASERUBY:/=\)>> $(MAKEFILE) !endif !if "$(RUBY_DEVEL)" == "yes" diff --git a/yjit.h b/yjit.h index 2f5317ad977b6e..dde9f750aaf72b 100644 --- a/yjit.h +++ b/yjit.h @@ -28,6 +28,7 @@ extern uint64_t rb_yjit_call_threshold; extern uint64_t rb_yjit_cold_threshold; extern uint64_t rb_yjit_live_iseq_count; +extern uint64_t rb_yjit_iseq_alloc_count; extern bool rb_yjit_enabled_p; void rb_yjit_incr_counter(const char *counter_name); void rb_yjit_invalidate_all_method_lookup_assumptions(void); diff --git a/yjit.rb b/yjit.rb index 50cb2483980fec..eedd00c358150b 100644 --- a/yjit.rb +++ b/yjit.rb @@ -346,6 +346,7 @@ def _print_stats(out: $stderr) # :nodoc: out.puts "bindings_set: " + format_number(13, stats[:binding_set]) out.puts "compilation_failure: " + format_number(13, compilation_failure) if compilation_failure != 0 out.puts "live_iseq_count: " + format_number(13, stats[:live_iseq_count]) + out.puts "iseq_alloc_count: " + format_number(13, stats[:iseq_alloc_count]) out.puts "compiled_iseq_entry: " + format_number(13, stats[:compiled_iseq_entry]) out.puts "cold_iseq_entry: " + format_number_pct(13, stats[:cold_iseq_entry], stats[:compiled_iseq_entry] + stats[:cold_iseq_entry]) out.puts "compiled_iseq_count: " + format_number(13, stats[:compiled_iseq_count]) diff --git a/yjit/src/asm/arm64/mod.rs b/yjit/src/asm/arm64/mod.rs index eb99c00ba7cba1..8365c34955a2b3 100644 --- a/yjit/src/asm/arm64/mod.rs +++ b/yjit/src/asm/arm64/mod.rs @@ -276,6 +276,9 @@ pub fn cmp(cb: &mut CodeBlock, rn: A64Opnd, rm: A64Opnd) { DataReg::cmp(rn.reg_no, rm.reg_no, rn.num_bits).into() }, + (A64Opnd::Reg(rn), A64Opnd::Imm(imm12)) => { + DataImm::cmp(rn.reg_no, (imm12 as u64).try_into().unwrap(), rn.num_bits).into() + }, (A64Opnd::Reg(rn), A64Opnd::UImm(imm12)) => { DataImm::cmp(rn.reg_no, imm12.try_into().unwrap(), rn.num_bits).into() }, diff --git a/yjit/src/backend/arm64/mod.rs b/yjit/src/backend/arm64/mod.rs index e9319b46e9ea37..092df6326f2e05 100644 --- a/yjit/src/backend/arm64/mod.rs +++ b/yjit/src/backend/arm64/mod.rs @@ -315,7 +315,11 @@ impl Assembler match opnd { Opnd::Reg(_) | Opnd::CArg(_) | Opnd::InsnOut { .. } => opnd, Opnd::Mem(_) => split_load_operand(asm, opnd), - Opnd::Imm(_) => asm.load(opnd), + Opnd::Imm(imm) => if ShiftedImmediate::try_from(imm as u64).is_ok() { + opnd + } else { + asm.load(opnd) + } Opnd::UImm(uimm) => { if ShiftedImmediate::try_from(uimm).is_ok() { opnd @@ -655,7 +659,7 @@ impl Assembler }, Insn::Mul { left, right, .. } => { let opnd0 = split_load_operand(asm, *left); - let opnd1 = split_shifted_immediate(asm, *right); + let opnd1 = split_load_operand(asm, *right); asm.mul(opnd0, opnd1); }, Insn::Test { left, right } => { @@ -1704,4 +1708,35 @@ mod tests { 0x8: csel x1, x11, x12, lt "}); } + + #[test] + fn test_add_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.add(Opnd::Reg(TEMP_REGS[1]), 1.into()); + let out = asm.add(out, 1_usize.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm!(cb, "2b0500b16b0500b1e1030baa", {" + 0x0: adds x11, x9, #1 + 0x4: adds x11, x11, #1 + 0x8: mov x1, x11 + "}); + } + + #[test] + fn test_mul_with_immediate() { + let (mut asm, mut cb) = setup_asm(); + + let out = asm.mul(Opnd::Reg(TEMP_REGS[1]), 3.into()); + asm.mov(Opnd::Reg(TEMP_REGS[0]), out); + asm.compile_with_num_regs(&mut cb, 2); + + assert_disasm!(cb, "6b0080d22b7d0b9be1030baa", {" + 0x0: mov x11, #3 + 0x4: mul x11, x9, x11 + 0x8: mov x1, x11 + "}); + } } diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index 0d536907c1abad..c6e50beab1b156 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -30,6 +30,7 @@ pub use crate::virtualmem::CodePtr; /// Status returned by code generation functions #[derive(PartialEq, Debug)] enum CodegenStatus { + SkipNextInsn, KeepCompiling, EndBlock, } @@ -1197,6 +1198,13 @@ pub fn gen_single_block( // Move to the next instruction to compile insn_idx += insn_len(opcode) as u16; + // Move past next instruction when instructed + if status == Some(SkipNextInsn) { + let next_pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx.into()) }; + let next_opcode: usize = unsafe { rb_iseq_opcode_at_pc(iseq, next_pc) }.try_into().unwrap(); + insn_idx += insn_len(next_opcode) as u16; + } + // If the instruction terminates this block if status == Some(EndBlock) { break; @@ -1343,7 +1351,7 @@ fn jit_putobject(asm: &mut Assembler, arg: VALUE) { fn gen_putobject_int2fix( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, + ocb: &mut OutlinedCb, ) -> Option { let opcode = jit.opcode; let cst_val: usize = if opcode == YARVINSN_putobject_INT2FIX_0_.as_usize() { @@ -1351,22 +1359,86 @@ fn gen_putobject_int2fix( } else { 1 }; + let cst_val = VALUE::fixnum_from_usize(cst_val); + + if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, cst_val, ocb) { + return Some(result); + } - jit_putobject(asm, VALUE::fixnum_from_usize(cst_val)); + jit_putobject(asm, cst_val); Some(KeepCompiling) } fn gen_putobject( jit: &mut JITState, asm: &mut Assembler, - _ocb: &mut OutlinedCb, + ocb: &mut OutlinedCb, ) -> Option { let arg: VALUE = jit.get_arg(0); + if let Some(result) = fuse_putobject_opt_ltlt(jit, asm, arg, ocb) { + return Some(result); + } + jit_putobject(asm, arg); Some(KeepCompiling) } +/// Combine `putobject` and and `opt_ltlt` together if profitable, for example when +/// left shifting an integer by a constant amount. +fn fuse_putobject_opt_ltlt( + jit: &mut JITState, + asm: &mut Assembler, + constant_object: VALUE, + ocb: &mut OutlinedCb, +) -> Option { + let next_opcode = unsafe { rb_vm_insn_addr2opcode(jit.pc.add(insn_len(jit.opcode).as_usize()).read().as_ptr()) }; + if next_opcode == YARVINSN_opt_ltlt as i32 && constant_object.fixnum_p() { + // Untag the fixnum shift amount + let shift_amt = constant_object.as_isize() >> 1; + if shift_amt > 63 || shift_amt < 0 { + return None; + } + if !jit.at_current_insn() { + defer_compilation(jit, asm, ocb); + return Some(EndBlock); + } + + let lhs = jit.peek_at_stack(&asm.ctx, 0); + if !lhs.fixnum_p() { + return None; + } + + if !assume_bop_not_redefined(jit, asm, ocb, INTEGER_REDEFINED_OP_FLAG, BOP_LTLT) { + return None; + } + + asm_comment!(asm, "integer left shift with rhs={shift_amt}"); + let lhs = asm.stack_opnd(0); + + // Guard that lhs is a fixnum if necessary + let lhs_type = asm.ctx.get_opnd_type(lhs.into()); + if lhs_type != Type::Fixnum { + asm_comment!(asm, "guard arg0 fixnum"); + asm.test(lhs, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); + + jit_chain_guard( + JCC_JZ, + jit, + asm, + ocb, + SEND_MAX_DEPTH, + Counter::guard_send_not_fixnums, + ); + } + + asm.stack_pop(1); + fixnum_left_shift_body(asm, lhs, shift_amt as u64); + return Some(SkipNextInsn); + } + return None; +} + fn gen_putself( _jit: &mut JITState, asm: &mut Assembler, @@ -5073,8 +5145,13 @@ fn jit_rb_int_lshift( Counter::lshift_amount_changed, ); + fixnum_left_shift_body(asm, lhs, shift_amt as u64); + true +} + +fn fixnum_left_shift_body(asm: &mut Assembler, lhs: Opnd, shift_amt: u64) { let in_val = asm.sub(lhs, 1.into()); - let shift_opnd = Opnd::UImm(shift_amt as u64); + let shift_opnd = Opnd::UImm(shift_amt); let out_val = asm.lshift(in_val, shift_opnd); let unshifted = asm.rshift(out_val, shift_opnd); @@ -5087,7 +5164,6 @@ fn jit_rb_int_lshift( let ret_opnd = asm.stack_push(Type::Fixnum); asm.mov(ret_opnd, out_val); - true } fn jit_rb_int_rshift( @@ -10384,57 +10460,6 @@ mod tests { assert!(cb.get_write_pos() > 0); } - #[test] - fn test_putobject_qtrue() { - // Test gen_putobject with Qtrue - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); - - let mut value_array: [u64; 2] = [0, Qtrue.into()]; - let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; - jit.pc = pc; - - let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - - let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); - - assert_eq!(status, Some(KeepCompiling)); - assert_eq!(tmp_type_top, Type::True); - asm.compile(&mut cb, None).unwrap(); - assert!(cb.get_write_pos() > 0); - } - - #[test] - fn test_putobject_fixnum() { - // Test gen_putobject with a Fixnum to test another conditional branch - let (mut jit, _context, mut asm, mut cb, mut ocb) = setup_codegen(); - - // The Fixnum 7 is encoded as 7 * 2 + 1, or 15 - let mut value_array: [u64; 2] = [0, 15]; - let pc: *mut VALUE = &mut value_array as *mut u64 as *mut VALUE; - jit.pc = pc; - - let status = gen_putobject(&mut jit, &mut asm, &mut ocb); - - let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); - - assert_eq!(status, Some(KeepCompiling)); - assert_eq!(tmp_type_top, Type::Fixnum); - asm.compile(&mut cb, None).unwrap(); - assert!(cb.get_write_pos() > 0); - } - - #[test] - fn test_int2fix() { - let (mut jit, _context, mut asm, _cb, mut ocb) = setup_codegen(); - jit.opcode = YARVINSN_putobject_INT2FIX_0_.as_usize(); - let status = gen_putobject_int2fix(&mut jit, &mut asm, &mut ocb); - - let tmp_type_top = asm.ctx.get_opnd_type(StackOpnd(0)); - - // Right now we're not testing the generated machine code to make sure a literal 1 or 0 was pushed. I've checked locally. - assert_eq!(status, Some(KeepCompiling)); - assert_eq!(tmp_type_top, Type::Fixnum); - } #[test] fn test_putself() { diff --git a/yjit/src/stats.rs b/yjit/src/stats.rs index 621854d48793b2..fb4ed03bd5c4af 100644 --- a/yjit/src/stats.rs +++ b/yjit/src/stats.rs @@ -15,10 +15,14 @@ use crate::cruby::*; use crate::options::*; use crate::yjit::yjit_enabled_p; -/// A running total of how many ISeqs are in the system. +/// Running total of how many ISeqs are in the system. #[no_mangle] pub static mut rb_yjit_live_iseq_count: u64 = 0; +/// Monotonically increasing total of how many ISEQs were allocated +#[no_mangle] +pub static mut rb_yjit_iseq_alloc_count: u64 = 0; + /// A middleware to count Rust-allocated bytes as yjit_alloc_size. #[global_allocator] static GLOBAL_ALLOCATOR: StatsAlloc = StatsAlloc { alloc_size: AtomicUsize::new(0) }; @@ -748,6 +752,7 @@ fn rb_yjit_gen_stats_dict(context: bool) -> VALUE { hash_aset_usize!(hash, "vm_insns_count", rb_vm_insns_count as usize); hash_aset_usize!(hash, "live_iseq_count", rb_yjit_live_iseq_count as usize); + hash_aset_usize!(hash, "iseq_alloc_count", rb_yjit_iseq_alloc_count as usize); } // If we're not generating stats, put only default counters