Switch to Refcounted Stack

MANIFEST

@@ -4897,6 +4897,7 @@ ext/XS-APItest/t/ptr_table.t			Test ptr_table_* APIs
 ext/XS-APItest/t/push.t				XS::APItest extension
 ext/XS-APItest/t/refs.t				Test typemap ref handling
 ext/XS-APItest/t/rmagical.t			XS::APItest extension
+ext/XS-APItest/t/rpp_invoke_xs.t		XS::APItest: test rpp_invoke_xs()
 ext/XS-APItest/t/rv2cv_op_cv.t			test rv2cv_op_cv() API
 ext/XS-APItest/t/savehints.t			test SAVEHINTS() API
 ext/XS-APItest/t/savestack.t			test savestack behavior, currently only in the regex engine
@@ -4943,6 +4944,7 @@ ext/XS-APItest/t/whichsig.t			XS::APItest: tests for whichsig() and variants
 ext/XS-APItest/t/win32.t			Test Win32 specific APIs
 ext/XS-APItest/t/xs_special_subs.t		Test that XS BEGIN/CHECK/INIT/END work
 ext/XS-APItest/t/xs_special_subs_require.t	for require too
+ext/XS-APItest/t/xsppwrap.t			Tests for XSPP_wrapped() macro
 ext/XS-APItest/t/xsub_h.t			Tests for XSUB.h
 ext/XS-APItest/typemap
 ext/XS-APItest/XSUB-redefined-macros.xs		XS code needing redefined macros.
@@ -6107,6 +6109,7 @@ t/op/readdir.t				See if readdir() works
 t/op/readline.t				See if <> / readline / rcatline work
 t/op/recurse.t				See if deep recursion works
 t/op/ref.t				See if refs and objects work
+t/op/refstack.t				See if a ref counted stack fixes things
 t/op/repeat.t				See if x operator works
 t/op/require_37033.t			See if require always closes rsfp
 t/op/require_errors.t			See if errors from require are reported correctly

autodoc.pl

@@ -144,6 +144,7 @@
 my $MRO_scn = 'MRO';
 my $multicall_scn = 'Multicall Functions';
 my $numeric_scn = 'Numeric Functions';
+my $rpp_scn = 'Reference-counted stack manipulation';
 
 # Now combined, as unclear which functions go where, but separate names kept
 # to avoid 1) other code changes; 2) in case it seems better to split again
@@ -326,6 +327,14 @@
             See L<perlform>.
             EOT
       },
+    $rpp_scn => {
+        header => <<~'EOT',
+            Functions for pushing and pulling items on the stack when the
+            stack is reference counted. They are intended as replacements
+            for the old PUSHs, POPi, EXTEND etc pp macros within pp
+            functions.
+            EOT
+      },
     $signals_scn => {},
     $site_scn => {
         header => <<~'EOT',
@@ -1724,7 +1733,7 @@ sub output {
 
         # We allow empty sections in perlintern.
         if (! $section_info && $podname eq 'perlapi') {
-            warn "Empty section '$section_name'; skipped";
+            warn "Empty section '$section_name' for $podname; skipped";
             next;
         }
 

av.c

@@ -42,8 +42,8 @@ Perl_av_reify(pTHX_ AV *av)
             SvREFCNT_inc_simple_void(sv);
     }
     key = AvARRAY(av) - AvALLOC(av);
-    while (key)
-        AvALLOC(av)[--key] = NULL;
+    if (key)
+        Zero(AvALLOC(av), key, SV*);
     AvREIFY_off(av);
     AvREAL_on(av);
 }
@@ -633,7 +633,6 @@ to it.
 void
 Perl_av_clear(pTHX_ AV *av)
 {
-    SSize_t extra;
     bool real;
     SSize_t orig_ix = 0;
 
@@ -678,12 +677,9 @@ Perl_av_clear(pTHX_ AV *av)
             SvREFCNT_dec(sv);
         }
     }
-    extra = AvARRAY(av) - AvALLOC(av);
-    if (extra) {
-        AvMAX(av) += extra;
-        AvARRAY(av) = AvALLOC(av);
-    }
     AvFILLp(av) = -1;
+    av_remove_offset(av);
+
     if (real) {
         /* disarm av's premature free guard */
         if (LIKELY(PL_tmps_ix == orig_ix))
@@ -902,6 +898,9 @@ Perl_av_unshift(pTHX_ AV *av, SSize_t num)
         AvMAX(av) += i;
         AvFILLp(av) += i;
         AvARRAY(av) = AvARRAY(av) - i;
+#ifdef PERL_RC_STACK
+        Zero(AvARRAY(av), i, SV*);
+#endif
     }
     if (num) {
         SV **ary;
@@ -955,8 +954,10 @@ Perl_av_shift(pTHX_ AV *av)
     if (AvFILL(av) < 0)
       return &PL_sv_undef;
     retval = *AvARRAY(av);
+#ifndef PERL_RC_STACK
     if (AvREAL(av))
         *AvARRAY(av) = NULL;
+#endif
     AvARRAY(av) = AvARRAY(av) + 1;
     AvMAX(av)--;
     AvFILLp(av)--;

class.c

@@ -47,7 +47,6 @@ Perl_newSVobject(pTHX_ Size_t fieldcount)
 
 PP(pp_initfield)
 {
-    dSP;
     UNOP_AUX_item *aux = cUNOP_AUX->op_aux;
 
     SV *self = PAD_SVl(PADIX_SELF);
@@ -62,8 +61,10 @@ PP(pp_initfield)
 
     switch(PL_op->op_private & (OPpINITFIELD_AV|OPpINITFIELD_HV)) {
         case 0:
-            if(PL_op->op_flags & OPf_STACKED)
-                val = newSVsv(POPs);
+            if(PL_op->op_flags & OPf_STACKED) {
+                val = newSVsv(*PL_stack_sp);
+                rpp_popfree_1();
+            }
             else
                 val = newSV(0);
             break;
@@ -73,15 +74,16 @@ PP(pp_initfield)
             AV *av;
             if(PL_op->op_flags & OPf_STACKED) {
                 SV **svp = PL_stack_base + POPMARK + 1;
-                STRLEN count = SP - svp + 1;
+                STRLEN count = PL_stack_sp - svp + 1;
 
                 av = newAV_alloc_x(count);
 
                 av_extend(av, count);
-                while(svp <= SP) {
+                while(svp <= PL_stack_sp) {
                     av_push_simple(av, newSVsv(*svp));
                     svp++;
                 }
+                rpp_popfree_to(PL_stack_sp - count);
             }
             else
                 av = newAV();
@@ -94,18 +96,19 @@ PP(pp_initfield)
             HV *hv = newHV();
             if(PL_op->op_flags & OPf_STACKED) {
                 SV **svp = PL_stack_base + POPMARK + 1;
-                STRLEN svcount = SP - svp + 1;
+                STRLEN svcount = PL_stack_sp - svp + 1;
 
                 if(svcount % 2)
                     Perl_warner(aTHX_
                             packWARN(WARN_MISC), "Odd number of elements in hash field initialization");
 
-                while(svp <= SP) {
+                while(svp <= PL_stack_sp) {
                     SV *key = *svp; svp++;
-                    SV *val = svp <= SP ? *svp : &PL_sv_undef; svp++;
+                    SV *val = svp <= PL_stack_sp ? *svp : &PL_sv_undef; svp++;
 
                     (void)hv_store_ent(hv, key, newSVsv(val), 0);
                 }
+                rpp_popfree_to(PL_stack_sp - svcount);
             }
             val = (SV *)hv;
             break;
@@ -121,7 +124,7 @@ PP(pp_initfield)
         save_freesv(sv);
     }
 
-    RETURN;
+    return NORMAL;
 }
 
 XS(injected_constructor);
@@ -239,7 +242,9 @@ XS(injected_constructor)
 /* TODO: People would probably expect to find this in pp.c  ;) */
 PP(pp_methstart)
 {
-    SV *self = av_shift(GvAV(PL_defgv));
+    /* note that if AvREAL(@_), be careful not to leak self:
+     * so keep it in @_ for now, and only shift it later */
+    SV *self = *(av_fetch(GvAV(PL_defgv), 0, 1));
     SV *rv = NULL;
 
     /* pp_methstart happens before the first OP_NEXTSTATE of the method body,
@@ -297,6 +302,11 @@ PP(pp_methstart)
         }
     }
 
+    /* safe to shift and free self now */
+    self = av_shift(GvAV(PL_defgv));
+    if (AvREAL(GvAV(PL_defgv)))
+        SvREFCNT_dec_NN(self);
+
     if(PL_op->op_private & OPpINITFIELDS) {
         SV *params = *av_fetch(GvAV(PL_defgv), 0, 0);
         if(params && SvTYPE(params) == SVt_PVHV) {
@@ -438,15 +448,10 @@ static const char *S_split_package_ver(pTHX_ SV *value, SV *pkgname, SV *pkgvers
 #define ensure_module_version(module, version)  S_ensure_module_version(aTHX_ module, version)
 static void S_ensure_module_version(pTHX_ SV *module, SV *version)
 {
-    dSP;
-
     ENTER;
 
-    PUSHMARK(SP);
-    PUSHs(module);
-    PUSHs(version);
-    PUTBACK;
-
+    PUSHMARK(PL_stack_sp);
+    rpp_xpush_2(module, version);
     call_method("VERSION", G_VOID);
 
     LEAVE;
@@ -1070,17 +1075,15 @@ Perl_ck_classname(pTHX_ OP *o)
 
 PP(pp_classname)
 {
-    dSP;
     dTARGET;
 
     SV *self = PAD_SVl(PADIX_SELF);
     assert(SvTYPE(SvRV(self)) == SVt_PVOBJ);
 
-    EXTEND(SP, 1);
-    PUSHs(TARG);
+    rpp_xpush_1(TARG);
     sv_ref(TARG, SvRV(self), true);
 
-    RETURN;
+    return NORMAL;
 }
 
 /*

cop.h

@@ -889,12 +889,14 @@ struct block_format {
     } STMT_END
 
 /* junk in @_ spells trouble when cloning CVs and in pp_caller(), so don't
- * leave any (a fast av_clear(ary), basically) */
+ * leave any (a fast av_clear(ary), basically).
+ * New code should probably be using Perl_clear_defarray_simple()
+ * and/or Perl_clear_defarray()
+ */
 #define CLEAR_ARGARRAY(ary) \
     STMT_START {							\
-        AvMAX(ary) += AvARRAY(ary) - AvALLOC(ary);			\
-        AvARRAY(ary) = AvALLOC(ary);					\
         AvFILLp(ary) = -1;						\
+        av_remove_offset(ary);                                          \
     } STMT_END
 
 
@@ -1259,6 +1261,15 @@ struct stackinfo {
     I32			si_markoff;	/* offset where markstack begins for us.
                                          * currently used only with DEBUGGING,
                                          * but not #ifdef-ed for bincompat */
+
+#ifdef PERL_RC_STACK
+                                        /* index of first entry in the argument
+                                           stack which is not ref-counted. If
+                                           set to 0 (default), all stack
+                                           elements are ref-counted */
+    I32                 si_stack_nonrc_base;
+#endif
+
 #if defined DEBUGGING && !defined DEBUGGING_RE_ONLY
 /* high water mark: for checking if the stack was correctly extended /
  * tested for extension by each pp function */
@@ -1292,54 +1303,31 @@ typedef struct stackinfo PERL_SI;
 #  define PUSHSTACK_INIT_HWM(si) NOOP
 #endif
 
+/* for backcompat; use push_stackinfo() instead */
+
 #define PUSHSTACKi(type) \
-    STMT_START {							\
-        PERL_SI *next = PL_curstackinfo->si_next;			\
-        DEBUG_l({							\
-            int i = 0; PERL_SI *p = PL_curstackinfo;			\
-            while (p) { i++; p = p->si_prev; }				\
-            Perl_deb(aTHX_ "push STACKINFO %d in %s at %s:%d\n",        \
-                         i, SAFE_FUNCTION__, __FILE__, __LINE__);})        \
-        if (!next) {							\
-            next = new_stackinfo(32, 2048/sizeof(PERL_CONTEXT) - 1);	\
-            next->si_prev = PL_curstackinfo;				\
-            PL_curstackinfo->si_next = next;				\
-        }								\
-        next->si_type = type;						\
-        next->si_cxix = -1;						\
-        next->si_cxsubix = -1;						\
-        PUSHSTACK_INIT_HWM(next);                                       \
-        AvFILLp(next->si_stack) = 0;					\
-        SWITCHSTACK(PL_curstack,next->si_stack);			\
-        PL_curstackinfo = next;						\
-        SET_MARK_OFFSET;						\
+    STMT_START {		\
+        PL_stack_sp = sp;       \
+        push_stackinfo(type, 0);\
+        sp = PL_stack_sp ;      \
     } STMT_END
 
 #define PUSHSTACK PUSHSTACKi(PERLSI_UNKNOWN)
 
-/* POPSTACK works with PL_stack_sp, so it may need to be bracketed by
+
+/* for backcompat; use pop_stackinfo() instead.
+ *
+ * POPSTACK works with PL_stack_sp, so it may need to be bracketed by
  * PUTBACK/SPAGAIN to flush/refresh any local SP that may be active */
-#define POPSTACK \
-    STMT_START {							\
-        dSP;								\
-        PERL_SI * const prev = PL_curstackinfo->si_prev;		\
-        DEBUG_l({							\
-            int i = -1; PERL_SI *p = PL_curstackinfo;			\
-            while (p) { i++; p = p->si_prev; }				\
-            Perl_deb(aTHX_ "pop  STACKINFO %d in %s at %s:%d\n",        \
-                         i, SAFE_FUNCTION__, __FILE__, __LINE__);})        \
-        if (!prev) {							\
-            Perl_croak_popstack();					\
-        }								\
-        SWITCHSTACK(PL_curstack,prev->si_stack);			\
-        /* don't free prev here, free them all at the END{} */		\
-        PL_curstackinfo = prev;						\
-    } STMT_END
+
+#define POPSTACK pop_stackinfo()
+
 
 #define POPSTACK_TO(s) \
     STMT_START {							\
         while (PL_curstack != s) {					\
             dounwind(-1);						\
+            rpp_obliterate_stack_to(0);					\
             POPSTACK;							\
         }								\
     } STMT_END

cv.h

@@ -138,6 +138,8 @@ See L<perlguts/Autoloading with XSUBs>.
 #define CVf_IsMETHOD    0x100000 /* CV is a (real) method of a real class. Not
                                    to be confused with what used to be called
                                    CVf_METHOD; now CVf_NOWARN_AMBIGUOUS */
+#define CVf_XS_RCSTACK  0x200000 /* the XS function understands a
+                                    reference-counted stack */
 
 /* This symbol for optimised communication between toke.c and op.c: */
 #define CVf_BUILTIN_ATTRS	(CVf_NOWARN_AMBIGUOUS|CVf_LVALUE|CVf_ANONCONST)
@@ -264,6 +266,10 @@ Helper macro to turn off the C<CvREFCOUNTED_ANYSV> flag.
 #define CvIsMETHOD_on(cv)	(CvFLAGS(cv) |= CVf_IsMETHOD)
 #define CvIsMETHOD_off(cv)	(CvFLAGS(cv) &= ~CVf_IsMETHOD)
 
+#define CvXS_RCSTACK(cv)        (CvFLAGS(cv) & CVf_XS_RCSTACK)
+#define CvXS_RCSTACK_on(cv)     (CvFLAGS(cv) |= CVf_XS_RCSTACK)
+#define CvXS_RCSTACK_off(cv)    (CvFLAGS(cv) &= ~CVf_XS_RCSTACK)
+
 /* Back-compat */
 #ifndef PERL_CORE
 #  define CVf_METHOD            CVf_NOWARN_AMBIGUOUS