patch 8.2.2784: Vim9: cannot use \=expr in :substitute

Problem:    Vim9: cannot use \=expr in :substitute.
Solution:   Compile the expression into instructions and execute them when
            invoked.

src/ex_cmds.c

@@ -3603,6 +3603,29 @@ typedef struct {
     int	do_ic;		// ignore case flag
 } subflags_T;
 
+/*
+ * Skip over the "sub" part in :s/pat/sub/ where "delimiter" is the separating
+ * character.
+ */
+    char_u *
+skip_substitute(char_u *start, int delimiter)
+{
+    char_u *p = start;
+
+    while (p[0])
+    {
+	if (p[0] == delimiter)		// end delimiter found
+	{
+	    *p++ = NUL;			// replace it with a NUL
+	    break;
+	}
+	if (p[0] == '\\' && p[1] != 0)	// skip escaped characters
+	    ++p;
+	MB_PTR_ADV(p);
+    }
+    return p;
+}
+
 /*
  * Perform a substitution from line eap->line1 to line eap->line2 using the
  * command pointed to by eap->arg which should be of the form:
@@ -3704,18 +3727,7 @@ ex_substitute(exarg_T *eap)
 	 * Vim we want to use '\n' to find/substitute a NUL.
 	 */
 	sub = cmd;	    // remember the start of the substitution
-
-	while (cmd[0])
-	{
-	    if (cmd[0] == delimiter)		// end delimiter found
-	    {
-		*cmd++ = NUL;			// replace it with a NUL
-		break;
-	    }
-	    if (cmd[0] == '\\' && cmd[1] != 0)	// skip escaped characters
-		++cmd;
-	    MB_PTR_ADV(cmd);
-	}
+	cmd = skip_substitute(cmd, delimiter);
 
 	if (!eap->skip)
 	{

src/globals.h

@@ -1379,6 +1379,9 @@ EXTERN char_u no_lines_msg[]	INIT(= N_("--No lines in buffer--"));
 EXTERN long	sub_nsubs;	// total number of substitutions
 EXTERN linenr_T	sub_nlines;	// total number of lines changed
 
+// Used when a compiled :substitute has an expression.
+EXTERN struct subs_expr_S	*substitute_instr INIT(= NULL);
+
 // table to store parsed 'wildmode'
 EXTERN char_u	wim_flags[4];
 

src/proto/ex_cmds.pro

@@ -27,6 +27,7 @@ void ex_change(exarg_T *eap);
 void ex_z(exarg_T *eap);
 int check_restricted(void);
 int check_secure(void);
+char_u *skip_substitute(char_u *start, int delimiter);
 void ex_substitute(exarg_T *eap);
 int do_sub_msg(int count_only);
 void ex_global(exarg_T *eap);

src/proto/vim9execute.pro

@@ -4,6 +4,7 @@ void funcstack_check_refcount(funcstack_T *funcstack);
 char_u *char_from_string(char_u *str, varnumber_T index);
 char_u *string_slice(char_u *str, varnumber_T first, varnumber_T last, int exclusive);
 int fill_partial_and_closure(partial_T *pt, ufunc_T *ufunc, ectx_T *ectx);
+char_u *exe_substitute_instr(void);
 int call_def_function(ufunc_T *ufunc, int argc_arg, typval_T *argv, partial_T *partial, typval_T *rettv);
 void ex_disassemble(exarg_T *eap);
 int tv2bool(typval_T *tv);

src/regexp.c

@@ -2069,6 +2069,9 @@ vim_regsub_both(
 		}
 		clear_tv(&rettv);
 	    }
+	    else if (substitute_instr != NULL)
+		// Execute instructions from ISN_SUBSTITUTE.
+		eval_result = exe_substitute_instr();
 	    else
 		eval_result = eval_to_string(source + 2, TRUE);
 

src/testdir/test_vim9_cmd.vim

@@ -1172,5 +1172,27 @@ def Test_lockvar()
   CheckDefFailure(lines, 'E1178', 2)
 enddef
 
+def Test_substitute_expr()
+  var to = 'repl'
+  new
+  setline(1, 'one from two')
+  s/from/\=to
+  assert_equal('one repl two', getline(1))
+
+  setline(1, 'one from two')
+  s/from/\=to .. '_x'
+  assert_equal('one repl_x two', getline(1))
+
+  setline(1, 'one from two from three')
+  var also = 'also'
+  s/from/\=to .. '_' .. also/g#e
+  assert_equal('one repl_also two repl_also three', getline(1))
+
+  CheckDefFailure(['s/from/\="x")/'], 'E488:')
+  CheckDefFailure(['s/from/\="x"/9'], 'E488:')
+
+  bwipe!
+enddef
+
 
 " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker

src/testdir/test_vim9_disassemble.vim

@@ -121,6 +121,25 @@ def Test_disassemble_exec_expr()
         res)
 enddef
 
+def s:Substitute()
+  var expr = "abc"
+  :%s/a/\=expr/&g#c
+enddef
+
+def Test_disassemble_substitute()
+  var res = execute('disass s:Substitute')
+  assert_match('<SNR>\d*_Substitute.*' ..
+        ' var expr = "abc"\_s*' ..
+        '\d PUSHS "abc"\_s*' ..
+        '\d STORE $0\_s*' ..
+        ' :%s/a/\\=expr/&g#c\_s*' ..
+        '\d SUBSTITUTE   :%s/a/\\=expr/&g#c\_s*' ..
+        '    0 LOAD $0\_s*' ..
+        '    -------------\_s*' ..
+        '\d RETURN 0',
+        res)
+enddef
+
 def s:YankRange()
   norm! m[jjm]
   :'[,']yank

src/version.c

@@ -750,6 +750,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    2784,
 /**/
     2783,
 /**/

src/vim9.h

@@ -19,6 +19,7 @@ typedef enum {
     ISN_ECHOMSG,    // echo Ex commands isn_arg.number items on top of stack
     ISN_ECHOERR,    // echo Ex commands isn_arg.number items on top of stack
     ISN_RANGE,	    // compute range from isn_arg.string, push to stack
+    ISN_SUBSTITUTE, // :s command with expression
 
     // get and set variables
     ISN_LOAD,	    // push local variable isn_arg.number
@@ -94,7 +95,8 @@ typedef enum {
 
     // expression operations
     ISN_JUMP,	    // jump if condition is matched isn_arg.jump
-    ISN_JUMP_IF_ARG_SET, // jump if argument is already set, uses isn_arg.jumparg
+    ISN_JUMP_IF_ARG_SET, // jump if argument is already set, uses
+			 // isn_arg.jumparg
 
     // loop
     ISN_FOR,	    // get next item from a list, uses isn_arg.forloop
@@ -165,7 +167,9 @@ typedef enum {
 
     ISN_UNPACK,	    // unpack list into items, uses isn_arg.unpack
     ISN_SHUFFLE,    // move item on stack up or down
-    ISN_DROP	    // pop stack and discard value
+    ISN_DROP,	    // pop stack and discard value
+
+    ISN_FINISH	    // end marker in list of instructions
 } isntype_T;
 
 
@@ -339,6 +343,12 @@ typedef struct {
     int		outer_depth;	// nesting level, stack frames to go up
 } isn_outer_T;
 
+// arguments to ISN_SUBSTITUTE
+typedef struct {
+    char_u	*subs_cmd;	// :s command
+    isn_T	*subs_instr;	// sequence of instructions
+} subs_T;
+
 /*
  * Instruction
  */
@@ -381,6 +391,7 @@ struct isn_S {
 	cmod_T		    cmdmod;
 	unpack_T	    unpack;
 	isn_outer_T	    outer;
+	subs_T		    subs;
     } isn_arg;
 };
 

src/vim9compile.c

@@ -2130,6 +2130,33 @@ generate_EXECCONCAT(cctx_T *cctx, int count)
     return OK;
 }
 
+    static int
+generate_substitute(char_u *cmd, int instr_start, cctx_T *cctx)
+{
+    isn_T	*isn;
+    isn_T	*instr;
+    int		instr_count = cctx->ctx_instr.ga_len - instr_start;
+
+    instr = ALLOC_MULT(isn_T, instr_count + 1);
+    if (instr == NULL)
+	return FAIL;
+    // Move the generated instructions into the ISN_SUBSTITUTE instructions,
+    // then truncate the list of instructions, so they are used only once.
+    mch_memmove(instr, ((isn_T *)cctx->ctx_instr.ga_data) + instr_start,
+					      instr_count * sizeof(isn_T));
+    instr[instr_count].isn_type = ISN_FINISH;
+    cctx->ctx_instr.ga_len = instr_start;
+
+    if ((isn = generate_instr(cctx, ISN_SUBSTITUTE)) == NULL)
+    {
+	vim_free(instr);
+	return FAIL;
+    }
+    isn->isn_arg.subs.subs_cmd = vim_strsave(cmd);
+    isn->isn_arg.subs.subs_instr = instr;
+    return OK;
+}
+
 /*
  * Generate ISN_RANGE.  Consumes "range".  Return OK/FAIL.
  */
@@ -8465,6 +8492,55 @@ compile_exec(char_u *line, exarg_T *eap, cctx_T *cctx)
     return nextcmd;
 }
 
+/*
+ * :s/pat/repl/
+ */
+    static char_u *
+compile_substitute(char_u *arg, exarg_T *eap, cctx_T *cctx)
+{
+    char_u  *cmd = eap->arg;
+    char_u  *expr = (char_u *)strstr((char *)cmd, "\\=");
+
+    if (expr != NULL)
+    {
+	int delimiter = *cmd++;
+
+	// There is a \=expr, find it in the substitute part.
+	cmd = skip_regexp_ex(cmd, delimiter, magic_isset(),
+							     NULL, NULL, NULL);
+	if (cmd[0] == delimiter && cmd[1] == '\\' && cmd[2] == '=')
+	{
+	    int	    instr_count = cctx->ctx_instr.ga_len;
+	    char_u  *end;
+
+	    cmd += 3;
+	    end = skip_substitute(cmd, delimiter);
+
+	    compile_expr0(&cmd, cctx);
+	    if (end[-1] == NUL)
+		end[-1] = delimiter;
+	    cmd = skipwhite(cmd);
+	    if (*cmd != delimiter && *cmd != NUL)
+	    {
+		semsg(_(e_trailing_arg), cmd);
+		return NULL;
+	    }
+
+	    if (generate_substitute(arg, instr_count, cctx) == FAIL)
+		return NULL;
+
+	    // skip over flags
+	    if (*end == '&')
+		++end;
+	    while (ASCII_ISALPHA(*end) || *end == '#')
+		++end;
+	    return end;
+	}
+    }
+
+    return compile_exec(arg, eap, cctx);
+}
+
 /*
  * Add a function to the list of :def functions.
  * This sets "ufunc->uf_dfunc_idx" but the function isn't compiled yet.
@@ -8996,6 +9072,16 @@ compile_def_function(
 		    line = compile_put(p, &ea, &cctx);
 		    break;
 
+	    case CMD_substitute:
+		    if (cctx.ctx_skip == SKIP_YES)
+			line = (char_u *)"";
+		    else
+		    {
+			ea.arg = p;
+			line = compile_substitute(line, &ea, &cctx);
+		    }
+		    break;
+
 	    // TODO: any other commands with an expression argument?
 
 	    case CMD_append:
@@ -9223,6 +9309,11 @@ delete_instr(isn_T *isn)
 	    vim_free(isn->isn_arg.string);
 	    break;
 
+	case ISN_SUBSTITUTE:
+	    vim_free(isn->isn_arg.subs.subs_cmd);
+	    vim_free(isn->isn_arg.subs.subs_instr);
+	    break;
+
 	case ISN_LOADS:
 	case ISN_STORES:
 	    vim_free(isn->isn_arg.loadstore.ls_name);
@@ -9400,6 +9491,7 @@ delete_instr(isn_T *isn)
 	case ISN_UNLETINDEX:
 	case ISN_UNLETRANGE:
 	case ISN_UNPACK:
+	case ISN_FINISH:
 	    // nothing allocated
 	    break;
     }