patch 9.1.1054: Vim doesn't work well with TERM=xterm-direct

Problem:  Vim doesn't work well with TERM=xterm-direct
          (Andrea Pappacoda)
Solution: detect if a terminal supports true-colors and
          enable termguicolors

The terminfo database for xterm-direct contains both the (non-standard)
termcap RGB capability and a number of colors == 0x1000000 so it seems
either of those two options can be used to detect a terminal capable of
displaying true colors.

So set the termguicolor option automatically, when either of the two
options is detected. (for some reasons, my debian xterm (v393) does not
respond to XTGETTCAP query attempts, so falling back to the number of
colors seems like a good compromize)

fixes: #16327
closes: #16490

Signed-off-by: Christian Brabandt <cb@256bit.org>

runtime/doc/options.txt

@@ -1,4 +1,4 @@
-*options.txt*	For Vim version 9.1.  Last change: 2025 Jan 23
+*options.txt*	For Vim version 9.1.  Last change: 2025 Jan 26
 
 
 		  VIM REFERENCE MANUAL	  by Bram Moolenaar
@@ -8501,6 +8501,11 @@ A jump table for the options with a short description can be found at |Q_op|.
 	When on, uses |highlight-guifg| and |highlight-guibg| attributes in
 	the terminal (thus using 24-bit color).
 
+	Will automatically be enabled, if Vim detects that it runs in a
+	capable terminal (when the terminal supports the RGB terminfo
+	capability or when the number of colors |t_Co| supported by the
+	terminal is 0x1000000, e.g. with $TERM=xterm-direct).
+
 	Requires a ISO-8613-3 compatible terminal.  If setting this option
 	does not work (produces a colorless UI) reading |xterm-true-color|
 	might help.

runtime/doc/version9.txt

@@ -1,4 +1,4 @@
-*version9.txt*  For Vim version 9.1.  Last change: 2025 Jan 25
+*version9.txt*  For Vim version 9.1.  Last change: 2025 Jan 26
 
 
 		  VIM REFERENCE MANUAL    by Bram Moolenaar
@@ -41628,6 +41628,8 @@ Changed~
 - New option value "nosort" for 'completeopt'
 - add |dist#vim9#Launch()| and |dist#vim9#Open()| to the |vim-script-library|
   and decouple it from |netrw|
+- 'termguicolors' is automatically enabled if the terminal supports the RGB
+  terminfo capability or supports 0x1000000 colors
 
 							*added-9.2*
 Added ~

src/term.c

@@ -1661,6 +1661,11 @@ set_color_count(int nr)
 	sprintf((char *)nr_colors, "%d", t_colors);
     else
 	*nr_colors = NUL;
+#ifdef FEAT_TERMGUICOLORS
+    // xterm-direct, enable termguicolors
+    if (t_colors == 0x1000000 && !p_tgc)
+	set_option_value((char_u *)"termguicolors", 1L, NULL, 0);
+#endif
     set_string_option_direct((char_u *)"t_Co", -1, nr_colors, OPT_FREE, 0);
 }
 
@@ -1694,8 +1699,9 @@ may_adjust_color_count(int val)
 static char *(key_names[]) =
 {
 # ifdef FEAT_TERMRESPONSE
-    // Do this one first, it may cause a screen redraw.
+    // Do those ones first, both may cause a screen redraw.
     "Co",
+    "RGB",
 # endif
     "ku", "kd", "kr", "kl",
     "#2", "#4", "%i", "*7",
@@ -5854,6 +5860,7 @@ handle_dcs(char_u *tp, char_u *argp, int len, char_u *key_name, int *slen)
 {
     int i, j;
 
+    LOG_TR(("Received DCS response: %s", (char*)tp));
     j = 1 + (tp[0] == ESC);
     if (len < j + 3)
 	i = len; // need more chars
@@ -7100,7 +7107,7 @@ req_codes_from_term(void)
     static void
 req_more_codes_from_term(void)
 {
-    char	buf[23];  // extra size to shut up LGTM
+    char	buf[32];  // extra size to shut up LGTM
     int		old_idx = xt_index_out;
 
     // Don't do anything when going to exit.
@@ -7115,7 +7122,10 @@ req_more_codes_from_term(void)
 
 	MAY_WANT_TO_LOG_THIS;
 	LOG_TR(("Requesting XT %d: %s", xt_index_out, key_name));
-	sprintf(buf, "\033P+q%02x%02x\033\\", key_name[0], key_name[1]);
+	if (key_name[2] != NUL)
+	    sprintf(buf, "\033P+q%02x%02x%02x\033\\", key_name[0], key_name[1], key_name[2]);
+	else
+	    sprintf(buf, "\033P+q%02x%02x\033\\", key_name[0], key_name[1]);
 	out_str_nf((char_u *)buf);
 	++xt_index_out;
     }
@@ -7135,21 +7145,24 @@ req_more_codes_from_term(void)
 got_code_from_term(char_u *code, int len)
 {
 #define XT_LEN 100
-    char_u	name[3];
+    char_u	name[4];
     char_u	str[XT_LEN];
     int		i;
     int		j = 0;
     int		c;
 
     // A '1' means the code is supported, a '0' means it isn't.
     // When half the length is > XT_LEN we can't use it.
-    // Our names are currently all 2 characters.
-    if (code[0] == '1' && code[7] == '=' && len / 2 < XT_LEN)
+    if (code[0] == '1' && (code[7] || code[9] == '=') && len / 2 < XT_LEN)
     {
 	// Get the name from the response and find it in the table.
 	name[0] = hexhex2nr(code + 3);
 	name[1] = hexhex2nr(code + 5);
-	name[2] = NUL;
+	if (code[9] == '=')
+	    name[2] = hexhex2nr(code + 7);
+	else
+	    name[2] = NUL;
+	name[3] = NUL;
 	for (i = 0; key_names[i] != NULL; ++i)
 	{
 	    if (STRCMP(key_names[i], name) == 0)
@@ -7163,7 +7176,8 @@ got_code_from_term(char_u *code, int len)
 
 	if (key_names[i] != NULL)
 	{
-	    for (i = 8; (c = hexhex2nr(code + i)) >= 0; i += 2)
+	    i = (code[7] == '=') ? 8 : 10;
+	    for (; (c = hexhex2nr(code + i)) >= 0; i += 2)
 		str[j++] = c;
 	    str[j] = NUL;
 	    if (name[0] == 'C' && name[1] == 'o')
@@ -7180,6 +7194,22 @@ got_code_from_term(char_u *code, int len)
 #endif
 		may_adjust_color_count(val);
 	    }
+#ifdef FEAT_TERMGUICOLORS
+	    // when RGB result comes back, it is supported when the result contains an '='
+	    else if (name[0] == 'R' && name[1] == 'G' && name[2] == 'B' && code[9] == '=')
+	    {
+		int val = atoi((char *)str);
+		// 8 bits per color channel
+		if (val == 8)
+		{
+#ifdef FEAT_EVAL
+		    ch_log(NULL, "got_code_from_term(RGB): xterm-direct colors detected");
+#endif
+		    // RGB capability set, enable termguicolors
+		    set_option_value((char_u *)"termguicolors", 1L, NULL, 0);
+		}
+	    }
+#endif
 	    else
 	    {
 		i = find_term_bykeys(str);

src/testdir/test_termcodes.vim

@@ -2739,4 +2739,40 @@ func Test_terminal_builtin_without_gui()
   call assert_notequal(-1, index(output, 'builtin_dumb'))
 endfunc
 
+func Test_xterm_direct_enables_termguicolors()
+  " TERM=xterm-direct enables termguicolors
+  CheckNotMSWindows
+  if !CanRunVimInTerminal()
+    throw 'Skipped: cannot make screendumps'
+  endif
+  if !executable('tput')
+    throw "Skipped: tput not executable!"
+  endif
+  if has("gui_running")
+    throw "Skipped: does not work in GUI mode"
+  endif
+  call system('tput -Txterm-direct RGB 2>/dev/null')
+  if v:shell_error
+    throw "Skipped: xterm-direct $TERM has no RGB capability"
+  endif
+  let colors  = systemlist('tput -Txterm-direct colors')[0]
+  defer delete('XTerm-direct.txt')
+
+  let buf = RunVimInTerminal('--cmd ":set noswapfile" --clean XTerm-direct.txt',
+        \  {'rows': 10, 'env': {'TERM': 'xterm-direct'}})
+  call TermWait(buf)
+  call term_sendkeys(buf, ":$put ='TERM: ' .. &term\<cr>")
+  " doesn't work. Vim cannot query xterm colors in the embedded terminal?
+  "call term_sendkeys(buf, ":$put ='Colors: ' .. &t_Co\<cr>")
+  call term_sendkeys(buf, ":$put ='Termguicolors: ' .. &tgc\<cr>")
+  call term_sendkeys(buf, ":wq\<cr>")
+  call TermWait(buf)
+
+  let result=readfile('XTerm-direct.txt')
+  " call assert_equal(['', 'TERM: xterm-direct', 'Colors: ' .. colors, 'Termguicolors: 1'], result)
+  call assert_equal(['', 'TERM: xterm-direct', 'Termguicolors: 1'], result)
+  " cleanup
+  bw!
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab

src/version.c

@@ -704,6 +704,8 @@ static char *(features[]) =
 
 static int included_patches[] =
 {   /* Add new patch number below this line */
+/**/
+    1054,
 /**/
     1053,
 /**/