fixup! tests: optionally write results as JUnit-style .xml

Make sure to write the .xml in UTF-8 encoding.

We also need to make sure that invalid UTF-8 encoding is turned into
valid UTF-8 (using the Replacement Character, \uFFFD) because t9902's
trace contains such invalid byte sequences, and the task that uploads
the test results would refuse to do anything if it was asked to parse an
.xml file with invalid UTF-8 in it.

Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>

Author: dscho

Makefile

@@ -757,6 +757,7 @@ TEST_BUILTINS_OBJS += test-submodule-config.o
 TEST_BUILTINS_OBJS += test-submodule-nested-repo-config.o
 TEST_BUILTINS_OBJS += test-subprocess.o
 TEST_BUILTINS_OBJS += test-urlmatch-normalization.o
+TEST_BUILTINS_OBJS += test-xml-encode.o
 TEST_BUILTINS_OBJS += test-wildmatch.o
 TEST_BUILTINS_OBJS += test-windows-named-pipe.o
 TEST_BUILTINS_OBJS += test-write-cache.o

t/helper/test-tool.c

@@ -51,6 +51,7 @@ static struct test_cmd cmds[] = {
 	{ "submodule-nested-repo-config", cmd__submodule_nested_repo_config },
 	{ "subprocess", cmd__subprocess },
 	{ "urlmatch-normalization", cmd__urlmatch_normalization },
+	{ "xml-encode", cmd__xml_encode },
 	{ "wildmatch", cmd__wildmatch },
 #ifdef GIT_WINDOWS_NATIVE
 	{ "windows-named-pipe", cmd__windows_named_pipe },

t/helper/test-tool.h

@@ -47,6 +47,7 @@ int cmd__submodule_config(int argc, const char **argv);
 int cmd__submodule_nested_repo_config(int argc, const char **argv);
 int cmd__subprocess(int argc, const char **argv);
 int cmd__urlmatch_normalization(int argc, const char **argv);
+int cmd__xml_encode(int argc, const char **argv);
 int cmd__wildmatch(int argc, const char **argv);
 #ifdef GIT_WINDOWS_NATIVE
 int cmd__windows_named_pipe(int argc, const char **argv);

t/helper/test-xml-encode.c

@@ -0,0 +1,80 @@
+#include "test-tool.h"
+
+static const char *utf8_replace_character = "�";
+
+/*
+ * Encodes (possibly incorrect) UTF-8 on <stdin> to <stdout>, to be embedded
+ * in an XML file.
+ */
+int cmd__xml_encode(int argc, const char **argv)
+{
+	unsigned char buf[1024], tmp[4], *tmp2 = NULL;
+	ssize_t cur = 0, len = 1, remaining = 0;
+	unsigned char ch;
+
+	for (;;) {
+		if (++cur == len) {
+			len = xread(0, buf, sizeof(buf));
+			if (!len)
+				return 0;
+			if (len < 0)
+				die_errno("Could not read <stdin>");
+			cur = 0;
+		}
+		ch = buf[cur];
+
+		if (tmp2) {
+			if ((ch & 0xc0) != 0x80) {
+				fputs(utf8_replace_character, stdout);
+				tmp2 = 0;
+				cur--;
+				continue;
+			}
+			*tmp2 = ch;
+			tmp2++;
+			if (--remaining == 0) {
+				fwrite(tmp, tmp2 - tmp, 1, stdout);
+				tmp2 = 0;
+			}
+			continue;
+		}
+
+		if (!(ch & 0x80)) {
+			/* 0xxxxxxx */
+			if (ch == '&')
+				fputs("&amp;", stdout);
+			else if (ch == '\'')
+				fputs("&apos;", stdout);
+			else if (ch == '"')
+				fputs("&quot;", stdout);
+			else if (ch == '<')
+				fputs("&lt;", stdout);
+			else if (ch == '>')
+				fputs("&gt;", stdout);
+			else if (ch >= 0x20)
+				fputc(ch, stdout);
+			else if (ch == 0x09 || ch == 0x0a || ch == 0x0d)
+				fprintf(stdout, "&#x%02x;", ch);
+			else
+				fputs(utf8_replace_character, stdout);
+		} else if ((ch & 0xe0) == 0xc0) {
+			/* 110XXXXx 10xxxxxx */
+			tmp[0] = ch;
+			remaining = 1;
+			tmp2 = tmp + 1;
+		} else if ((ch & 0xf0) == 0xe0) {
+			/* 1110XXXX 10Xxxxxx 10xxxxxx */
+			tmp[0] = ch;
+			remaining = 2;
+			tmp2 = tmp + 1;
+		} else if ((ch & 0xf8) == 0xf0) {
+			/* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
+			tmp[0] = ch;
+			remaining = 3;
+			tmp2 = tmp + 1;
+		} else
+			fputs(utf8_replace_character, stdout);
+	}
+
+	return 0;
+}

t/test-lib.sh

@@ -888,18 +888,7 @@ write_junit_xml () {
 }
 
 xml_attr_encode () {
-	# We do not translate CR to 
 because BSD sed does not handle
-	# \r in the regex. In practice, the output should not even have any
-	# carriage returns.
-	printf '%s\n' "$@" |
-	sed -e 's/&/\&/g' -e "s/'/\'/g" -e 's/"/\"/g' \
-		-e 's/</\&lt;/g' -e 's/>/\&gt;/g' \
-		-e "s/$(printf \\x1c)/\\&#xfffd;/g" \
-		-e "s/$(printf \\x1d)/\\&#xfffd;/g" \
-		-e "s/$(printf \\x1e)/\\&#xfffd;/g" \
-		-e "s/$(printf \\x1f)/\\&#xfffd;/g" \
-		-e 's/	/\&#x09;/g' -e 's/$/\&#x0a;/' -e '$s/&#x0a;$//' |
-	tr -d '\012\015'
+	printf '%s\n' "$@" | test-tool xml-encode
 }
 
 write_junit_xml_testcase () {