detect csv delimiter in csv rendering

fixes go-gitea#7868
noerw · Aug 14, 2019 · 28e5897 · 28e5897
1 parent 14230ca
commit 28e5897
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 5 deletions.
diff --git a/modules/markup/csv/csv.go b/modules/markup/csv/csv.go
@@ -9,6 +9,8 @@ import (
 	"encoding/csv"
 	"html"
 	"io"
+	"math"
+	"strings"
 
 	"code.gitea.io/gitea/modules/markup"
 )
@@ -28,12 +30,13 @@ func (Parser) Name() string {
 
 // Extensions implements markup.Parser
 func (Parser) Extensions() []string {
-	return []string{".csv"}
+	return []string{".csv", ".tsv"}
 }
 
 // Render implements markup.Parser
-func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
+func (p Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string, isWiki bool) []byte {
 	rd := csv.NewReader(bytes.NewReader(rawBytes))
+	rd.Comma = p.bestDelimiter(rawBytes)
 	var tmpBlock bytes.Buffer
 	tmpBlock.WriteString(`<table class="table">`)
 	for {
@@ -56,3 +59,52 @@ func (Parser) Render(rawBytes []byte, urlPrefix string, metas map[string]string,
 
 	return tmpBlock.Bytes()
 }
+
+func (p Parser) bestDelimiter(data []byte) rune {
+	// Scores the input data against delimiters, and returns the best matching.
+	// Reads at most 10k bytes & 10 lines.
+	maxLines := 10
+	maxBytes := int(math.Min(float64(len(data)), 1e4))
+	text := string(data[:maxBytes])
+	lines := strings.SplitN(text, "\n", maxLines+1)[:maxLines]
+
+	delimiters := []rune{',', ';', '\t', '|'}
+	bestDelim := delimiters[0]
+	bestScore := 0.0
+
+	for _, delim := range delimiters {
+		score := p.scoreDelimiter(lines, delim)
+		if score > bestScore {
+			bestScore = score
+			bestDelim = delim
+		}
+	}
+
+	return bestDelim
+}
+
+func (Parser) scoreDelimiter(lines []string, delim rune) (score float64) {
+	// Scores a delimiter against input csv data with a count and regularity metric.
+
+	countTotal := 0.0
+	countLineMax := 0.0
+	linesNotEqual := 0.0
+
+	for _, line := range lines {
+		if len(line) == 0 {
+			continue
+		}
+
+		countLine := float64(strings.Count(line, string(delim)))
+		countTotal += countLine
+
+		if countLine != countLineMax {
+			if countLineMax != 0 {
+				linesNotEqual += 1
+			}
+			countLineMax = math.Max(countLine, countLineMax)
+		}
+	}
+
+	return countTotal * (1 - linesNotEqual/float64(len(lines)))
+}
diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go
@@ -13,9 +13,13 @@ import (
 func TestRenderCSV(t *testing.T) {
 	var parser Parser
 	var kases = map[string]string{
-		"a":     "<table class=\"table\"><tr><td>a</td><tr></table>",
-		"1,2":   "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
-		"<br/>": "<table class=\"table\"><tr><td>&lt;br/&gt;</td><tr></table>",
+		"a":                        "<table class=\"table\"><tr><td>a</td><tr></table>",
+		"1,2":                      "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
+		"1;2":                      "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
+		"1\t2":                     "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
+		"1|2":                      "<table class=\"table\"><tr><td>1</td><td>2</td><tr></table>",
+		"1,2,3;4,5,6;7,8,9\na;b;c": "<table class=\"table\"><tr><td>1,2,3</td><td>4,5,6</td><td>7,8,9</td><tr><tr><td>a</td><td>b</td<td>c</td><tr></table>",
+		"<br/>":                    "<table class=\"table\"><tr><td>&lt;br/&gt;</td><tr></table>",
 	}
 
 	for k, v := range kases {