refactor: move to JSON-formatted tests

internal/lint/code/comments.go

@@ -3,21 +3,19 @@ package code
 import (
 	"context"
 	"fmt"
+	"regexp"
 	"strings"
 
 	sitter "github.com/smacker/go-tree-sitter"
-	"github.com/smacker/go-tree-sitter/golang"
-	"github.com/smacker/go-tree-sitter/python"
-	"github.com/smacker/go-tree-sitter/rust"
 )
 
 // Language represents a supported programming language.
 //
 // NOTE: What about haskell, less, perl, php, powershell, r, sass, swift?
 type Language struct {
-	Delimiters []string
-	Sitter     *sitter.Language
-	Query      string
+	Delims *regexp.Regexp
+	Parser *sitter.Language
+	Query  string
 }
 
 // Comment represents an in-code comment (line or block).
@@ -31,23 +29,11 @@ type Comment struct {
 func getLanguageFromExt(ext string) (*Language, error) {
 	switch ext {
 	case ".go":
-		return &Language{
-			Delimiters: []string{"//", "/*", "*/"},
-			Sitter:     golang.GetLanguage(),
-			Query:      "(comment)+ @comment",
-		}, nil
+		return Go(), nil
 	case ".rs":
-		return &Language{
-			Delimiters: []string{"///", "//"},
-			Sitter:     rust.GetLanguage(),
-			Query:      "(line_comment)+ @comment",
-		}, nil
+		return Rust(), nil
 	case ".py":
-		return &Language{
-			Delimiters: []string{"#"},
-			Sitter:     python.GetLanguage(),
-			Query:      `(comment) @comment`,
-		}, nil
+		return Python(), nil
 	default:
 		return nil, fmt.Errorf("unsupported extension: '%s'", ext)
 	}
@@ -57,15 +43,15 @@ func getComments(source []byte, lang *Language) ([]Comment, error) {
 	var comments []Comment
 
 	parser := sitter.NewParser()
-	parser.SetLanguage(lang.Sitter)
+	parser.SetLanguage(lang.Parser)
 
 	tree, err := parser.ParseCtx(context.Background(), nil, source)
 	if err != nil {
 		return comments, err
 	}
 	n := tree.RootNode()
 
-	q, err := sitter.NewQuery([]byte(lang.Query), lang.Sitter)
+	q, err := sitter.NewQuery([]byte(lang.Query), lang.Parser)
 	if err != nil {
 		return comments, err
 	}
@@ -79,12 +65,9 @@ func getComments(source []byte, lang *Language) ([]Comment, error) {
 			break
 		}
 
+		m = qc.FilterPredicates(m, source)
 		for _, c := range m.Captures {
-			text := c.Node.Content(source)
-
-			for _, d := range lang.Delimiters {
-				text = strings.Trim(text, d)
-			}
+			text := lang.Delims.ReplaceAllString(c.Node.Content(source), "")
 
 			scope := "text.comment.line"
 			if strings.Count(text, "\n") > 1 {

internal/lint/code/comments_test.go

@@ -1,23 +1,20 @@
 package code
 
 import (
-	"encoding/json"
 	"fmt"
 	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
 )
 
-func toJSON(comments []Comment) string {
-	j, _ := json.MarshalIndent(comments, "", "    ")
-	return string(j)
-}
+var testDir = "../../../testdata/comments"
+var binDir = "../../../bin"
 
 func TestComments(t *testing.T) {
 	var cleaned []fs.DirEntry
 
-	cases, err := os.ReadDir("../../testdata/comments/in")
+	cases, err := os.ReadDir(testDir + "/in")
 	if err != nil {
 		t.Error(err)
 	}
@@ -30,30 +27,29 @@ func TestComments(t *testing.T) {
 	}
 
 	for i, f := range cleaned {
-		b, err1 := os.ReadFile(fmt.Sprintf("../../testdata/comments/in/%s", f.Name()))
+		b, err1 := os.ReadFile(fmt.Sprintf("%s/in/%s", testDir, f.Name()))
 		if err1 != nil {
 			t.Error(err1)
 		}
 
 		lang, err2 := getLanguageFromExt(filepath.Ext(f.Name()))
 		if err2 != nil {
-			t.Fatal(err2)
+			t.Error(err2)
 		}
 
 		comments, err3 := getComments(b, lang)
 		if err3 != nil {
-			t.Fatal(err3)
+			t.Error(err3)
 		}
-		comments = coalesce(comments)
 
-		b2, err4 := os.ReadFile(fmt.Sprintf("../../testdata/comments/out/%d.json", i))
+		b2, err4 := os.ReadFile(fmt.Sprintf("%s/out/%d.json", testDir, i))
 		if err4 != nil {
-			t.Fatal(err4)
+			t.Error(err4)
 		}
 
 		markup := toJSON(comments)
 		if markup != string(b2) {
-			bin := filepath.Join("..", "..", "bin", fmt.Sprintf("%d.json", i))
+			bin := filepath.Join(binDir, fmt.Sprintf("%d.json", i))
 			_ = os.WriteFile(bin, []byte(markup), os.ModePerm)
 			t.Errorf("%s", markup)
 		}

internal/lint/code/go.go

@@ -0,0 +1,15 @@
+package code
+
+import (
+	"regexp"
+
+	"github.com/smacker/go-tree-sitter/golang"
+)
+
+func Go() *Language {
+	return &Language{
+		Delims: regexp.MustCompile(`//\s?|/\*\s?|\s?\*/`),
+		Parser: golang.GetLanguage(),
+		Query:  `(comment)+ @comment`,
+	}
+}

internal/lint/code/py.go

@@ -0,0 +1,15 @@
+package code
+
+import (
+	"regexp"
+
+	"github.com/smacker/go-tree-sitter/python"
+)
+
+func Python() *Language {
+	return &Language{
+		Delims: regexp.MustCompile(`#\s?`),
+		Parser: python.GetLanguage(),
+		Query:  `(comment)+ @comment`,
+	}
+}

internal/lint/code/rs.go

@@ -0,0 +1,15 @@
+package code
+
+import (
+	"regexp"
+
+	"github.com/smacker/go-tree-sitter/rust"
+)
+
+func Rust() *Language {
+	return &Language{
+		Delims: regexp.MustCompile(`/{2,3}\s?`),
+		Parser: rust.GetLanguage(),
+		Query:  `(line_comment)+ @comment`,
+	}
+}

internal/lint/code/util.go

@@ -0,0 +1,8 @@
+package code
+
+import "encoding/json"
+
+func toJSON(comments []Comment) string {
+	j, _ := json.MarshalIndent(comments, "", "    ")
+	return string(j)
+}

internal/lint/fragment.go

@@ -1,58 +1,12 @@
 package lint
 
 import (
-	"bytes"
 	"fmt"
-	"strings"
 
 	"github.com/errata-ai/vale/v3/internal/core"
 	"github.com/errata-ai/vale/v3/internal/lint/code"
 )
 
-func shouldMerge(curr, prev code.Comment) bool {
-	return true
-}
-
-func coalesce(comments []code.Comment) []code.Comment {
-	var joined []code.Comment
-
-	buf := bytes.Buffer{}
-	for i, comment := range comments {
-		offset := comment.Offset
-		if i > 0 {
-			offset += comments[i-1].Offset
-		}
-
-		if comment.Scope == "text.comment.block" { //nolint:gocritic
-			joined = append(joined, comment)
-		} else if i == 0 || shouldMerge(comment, comments[i-1]) {
-			if buf.Len() > 0 {
-				// We have comments to merge ...
-				last := joined[len(joined)-1]
-				last.Text += buf.String()
-				joined[len(joined)-1] = last
-				buf.Reset()
-			}
-			joined = append(joined, comment)
-		} else {
-			buf.WriteString(comment.Text)
-		}
-	}
-
-	if buf.Len() > 0 {
-		last := joined[len(joined)-1]
-		last.Text += buf.String()
-		joined[len(joined)-1] = last
-		buf.Reset()
-	}
-
-	for i, comment := range joined {
-		joined[i].Text = strings.TrimLeft(comment.Text, " ")
-	}
-
-	return joined
-}
-
 func adjustAlerts(last int, ctx code.Comment, alerts []core.Alert) []core.Alert {
 	for i := range alerts {
 		if i >= last {

testdata/comments/in/2.py

@@ -1,5 +1,6 @@
 # FIXME:
 
+
 def FIXME():
     """
     FIXME:
@@ -8,10 +9,11 @@ def FIXME():
 FIXME: This should *not* be linted.
 """)
 
+
 # XXX: This should be flagged!
 
-NOTE = False # XXX:
-XXX = True # NOTE:
+NOTE = False  # XXX:
+XXX = True  # NOTE:
 
 r"""
 NOTE:
@@ -23,18 +25,20 @@ def FIXME():
 XXX:
 """
 
+
 def NOTE():
-    '''
+    """
     NOTE:
-    '''
-    XXX = '''
+    """
+    XXX = """
 XXX: This should *not* be linted.
-'''
+"""
+
 
 def foo(self):
     """NOTE This is the start of a block.
 
     TODO: Assume that a file is modified since an invalid timestamp as per RFC
     2616, section 14.25. GMT
     """
-    invalid_date = 'FIXME: Mon, 28 May 999999999999 28:25:26 GMT'
+    invalid_date = "FIXME: Mon, 28 May 999999999999 28:25:26 GMT"

testdata/comments/out/0.json

@@ -1,16 +1,46 @@
 [
     {
-        "Text": "\nPackage lint implments Vale's syntax-aware linting functionality.\n\nThe package is split into core linting logic (this file), source code\n(code.go), and markup (markup.go).\n",
+        "Text": "Package lint implments Vale's syntax-aware linting functionality.\n\nThe package is split into core linting logic (this file), source code\n(code.go), and markup (markup.go).",
         "Line": 1,
         "Offset": 0,
         "Scope": "text.comment.block"
     },
     {
-        "Text": "Println formats using the default formats for its oprands and writes to standard output. Spaces are always added between operands and a newline is appended. It returns the number of bytes written and any write error encountered.",
+        "Text": "Println formats using the default formats for its oprands and writes to",
         "Line": 11,
         "Offset": 0,
         "Scope": "text.comment.line"
     },
+    {
+        "Text": "standard output.",
+        "Line": 12,
+        "Offset": 0,
+        "Scope": "text.comment.line"
+    },
+    {
+        "Text": "",
+        "Line": 13,
+        "Offset": 0,
+        "Scope": "text.comment.line"
+    },
+    {
+        "Text": "Spaces are always added between operands and a newline is appended.",
+        "Line": 14,
+        "Offset": 0,
+        "Scope": "text.comment.line"
+    },
+    {
+        "Text": "",
+        "Line": 15,
+        "Offset": 0,
+        "Scope": "text.comment.line"
+    },
+    {
+        "Text": "It returns the number of bytes written and any write error encountered.",
+        "Line": 16,
+        "Offset": 0,
+        "Scope": "text.comment.line"
+    },
     {
         "Text": "foo bar",
         "Line": 21,