Ensure all analyzers have README info

Magefile.go

@@ -316,6 +316,7 @@ func (Run) SourceDiffLocal(ctx context.Context, archive string, source string) e
 
 }
 
+// Readme re-generates the "Analyzers" table in README.md.
 func (Gen) Readme() error {
 	return sh.RunV("go", "test", "-v", "./pkg/genreadme", "-generate")
 }

README.md

@@ -239,25 +239,31 @@ Run "mage gen:readme" to regenerate this section.
 | Legacy Grafana Toolkit usage / `legacybuilder` | Detects the usage of the not longer supported Grafana Toolkit. | None |
 | Legacy Platform / `legacyplatform` | Detects use of Angular which is deprecated. | None |
 | License Type / `license` | Checks the declared license is one of: BSD, MIT, Apache 2.0, LGPL3, GPL3, AGPL3. | None |
+| LLM Review / `llmreview` | Runs the code through Gemini LLM to check for security issues or disallowed usage. | Gemini API key |
 | Logos / `logos` | Detects whether the plugin includes small and large logos to display in the plugin catalog. | None |
 | Manifest (Signing) / `manifest` | When a plugin is signed, the zip file will contain a signed `MANIFEST.txt` file. | None |
+| Metadata / `metadata` | Checks that `plugin.json` exists and is valid. | None |
 | Metadata Paths / `metadatapaths` | Ensures all paths are valid and images referenced exist. | None |
 | Metadata Validity / `metadatavalid` | Ensures metadata is valid and matches plugin schema. | None |
 | module.js (exists) / `modulejs` | All plugins require a `module.js` to be loaded. | None |
 | Nested includes metadata / `includesnested` | Validates that nested plugins have the correct metadata. | None |
+| Nested Metadata / `nestedmetadata` | Recursively checks that all `plugin.json` exist and are valid. | None |
 | No Tracking Scripts / `trackingscripts` | Detects if there are any known tracking scripts, which are not allowed. | None |
 | Organization (exists) / `org` | Verifies the org specified in the plugin ID exists. | None |
+| package.json / `packagejson` | Ensures that package.json exists and the version matches the plugin.json | None |
 | Plugin Name formatting / `pluginname` | Validates the plugin ID used conforms to our naming convention. | None |
 | Published / `published-plugin` | Detects whether any version of this plugin exists in the Grafana plugin catalog currently. | None |
 | Readme (exists) / `readme` | Ensures a `README.md` file exists within the zip file. | None |
 | Restrictive Dependency / `restrictivedep` | Specifies a valid range of Grafana versions that work with this version of the plugin. | None |
 | Screenshots / `screenshots` | Screenshots are specified in `plugin.json` that will be used in the Grafana plugin catalog. | None |
+| SDK Usage / `sdkusage` | Ensures that `grafana-plugin-sdk-go` is up-to-date. | None |
 | Signature / `signature` | Ensures the plugin has a valid signature. | None |
 | Source Code / `sourcecode` | A comparison is made between the zip file and the source code to ensure what is released matches the repo associated with it. | `sourceCodeUri` |
 | Type Suffix (panel/app/datasource) / `typesuffix` | Ensures the plugin has a valid type specified. | None |
 | Unique README.md / `templatereadme` | Ensures the plugin doesn't re-use the template from the `create-plugin` tool. | None |
 | Unsafe SVG / `manifest` | Checks if any svg files are safe based on a whitelist of elements and attributes. | None |
 | Version / `version` | Ensures the version submitted is newer than the currently published plugin. If this is a new/unpublished plugin, this is skipped. | None |
+| Virus Scan / `virusscan` | Runs a virus scan on the plugin archive and source code using `clamscan` (`clamav`). | clamscan |
 | Vulnerability Scanner / `osv-scanner` | Detects critical vulnerabilities in Go modules and yarn lock files. | [osv-scanner](https://github.com/google/osv-scanner), `sourceCodeUri` |
 <!-- analyzers-table-end -->
 

pkg/analysis/passes/llmreview/llmreview.go

@@ -15,15 +15,19 @@ import (
 var geminiKey = os.Getenv("GEMINI_API_KEY")
 
 var (
-	llminiterror  = &analysis.Rule{Name: "llminit-error", Severity: analysis.Warning}
 	llmIssueFound = &analysis.Rule{Name: "llm-issue-found", Severity: analysis.SuspectedProblem}
 )
 
 var Analyzer = &analysis.Analyzer{
 	Name:     "llmreview",
 	Requires: []*analysis.Analyzer{sourcecode.Analyzer},
 	Run:      run,
-	Rules:    []*analysis.Rule{llminiterror, llmIssueFound},
+	Rules:    []*analysis.Rule{llmIssueFound},
+	ReadmeInfo: analysis.ReadmeInfo{
+		Name:         "LLM Review",
+		Description:  "Runs the code through Gemini LLM to check for security issues or disallowed usage.",
+		Dependencies: "Gemini API key",
+	},
 }
 
 var questions = []string{

pkg/analysis/passes/metadata/metadata.go

@@ -17,6 +17,10 @@ var Analyzer = &analysis.Analyzer{
 	Requires: []*analysis.Analyzer{archive.Analyzer},
 	Run:      run,
 	Rules:    []*analysis.Rule{missingMetadata},
+	ReadmeInfo: analysis.ReadmeInfo{
+		Name:        "Metadata",
+		Description: "Checks that `plugin.json` exists and is valid.",
+	},
 }
 
 func run(pass *analysis.Pass) (interface{}, error) {

pkg/analysis/passes/nestedmetadata/nestedmetadata.go

@@ -23,6 +23,10 @@ var Analyzer = &analysis.Analyzer{
 	Requires: []*analysis.Analyzer{archive.Analyzer},
 	Run:      run,
 	Rules:    []*analysis.Rule{missingMetadata, errorReadingMetadata},
+	ReadmeInfo: analysis.ReadmeInfo{
+		Name:        "Nested Metadata",
+		Description: "Recursively checks that all `plugin.json` exist and are valid.",
+	},
 }
 
 var MainPluginJson = "plugin.json"

pkg/analysis/passes/packagejson/packagejson.go

@@ -23,6 +23,10 @@ var Analyzer = &analysis.Analyzer{
 	Requires: []*analysis.Analyzer{metadata.Analyzer, sourcecode.Analyzer},
 	Run:      run,
 	Rules:    []*analysis.Rule{packagejsonNotFound},
+	ReadmeInfo: analysis.ReadmeInfo{
+		Name:        "package.json",
+		Description: "Ensures that package.json exists and the version matches the plugin.json",
+	},
 }
 
 func run(pass *analysis.Pass) (interface{}, error) {

pkg/analysis/passes/sdkusage/sdkusage.go

@@ -42,6 +42,10 @@ var Analyzer = &analysis.Analyzer{
 		goSdkOlderThanTwoMonths,
 		goSdkOlderThanFiveMonths,
 	},
+	ReadmeInfo: analysis.ReadmeInfo{
+		Name:        "SDK Usage",
+		Description: "Ensures that `grafana-plugin-sdk-go` is up-to-date.",
+	},
 }
 
 func run(pass *analysis.Pass) (interface{}, error) {

pkg/analysis/passes/virusscan/virusscan.go

@@ -42,6 +42,11 @@ var Analyzer = &analysis.Analyzer{
 	Rules: []*analysis.Rule{
 		virusScanFailed,
 	},
+	ReadmeInfo: analysis.ReadmeInfo{
+		Name:         "Virus Scan",
+		Description:  "Runs a virus scan on the plugin archive and source code using `clamscan` (`clamav`).",
+		Dependencies: "clamscan",
+	},
 }
 
 func run(pass *analysis.Pass) (interface{}, error) {

pkg/genreadme/genreadme.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/grafana/plugin-validator/pkg/analysis"
 	"github.com/grafana/plugin-validator/pkg/analysis/passes"
-	"github.com/grafana/plugin-validator/pkg/logme"
 )
 
 const (
@@ -25,6 +24,9 @@ Run "mage gen:readme" to regenerate this section.
 `
 )
 
+// generate returns the new README content with the analyzers table updated.
+// The table's content is generated from `passes.Analyzers`.
+// The README must contain the magic tags, otherwise an error is returned.
 func generate(readme io.Reader) (string, error) {
 	var tableBuilder strings.Builder
 	tableBuilder.WriteString(header)
@@ -37,7 +39,6 @@ func generate(readme io.Reader) (string, error) {
 	// Generate table content
 	for _, analyzer := range passes.Analyzers {
 		if analyzer.ReadmeInfo.Name == "" && analyzer.ReadmeInfo.Description == "" {
-			logme.ErrorF("Warning: Analyzer %q does not have README data.\n", analyzer.Name)
 			continue
 		}
 		dependencies := analyzer.ReadmeInfo.Dependencies

pkg/genreadme/genreadme_test.go

@@ -9,7 +9,10 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/plugin-validator/pkg/analysis/passes"
 )
 
 const readmeFileName = "README.md"
@@ -56,6 +59,16 @@ func TestGenReadme(t *testing.T) {
 			"README.md is not up-to-date. Run `mage gen:readme` to update it.",
 		)
 	})
+
+	t.Run("all analyzers must have readme data", func(t *testing.T) {
+		for _, analyzer := range passes.Analyzers {
+			assert.Falsef(
+				t,
+				analyzer.ReadmeInfo.Name == "" && analyzer.ReadmeInfo.Description == "",
+				"analyzer %q does not have README data", analyzer.Name,
+			)
+		}
+	})
 }
 
 func generateToFile(fn string) error {