Normalize cataloger configuration patterns (#2365)

* normalize cataloger patterns Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> * remove central reference for maven configurable Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com> --------- Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
anchore · Nov 28, 2023 · 1cfc4c7 · 1cfc4c7
1 parent 4d0da70
commit 1cfc4c7
Show file tree

Hide file tree

Showing 49 changed files with 335 additions and 288 deletions.
diff --git a/cmd/syft/cli/options/catalog.go b/cmd/syft/cli/options/catalog.go
@@ -12,6 +12,7 @@ import (
 	"github.com/anchore/clio"
 	"github.com/anchore/fangs"
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/pkg/cataloger"
 	golangCataloger "github.com/anchore/syft/syft/pkg/cataloger/golang"
 	javaCataloger "github.com/anchore/syft/syft/pkg/cataloger/java"
@@ -126,19 +127,24 @@ func (cfg Catalog) ToCatalogerConfig() cataloger.Config {
 		},
 		Catalogers:  cfg.Catalogers,
 		Parallelism: cfg.Parallelism,
-		Golang: golangCataloger.NewGoCatalogerOpts().
+		Golang: golangCataloger.DefaultCatalogerConfig().
 			WithSearchLocalModCacheLicenses(cfg.Golang.SearchLocalModCacheLicenses).
 			WithLocalModCacheDir(cfg.Golang.LocalModCacheDir).
 			WithSearchRemoteLicenses(cfg.Golang.SearchRemoteLicenses).
 			WithProxy(cfg.Golang.Proxy).
 			WithNoProxy(cfg.Golang.NoProxy),
-		LinuxKernel: kernel.LinuxCatalogerConfig{
+		LinuxKernel: kernel.LinuxKernelCatalogerConfig{
 			CatalogModules: cfg.LinuxKernel.CatalogModules,
 		},
-		Java: javaCataloger.DefaultCatalogerOpts().
+		Java: javaCataloger.DefaultArchiveCatalogerConfig().
 			WithUseNetwork(cfg.Java.UseNetwork).
-			WithMavenURL(cfg.Java.MavenURL).
-			WithMaxParentRecursiveDepth(cfg.Java.MaxParentRecursiveDepth),
+			WithMavenBaseURL(cfg.Java.MavenURL).
+			WithArchiveTraversal(
+				cataloging.ArchiveSearchConfig{
+					IncludeIndexedArchives:   cfg.Package.SearchIndexedArchives,
+					IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
+				},
+				cfg.Java.MaxParentRecursiveDepth),
 		Python: pythonCataloger.CatalogerConfig{
 			GuessUnpinnedRequirements: cfg.Python.GuessUnpinnedRequirements,
 		},

diff --git a/syft/cataloging/config.go b/syft/cataloging/config.go
@@ -0,0 +1,6 @@
+package cataloging
+
+type ArchiveSearchConfig struct {
+	IncludeIndexedArchives   bool `yaml:"include-indexed-archives" json:"include-indexed-archives" mapstructure:"include-indexed-archives"`
+	IncludeUnindexedArchives bool `yaml:"include-unindexed-archives" json:"include-unindexed-archives" mapstructure:"include-unindexed-archives"`
+}
diff --git a/syft/pkg/cataloger/alpine/cataloger.go b/syft/pkg/cataloger/alpine/cataloger.go
@@ -9,7 +9,7 @@ import (
 )
 
 // NewDBCataloger returns a new cataloger object initialized for Alpine package DB flat-file stores.
-func NewDBCataloger() *generic.Cataloger {
+func NewDBCataloger() pkg.Cataloger {
 	return generic.NewCataloger("apk-db-cataloger").
 		WithParserByGlobs(parseApkDB, pkg.ApkDBGlob)
 }
diff --git a/syft/pkg/cataloger/arch/cataloger.go b/syft/pkg/cataloger/arch/cataloger.go
@@ -9,7 +9,7 @@ import (
 )
 
 // NewDBCataloger returns a new cataloger object initialized for arch linux pacman database flat-file stores.
-func NewDBCataloger() *generic.Cataloger {
+func NewDBCataloger() pkg.Cataloger {
 	return generic.NewCataloger("alpm-db-cataloger").
 		WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob)
 }
diff --git a/syft/pkg/cataloger/binary/cataloger.go b/syft/pkg/cataloger/binary/cataloger.go
@@ -12,7 +12,7 @@ import (
 
 const catalogerName = "binary-cataloger"
 
-func NewCataloger() *Cataloger {
+func NewCataloger() pkg.Cataloger {
 	return &Cataloger{}
 }
 

diff --git a/syft/pkg/cataloger/config.go b/syft/pkg/cataloger/config.go
@@ -1,6 +1,7 @@
 package cataloger
 
 import (
+	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/pkg/cataloger/golang"
 	"github.com/anchore/syft/syft/pkg/cataloger/java"
 	"github.com/anchore/syft/syft/pkg/cataloger/kernel"
@@ -10,10 +11,10 @@ import (
 // TODO: these field naming vs helper function naming schemes are inconsistent.
 type Config struct {
 	Search                          SearchConfig
-	Golang                          golang.GoCatalogerOpts
-	LinuxKernel                     kernel.LinuxCatalogerConfig
+	Golang                          golang.CatalogerConfig
+	LinuxKernel                     kernel.LinuxKernelCatalogerConfig
 	Python                          python.CatalogerConfig
-	Java                            java.CatalogerOpts
+	Java                            java.ArchiveCatalogerConfig
 	Catalogers                      []string
 	Parallelism                     int
 	ExcludeBinaryOverlapByOwnership bool
@@ -25,20 +26,22 @@ func DefaultConfig() Config {
 		Parallelism:                     1,
 		LinuxKernel:                     kernel.DefaultLinuxCatalogerConfig(),
 		Python:                          python.DefaultCatalogerConfig(),
-		Java:                            java.DefaultCatalogerOpts(),
+		Java:                            java.DefaultArchiveCatalogerConfig(),
 		ExcludeBinaryOverlapByOwnership: true,
 	}
 }
 
 // JavaConfig merges relevant config values from Config to return a java.Config struct.
 // Values like IncludeUnindexedArchives and IncludeIndexedArchives are used across catalogers
 // and are not specific to Java requiring this merge.
-func (c Config) JavaConfig() java.Config {
-	return java.Config{
-		SearchUnindexedArchives: c.Search.IncludeUnindexedArchives,
-		SearchIndexedArchives:   c.Search.IncludeIndexedArchives,
+func (c Config) JavaConfig() java.ArchiveCatalogerConfig {
+	return java.ArchiveCatalogerConfig{
+		ArchiveSearchConfig: cataloging.ArchiveSearchConfig{
+			IncludeUnindexedArchives: c.Search.IncludeUnindexedArchives,
+			IncludeIndexedArchives:   c.Search.IncludeIndexedArchives,
+		},
 		UseNetwork:              c.Java.UseNetwork,
-		MavenBaseURL:            c.Java.MavenURL,
+		MavenBaseURL:            c.Java.MavenBaseURL,
 		MaxParentRecursiveDepth: c.Java.MaxParentRecursiveDepth,
 	}
 }
diff --git a/syft/pkg/cataloger/cpp/cataloger.go b/syft/pkg/cataloger/cpp/cataloger.go
@@ -4,18 +4,19 @@ Package cpp provides a concrete Cataloger implementations for the C/C++ language
 package cpp
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewConanCataloger returns a new C/C++ conanfile.txt and conan.lock cataloger object.
-func NewConanCataloger() *generic.Cataloger {
+func NewConanCataloger() pkg.Cataloger {
 	return generic.NewCataloger("conan-cataloger").
 		WithParserByGlobs(parseConanfile, "**/conanfile.txt").
 		WithParserByGlobs(parseConanlock, "**/conan.lock")
 }
 
 // NewConanInfoCataloger returns a new C/C++ conaninfo.txt cataloger object.
-func NewConanInfoCataloger() *generic.Cataloger {
+func NewConanInfoCataloger() pkg.Cataloger {
 	return generic.NewCataloger("conan-info-cataloger").
 		WithParserByGlobs(parseConaninfo, "**/conaninfo.txt")
 }
diff --git a/syft/pkg/cataloger/dart/cataloger.go b/syft/pkg/cataloger/dart/cataloger.go
@@ -4,11 +4,12 @@ Package dart provides a concrete Cataloger implementations for the Dart language
 package dart
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewPubspecLockCataloger returns a new Dartlang cataloger object base on pubspec lock files.
-func NewPubspecLockCataloger() *generic.Cataloger {
+func NewPubspecLockCataloger() pkg.Cataloger {
 	return generic.NewCataloger("dart-pubspec-lock-cataloger").
 		WithParserByGlobs(parsePubspecLock, "**/pubspec.lock")
 }
diff --git a/syft/pkg/cataloger/debian/cataloger.go b/syft/pkg/cataloger/debian/cataloger.go
@@ -4,11 +4,12 @@ Package debian provides a concrete Cataloger implementation relating to packages
 package debian
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewDBCataloger returns a new Deb package cataloger capable of parsing DPKG status DB flat-file stores.
-func NewDBCataloger() *generic.Cataloger {
+func NewDBCataloger() pkg.Cataloger {
 	return generic.NewCataloger("dpkg-db-cataloger").
 		// note: these globs have been intentionally split up in order to improve search performance,
 		// please do NOT combine into: "**/var/lib/dpkg/{status,status.d/*}"

diff --git a/syft/pkg/cataloger/dotnet/cataloger.go b/syft/pkg/cataloger/dotnet/cataloger.go
@@ -4,17 +4,18 @@ Package dotnet provides a concrete Cataloger implementation relating to packages
 package dotnet
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewDotnetDepsCataloger returns a new Dotnet cataloger object base on deps json files.
-func NewDotnetDepsCataloger() *generic.Cataloger {
+func NewDotnetDepsCataloger() pkg.Cataloger {
 	return generic.NewCataloger("dotnet-deps-cataloger").
 		WithParserByGlobs(parseDotnetDeps, "**/*.deps.json")
 }
 
 // NewDotnetPortableExecutableCataloger returns a new Dotnet cataloger object base on portable executable files.
-func NewDotnetPortableExecutableCataloger() *generic.Cataloger {
+func NewDotnetPortableExecutableCataloger() pkg.Cataloger {
 	return generic.NewCataloger("dotnet-portable-executable-cataloger").
 		WithParserByGlobs(parseDotnetPortableExecutable, "**/*.dll", "**/*.exe")
 }
diff --git a/syft/pkg/cataloger/dotnet/cataloger_test.go b/syft/pkg/cataloger/dotnet/cataloger_test.go
@@ -3,15 +3,15 @@ package dotnet
 import (
 	"testing"
 
-	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
 )
 
 func TestCataloger_Globs(t *testing.T) {
 	tests := []struct {
 		name      string
 		fixture   string
-		cataloger *generic.Cataloger
+		cataloger pkg.Cataloger
 		expected  []string
 	}{
 		{

diff --git a/syft/pkg/cataloger/elixir/cataloger.go b/syft/pkg/cataloger/elixir/cataloger.go
@@ -4,11 +4,12 @@ Package elixir provides a concrete Cataloger implementation relating to packages
 package elixir
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewMixLockCataloger returns a cataloger object for Elixir mix.lock files.
-func NewMixLockCataloger() *generic.Cataloger {
+func NewMixLockCataloger() pkg.Cataloger {
 	return generic.NewCataloger("elixir-mix-lock-cataloger").
 		WithParserByGlobs(parseMixLock, "**/mix.lock")
 }
diff --git a/syft/pkg/cataloger/erlang/cataloger.go b/syft/pkg/cataloger/erlang/cataloger.go
@@ -4,11 +4,12 @@ Package erlang provides a concrete Cataloger implementation relating to packages
 package erlang
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewRebarLockCataloger returns a new cataloger instance for Erlang rebar.lock files.
-func NewRebarLockCataloger() *generic.Cataloger {
+func NewRebarLockCataloger() pkg.Cataloger {
 	return generic.NewCataloger("erlang-rebar-lock-cataloger").
 		WithParserByGlobs(parseRebarLock, "**/rebar.lock")
 }
diff --git a/syft/pkg/cataloger/gentoo/cataloger.go b/syft/pkg/cataloger/gentoo/cataloger.go
@@ -4,11 +4,12 @@ Package gentoo provides a concrete Cataloger implementation related to packages
 package gentoo
 
 import (
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
 )
 
 // NewPortageCataloger returns a new cataloger object initialized for Gentoo Portage package manager files (a flat-file store).
-func NewPortageCataloger() *generic.Cataloger {
+func NewPortageCataloger() pkg.Cataloger {
 	return generic.NewCataloger("portage-cataloger").
 		WithParserByGlobs(parsePortageContents, "**/var/db/pkg/*/*/CONTENTS")
 }
diff --git a/syft/pkg/cataloger/githubactions/cataloger.go b/syft/pkg/cataloger/githubactions/cataloger.go
@@ -3,17 +3,20 @@ Package githubactions provides a concrete Cataloger implementation for GitHub Ac
 */
 package githubactions
 
-import "github.com/anchore/syft/syft/pkg/cataloger/generic"
+import (
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+)
 
 // NewActionUsageCataloger returns GitHub Actions used within workflows and composite actions.
-func NewActionUsageCataloger() *generic.Cataloger {
+func NewActionUsageCataloger() pkg.Cataloger {
 	return generic.NewCataloger("github-actions-usage-cataloger").
 		WithParserByGlobs(parseWorkflowForActionUsage, "**/.github/workflows/*.yaml", "**/.github/workflows/*.yml").
 		WithParserByGlobs(parseCompositeActionForActionUsage, "**/.github/actions/*/action.yml", "**/.github/actions/*/action.yaml")
 }
 
 // NewWorkflowUsageCataloger returns shared workflows used within workflows.
-func NewWorkflowUsageCataloger() *generic.Cataloger {
+func NewWorkflowUsageCataloger() pkg.Cataloger {
 	return generic.NewCataloger("github-action-workflow-usage-cataloger").
 		WithParserByGlobs(parseWorkflowForWorkflowUsage, "**/.github/workflows/*.yaml", "**/.github/workflows/*.yml")
 }
diff --git a/syft/pkg/cataloger/githubactions/cataloger_test.go b/syft/pkg/cataloger/githubactions/cataloger_test.go
@@ -3,15 +3,15 @@ package githubactions
 import (
 	"testing"
 
-	"github.com/anchore/syft/syft/pkg/cataloger/generic"
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
 )
 
 func TestCataloger_Globs(t *testing.T) {
 	tests := []struct {
 		name      string
 		fixture   string
-		cataloger *generic.Cataloger
+		cataloger pkg.Cataloger
 		expected  []string
 	}{
 		{

diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go
@@ -11,7 +11,6 @@ import (
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/cpe"
-	"github.com/anchore/syft/syft/event/monitor"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/pkg/cataloger/generic"
@@ -20,31 +19,28 @@ import (
 var versionCandidateGroups = regexp.MustCompile(`(?P<version>\d+(\.\d+)?(\.\d+)?)(?P<candidate>\w*)`)
 
 // NewGoModuleFileCataloger returns a new cataloger object that searches within go.mod files.
-func NewGoModuleFileCataloger(opts GoCatalogerOpts) pkg.Cataloger {
+func NewGoModuleFileCataloger(opts CatalogerConfig) pkg.Cataloger {
 	c := goModCataloger{
 		licenses: newGoLicenses(opts),
 	}
 	return &progressingCataloger{
-		progress: c.licenses.progress,
 		cataloger: generic.NewCataloger("go-module-file-cataloger").
 			WithParserByGlobs(c.parseGoModFile, "**/go.mod"),
 	}
 }
 
 // NewGoModuleBinaryCataloger returns a new cataloger object that searches within binaries built by the go compiler.
-func NewGoModuleBinaryCataloger(opts GoCatalogerOpts) pkg.Cataloger {
+func NewGoModuleBinaryCataloger(opts CatalogerConfig) pkg.Cataloger {
 	c := goBinaryCataloger{
 		licenses: newGoLicenses(opts),
 	}
 	return &progressingCataloger{
-		progress: c.licenses.progress,
 		cataloger: generic.NewCataloger("go-module-binary-cataloger").
 			WithParserByMimeTypes(c.parseGoBinary, internal.ExecutableMIMETypeSet.List()...),
 	}
 }
 
 type progressingCataloger struct {
-	progress  *monitor.CatalogerTask
 	cataloger *generic.Cataloger
 }
 
@@ -53,7 +49,6 @@ func (p *progressingCataloger) Name() string {
 }
 
 func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) {
-	defer p.progress.SetCompleted()
 	pkgs, relationships, err := p.cataloger.Catalog(resolver)
 	goCompilerPkgs := []pkg.Package{}
 	totalLocations := file.NewLocationSet()
@@ -76,6 +71,7 @@ func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, [
 	pkgs = append(pkgs, goCompilerPkgs...)
 	return pkgs, relationships, err
 }
+
 func newGoStdLib(version string, location file.LocationSet) *pkg.Package {
 	stdlibCpe, err := generateStdlibCpe(version)
 	if err != nil {

diff --git a/syft/pkg/cataloger/golang/cataloger_test.go b/syft/pkg/cataloger/golang/cataloger_test.go
@@ -30,7 +30,7 @@ func Test_Mod_Cataloger_Globs(t *testing.T) {
 				FromDirectory(t, test.fixture).
 				ExpectsResolverContentQueries(test.expected).
 				IgnoreUnfulfilledPathResponses("src/go.sum").
-				TestCataloger(t, NewGoModuleFileCataloger(GoCatalogerOpts{}))
+				TestCataloger(t, NewGoModuleFileCataloger(CatalogerConfig{}))
 		})
 	}
 }
@@ -55,7 +55,7 @@ func Test_Binary_Cataloger_Globs(t *testing.T) {
 			pkgtest.NewCatalogTester().
 				FromDirectory(t, test.fixture).
 				ExpectsResolverContentQueries(test.expected).
-				TestCataloger(t, NewGoModuleBinaryCataloger(GoCatalogerOpts{}))
+				TestCataloger(t, NewGoModuleBinaryCataloger(CatalogerConfig{}))
 		})
 	}
 }