Replace core SBOM-creation API with builder pattern

README.md

@@ -160,9 +160,9 @@ This default behavior can be overridden with the `default-image-pull-source` con
 
 By default, Syft will catalog file details and digests for files that are owned by discovered packages. You can change this behavior by using the `SYFT_FILE_METADATA_SELECTION` environment variable or the `file.metadata.selection` configuration option. The options are:
 
-- `all-files`: capture all files from the search space
-- `owned-files`: capture only files owned by packages (default)
-- `no-files`: disable capturing any file information
+- `all`: capture all files from the search space
+- `owned-by-package`: capture only files owned by packages (default)
+- `none`: disable capturing any file information
 
 
 ### Package cataloger selection

cmd/syft/cli/options/catalog.go

@@ -15,6 +15,7 @@ import (
 	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/cataloging/filecataloging"
 	"github.com/anchore/syft/syft/cataloging/pkgcataloging"
+	"github.com/anchore/syft/syft/file/cataloger/filecontent"
 	"github.com/anchore/syft/syft/pkg/cataloger/binary"
 	"github.com/anchore/syft/syft/pkg/cataloger/golang"
 	"github.com/anchore/syft/syft/pkg/cataloger/java"
@@ -114,6 +115,10 @@ func (cfg Catalog) ToFilesConfig() filecataloging.Config {
 	return filecataloging.Config{
 		Selection: cfg.File.Metadata.Selection,
 		Hashers:   hashers,
+		Content: filecontent.Config{
+			Globs:              cfg.File.Content.Globs,
+			SkipFilesAboveSize: cfg.File.Content.SkipFilesAboveSize,
+		},
 	}
 }
 

cmd/syft/cli/options/file.go

@@ -25,18 +25,18 @@ type fileContent struct {
 func defaultFileConfig() fileConfig {
 	return fileConfig{
 		Metadata: fileMetadata{
-			Selection: file.OwnedFilesSelection,
+			Selection: file.FilesOwnedByPackageSelection,
 			Digests:   []string{"sha1", "sha256"},
 		},
 		Content: fileContent{
-			SkipFilesAboveSize: 1 * intFile.MB,
+			SkipFilesAboveSize: 250 * intFile.KB,
 		},
 	}
 }
 
 func (c *fileConfig) PostLoad() error {
 	switch c.Metadata.Selection {
-	case file.NoFilesSelection, file.OwnedFilesSelection, file.AllFilesSelection:
+	case file.NoFilesSelection, file.FilesOwnedByPackageSelection, file.AllFilesSelection:
 		return nil
 	}
 	return fmt.Errorf("invalid file metadata selection: %q", c.Metadata.Selection)

internal/sbomsync/builder.go

@@ -4,7 +4,6 @@ import (
 	"sync"
 
 	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/linux"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/sbom"
@@ -20,10 +19,6 @@ type Builder interface {
 	// nodes
 
 	AddPackages(...pkg.Package)
-	AddFileMetadata(file.Coordinates, file.Metadata)
-	AddFileDigests(file.Coordinates, ...file.Digest)
-	AddFileContents(file.Coordinates, string)
-	AddFileLicenses(file.Coordinates, ...file.License)
 
 	// edges
 
@@ -73,34 +68,6 @@ func (b sbomBuilder) AddPackages(p ...pkg.Package) {
 	b.sbom.Artifacts.Packages.Add(p...)
 }
 
-func (b sbomBuilder) AddFileMetadata(coordinates file.Coordinates, metadata file.Metadata) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	b.sbom.Artifacts.FileMetadata[coordinates] = metadata
-}
-
-func (b sbomBuilder) AddFileDigests(coordinates file.Coordinates, digest ...file.Digest) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	b.sbom.Artifacts.FileDigests[coordinates] = append(b.sbom.Artifacts.FileDigests[coordinates], digest...)
-}
-
-func (b sbomBuilder) AddFileContents(coordinates file.Coordinates, s string) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	b.sbom.Artifacts.FileContents[coordinates] = s
-}
-
-func (b sbomBuilder) AddFileLicenses(coordinates file.Coordinates, license ...file.License) {
-	b.lock.Lock()
-	defer b.lock.Unlock()
-
-	b.sbom.Artifacts.FileLicenses[coordinates] = append(b.sbom.Artifacts.FileLicenses[coordinates], license...)
-}
-
 func (b sbomBuilder) AddRelationships(relationship ...artifact.Relationship) {
 	b.lock.Lock()
 	defer b.lock.Unlock()

internal/task/file_tasks.go

@@ -8,6 +8,7 @@ import (
 	"github.com/anchore/syft/internal/sbomsync"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/file/cataloger/filecontent"
 	"github.com/anchore/syft/syft/file/cataloger/filedigest"
 	"github.com/anchore/syft/syft/file/cataloger/filemetadata"
 	"github.com/anchore/syft/syft/pkg"
@@ -24,24 +25,10 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash
 	fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
 		accessor := builder.(sbomsync.Accessor)
 
-		var coordinates []file.Coordinates
-
-		accessor.ReadFromSBOM(func(sbom *sbom.SBOM) {
-			if selection == file.OwnedFilesSelection {
-				for _, r := range sbom.Relationships {
-					// TODO: double check this logic
-					if r.Type != artifact.ContainsRelationship {
-						continue
-					}
-					if _, ok := r.From.(pkg.Package); !ok {
-						continue
-					}
-					if c, ok := r.To.(file.Coordinates); ok {
-						coordinates = append(coordinates, c)
-					}
-				}
-			}
-		})
+		coordinates, ok := coordinatesForSelection(selection, builder)
+		if !ok {
+			return nil
+		}
 
 		result, err := digestsCataloger.Catalog(resolver, coordinates...)
 		if err != nil {
@@ -68,23 +55,10 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
 	fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
 		accessor := builder.(sbomsync.Accessor)
 
-		var coordinates []file.Coordinates
-
-		accessor.ReadFromSBOM(func(sbom *sbom.SBOM) {
-			if selection == file.OwnedFilesSelection {
-				for _, r := range sbom.Relationships {
-					if r.Type != artifact.ContainsRelationship {
-						continue
-					}
-					if _, ok := r.From.(pkg.Package); !ok {
-						continue
-					}
-					if c, ok := r.To.(file.Coordinates); ok {
-						coordinates = append(coordinates, c)
-					}
-				}
-			}
-		})
+		coordinates, ok := coordinatesForSelection(selection, builder)
+		if !ok {
+			return nil
+		}
 
 		result, err := metadataCataloger.Catalog(resolver, coordinates...)
 		if err != nil {
@@ -100,3 +74,64 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task {
 
 	return NewTask("file-metadata-cataloger", fn)
 }
+
+func NewFileContentCatalogerTask(cfg filecontent.Config) Task {
+	if len(cfg.Globs) == 0 {
+		return nil
+	}
+
+	cat := filecontent.NewCataloger(cfg)
+
+	fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error {
+		accessor := builder.(sbomsync.Accessor)
+
+		result, err := cat.Catalog(resolver)
+		if err != nil {
+			return err
+		}
+
+		accessor.WriteToSBOM(func(sbom *sbom.SBOM) {
+			sbom.Artifacts.FileContents = result
+		})
+
+		return nil
+	}
+
+	return NewTask("file-content-cataloger", fn)
+}
+
+// TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger
+// Today internal to both cataloger this functions differently: a slice of coordinates vs a channel of locations
+func coordinatesForSelection(selection file.Selection, builder sbomsync.Builder) ([]file.Coordinates, bool) {
+	if selection == file.AllFilesSelection {
+		return nil, true
+	}
+
+	if selection == file.FilesOwnedByPackageSelection {
+		var coordinates []file.Coordinates
+
+		accessor := builder.(sbomsync.Accessor)
+
+		accessor.ReadFromSBOM(func(sbom *sbom.SBOM) {
+			for _, r := range sbom.Relationships {
+				if r.Type != artifact.ContainsRelationship {
+					continue
+				}
+				if _, ok := r.From.(pkg.Package); !ok {
+					continue
+				}
+				if c, ok := r.To.(file.Coordinates); ok {
+					coordinates = append(coordinates, c)
+				}
+			}
+		})
+
+		if len(coordinates) == 0 {
+			return nil, false
+		}
+
+		return coordinates, true
+	}
+
+	return nil, false
+}

syft/cataloging/filecataloging/config.go

@@ -9,16 +9,19 @@ import (
 	intFile "github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/file"
+	"github.com/anchore/syft/syft/file/cataloger/filecontent"
 )
 
 type Config struct {
-	Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
-	Hashers   []crypto.Hash  `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
+	Selection file.Selection     `yaml:"selection" json:"selection" mapstructure:"selection"`
+	Hashers   []crypto.Hash      `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
+	Content   filecontent.Config `yaml:"content" json:"content" mapstructure:"content"`
 }
 
 type configMarshaledForm struct {
-	Selection file.Selection `yaml:"selection" json:"selection" mapstructure:"selection"`
-	Hashers   []string       `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
+	Selection file.Selection     `yaml:"selection" json:"selection" mapstructure:"selection"`
+	Hashers   []string           `yaml:"hashers" json:"hashers" mapstructure:"hashers"`
+	Content   filecontent.Config `yaml:"content" json:"content" mapstructure:"content"`
 }
 
 func DefaultConfig() Config {
@@ -27,8 +30,9 @@ func DefaultConfig() Config {
 		log.WithFields("error", err).Warn("unable to create file hashers")
 	}
 	return Config{
-		Selection: file.OwnedFilesSelection,
+		Selection: file.FilesOwnedByPackageSelection,
 		Hashers:   hashers,
+		Content:   filecontent.DefaultConfig(),
 	}
 }
 

syft/cataloging/filecataloging/config_test.go

@@ -21,10 +21,10 @@ func TestConfig_MarshalJSON(t *testing.T) {
 		{
 			name: "converts hashers to strings",
 			cfg: Config{
-				Selection: file.OwnedFilesSelection,
+				Selection: file.FilesOwnedByPackageSelection,
 				Hashers:   []crypto.Hash{crypto.SHA256},
 			},
-			want: []byte(`{"selection":"owned-files","hashers":["sha-256"]}`),
+			want: []byte(`{"selection":"owned-by-package","hashers":["sha-256"],"content":{"globs":null,"skip-files-above-size":0}}`),
 		},
 	}
 	for _, tt := range tests {
@@ -54,9 +54,9 @@ func TestConfig_UnmarshalJSON(t *testing.T) {
 	}{
 		{
 			name: "converts strings to hashers",
-			data: []byte(`{"selection":"owned-files","hashers":["sha-256"]}`),
+			data: []byte(`{"selection":"owned-by-package","hashers":["sha-256"]}`),
 			want: Config{
-				Selection: file.OwnedFilesSelection,
+				Selection: file.FilesOwnedByPackageSelection,
 				Hashers:   []crypto.Hash{crypto.SHA256},
 			},
 		},

syft/configuration_audit_trail_test.go

@@ -101,8 +101,9 @@ func collectJSONTags(t *testing.T, v reflect.Value, tags *[]string, parentTag st
 }
 
 func assertLowercaseKebab(t *testing.T, tag string) {
+	t.Helper()
 	require.NotEmpty(t, tag)
-	assert.Equal(t, tag, strcase.ToKebab(tag))
+	assert.Equal(t, strcase.ToKebab(tag), tag)
 }
 
 func Test_collectJSONTags(t *testing.T) {
@@ -225,7 +226,7 @@ func Test_configurationAuditTrail_MarshalJSON(t *testing.T) {
 			cfg: configurationAuditTrail{
 
 				Files: filecataloging.Config{
-					Selection: file.OwnedFilesSelection,
+					Selection: file.FilesOwnedByPackageSelection,
 					Hashers: []crypto.Hash{
 						crypto.SHA256,
 					},

syft/create_sbom.go

@@ -21,6 +21,8 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
+// CreateSBOM creates a software bill-of-materials from the given source. If the CreateSBOMConfig is nil, then
+// default options will be used.
 func CreateSBOM(ctx context.Context, src source.Source, cfg *CreateSBOMConfig) (*sbom.SBOM, error) {
 	if cfg == nil {
 		cfg = DefaultCreateSBOMConfig()

syft/create_sbom_config.go

@@ -146,7 +146,7 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task
 	}
 
 	// combine the user-provided and configured tasks
-	if c.Files.Selection == file.OwnedFilesSelection {
+	if c.Files.Selection == file.FilesOwnedByPackageSelection {
 		// special case: we need the package info when we are cataloging files owned by packages
 		taskGroups = append(taskGroups, pkgTasks, fileTasks)
 	} else {
@@ -182,6 +182,10 @@ func (c *CreateSBOMConfig) fileTasks() []task.Task {
 	if t := task.NewFileMetadataCatalogerTask(c.Files.Selection); t != nil {
 		tsks = append(tsks, t)
 	}
+	if t := task.NewFileContentCatalogerTask(c.Files.Content); t != nil {
+		tsks = append(tsks, t)
+	}
+
 	return tsks
 }