add file source digest support (anchore#1914)

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

.gitignore

@@ -1,3 +1,5 @@
+go.work
+go.work.sum
 /.bin
 CHANGELOG.md
 VERSION

README.md

@@ -605,7 +605,7 @@ file-metadata:
     # SYFT_FILE_METADATA_CATALOGER_SCOPE env var
     scope: "squashed"
 
-  # the file digest algorithms to use when cataloging files (options: "sha256", "md5", "sha1")
+  # the file digest algorithms to use when cataloging files (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")
   # SYFT_FILE_METADATA_DIGESTS env var
   digests: ["sha256"]
 
@@ -643,11 +643,27 @@ secrets:
   # SYFT_SECRETS_EXCLUDE_PATTERN_NAMES env var
   exclude-pattern-names: []
 
+# options that apply to all scan sources
+source:
+  # alias name for the source
+  # SYFT_SOURCE_NAME env var; --source-name flag
+  name: ""
+
+  # alias version for the source
+  # SYFT_SOURCE_VERSION env var; --source-version flag
+  version: ""
+
+  # options affecting the file source type
+  file:
+    # the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")
+    digests: ["sha256"]
+
 # options when pulling directly from a registry via the "registry:" scheme
 registry:
   # skip TLS verification when communicating with the registry
   # SYFT_REGISTRY_INSECURE_SKIP_TLS_VERIFY env var
   insecure-skip-tls-verify: false
+
   # use http instead of https when connecting to the registry
   # SYFT_REGISTRY_INSECURE_USE_HTTP env var
   insecure-use-http: false

cmd/syft/cli/attest/attest.go

@@ -18,6 +18,7 @@ import (
 	"github.com/anchore/syft/cmd/syft/cli/packages"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/config"
+	"github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/ui"
 	"github.com/anchore/syft/syft"
@@ -74,18 +75,23 @@ func buildSBOM(app *config.Application, userInput string, errs chan error) (*sbo
 		}
 	}
 
+	hashers, err := file.Hashers(app.Source.File.Digests...)
+	if err != nil {
+		return nil, fmt.Errorf("invalid hash: %w", err)
+	}
+
 	src, err := detection.NewSource(
 		source.DetectionSourceConfig{
 			Alias: source.Alias{
-				Name:    app.SourceName,
-				Version: app.SourceVersion,
+				Name:    app.Source.Name,
+				Version: app.Source.Version,
 			},
 			RegistryOptions: app.Registry.ToOptions(),
 			Platform:        platform,
 			Exclude: source.ExcludeConfig{
 				Paths: app.Exclusions,
 			},
-			DigestAlgorithms: nil,
+			DigestAlgorithms: hashers,
 		},
 	)
 

cmd/syft/cli/eventloop/tasks.go

@@ -1,13 +1,10 @@
 package eventloop
 
 import (
-	"crypto"
-	"fmt"
-
 	"github.com/anchore/syft/internal/config"
+	"github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft"
 	"github.com/anchore/syft/syft/artifact"
-	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/file/cataloger/filecontent"
 	"github.com/anchore/syft/syft/file/cataloger/filedigest"
 	"github.com/anchore/syft/syft/file/cataloger/filemetadata"
@@ -89,23 +86,9 @@ func generateCatalogFileDigestsTask(app *config.Application) (Task, error) {
 		return nil, nil
 	}
 
-	supportedHashAlgorithms := make(map[string]crypto.Hash)
-	for _, h := range []crypto.Hash{
-		crypto.MD5,
-		crypto.SHA1,
-		crypto.SHA256,
-	} {
-		supportedHashAlgorithms[file.DigestAlgorithmName(h)] = h
-	}
-
-	var hashes []crypto.Hash
-	for _, hashStr := range app.FileMetadata.Digests {
-		name := file.CleanDigestAlgorithmName(hashStr)
-		hashObj, ok := supportedHashAlgorithms[name]
-		if !ok {
-			return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr)
-		}
-		hashes = append(hashes, hashObj)
+	hashes, err := file.Hashers(app.FileMetadata.Digests...)
+	if err != nil {
+		return nil, err
 	}
 
 	digestsCataloger := filedigest.NewCataloger(hashes)

cmd/syft/cli/options/packages.go

@@ -86,11 +86,11 @@ func bindPackageConfigOptions(flags *pflag.FlagSet, v *viper.Viper) error {
 		return err
 	}
 
-	if err := v.BindPFlag("source-name", flags.Lookup("source-name")); err != nil {
+	if err := v.BindPFlag("source.name", flags.Lookup("source-name")); err != nil {
 		return err
 	}
 
-	if err := v.BindPFlag("source-version", flags.Lookup("source-version")); err != nil {
+	if err := v.BindPFlag("source.version", flags.Lookup("source-version")); err != nil {
 		return err
 	}
 

cmd/syft/cli/packages/packages.go

@@ -13,6 +13,7 @@ import (
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/config"
+	"github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/internal/ui"
 	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft"
@@ -77,18 +78,24 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) <
 			}
 		}
 
+		hashers, err := file.Hashers(app.Source.File.Digests...)
+		if err != nil {
+			errs <- fmt.Errorf("invalid hash: %w", err)
+			return
+		}
+
 		src, err := detection.NewSource(
 			source.DetectionSourceConfig{
 				Alias: source.Alias{
-					Name:    app.SourceName,
-					Version: app.SourceVersion,
+					Name:    app.Source.Name,
+					Version: app.Source.Version,
 				},
 				RegistryOptions: app.Registry.ToOptions(),
 				Platform:        platform,
 				Exclude: source.ExcludeConfig{
 					Paths: app.Exclusions,
 				},
-				DigestAlgorithms: nil,
+				DigestAlgorithms: hashers,
 			},
 		)
 

cmd/syft/cli/poweruser/poweruser.go

@@ -94,8 +94,8 @@ func execWorker(app *config.Application, userInput string, writer sbom.Writer) <
 		src, err := detection.NewSource(
 			source.DetectionSourceConfig{
 				Alias: source.Alias{
-					Name:    app.SourceName,
-					Version: app.SourceVersion,
+					Name:    app.Source.Name,
+					Version: app.Source.Version,
 				},
 				RegistryOptions: app.Registry.ToOptions(),
 				Platform:        platform,

internal/config/application.go

@@ -61,8 +61,7 @@ type Application struct {
 	Exclusions             []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 	Platform               string             `yaml:"platform" json:"platform" mapstructure:"platform"`
 	Name                   string             `yaml:"name" json:"name" mapstructure:"name"`
-	SourceName             string             `yaml:"source-name" json:"source-name" mapstructure:"source-name"`
-	SourceVersion          string             `yaml:"source-version" json:"source-version" mapstructure:"source-version"`
+	Source                 sourceCfg          `yaml:"source" json:"source" mapstructure:"source"`
 	Parallelism            int                `yaml:"parallelism" json:"parallelism" mapstructure:"parallelism"`                                           // the number of catalog workers to run in parallel
 	DefaultImagePullSource string             `yaml:"default-image-pull-source" json:"default-image-pull-source" mapstructure:"default-image-pull-source"` // specify default image pull source
 }
@@ -147,8 +146,8 @@ func (cfg *Application) parseConfigValues() error {
 
 	if cfg.Name != "" {
 		log.Warnf("name parameter is deprecated. please use: source-name. name will be removed in a future version")
-		if cfg.SourceName == "" {
-			cfg.SourceName = cfg.Name
+		if cfg.Source.Name == "" {
+			cfg.Source.Name = cfg.Name
 		}
 	}
 

internal/config/source.go

@@ -0,0 +1,17 @@
+package config
+
+import "github.com/spf13/viper"
+
+type sourceCfg struct {
+	Name    string     `json:"name" yaml:"name" mapstructure:"name"`
+	Version string     `json:"version" yaml:"version" mapstructure:"version"`
+	File    fileSource `json:"file" yaml:"file" mapstructure:"file"`
+}
+
+type fileSource struct {
+	Digests []string `json:"digests" yaml:"digests" mapstructure:"digests"`
+}
+
+func (cfg sourceCfg) loadDefaultValues(v *viper.Viper) {
+	v.SetDefault("source.file.digests", []string{"sha256"})
+}

internal/file/digest.go

@@ -0,0 +1,76 @@
+package file
+
+import (
+	"crypto"
+	"fmt"
+	"hash"
+	"io"
+	"strings"
+
+	"github.com/anchore/syft/syft/file"
+)
+
+func supportedHashAlgorithms() []crypto.Hash {
+	return []crypto.Hash{
+		crypto.MD5,
+		crypto.SHA1,
+		crypto.SHA224,
+		crypto.SHA256,
+		crypto.SHA384,
+		crypto.SHA512,
+	}
+}
+
+func NewDigestsFromFile(closer io.ReadCloser, hashes []crypto.Hash) ([]file.Digest, error) {
+	// create a set of hasher objects tied together with a single writer to feed content into
+	hashers := make([]hash.Hash, len(hashes))
+	writers := make([]io.Writer, len(hashes))
+	for idx, hashObj := range hashes {
+		hashers[idx] = hashObj.New()
+		writers[idx] = hashers[idx]
+	}
+
+	size, err := io.Copy(io.MultiWriter(writers...), closer)
+	if err != nil {
+		return nil, err
+	}
+
+	if size == 0 {
+		return make([]file.Digest, 0), nil
+	}
+
+	result := make([]file.Digest, len(hashes))
+	// only capture digests when there is content. It is important to do this based on SIZE and not
+	// FILE TYPE. The reasoning is that it is possible for a tar to be crafted with a header-only
+	// file type but a body is still allowed.
+	for idx, hasher := range hashers {
+		result[idx] = file.Digest{
+			Algorithm: CleanDigestAlgorithmName(hashes[idx].String()),
+			Value:     fmt.Sprintf("%+x", hasher.Sum(nil)),
+		}
+	}
+
+	return result, nil
+}
+
+func Hashers(names ...string) ([]crypto.Hash, error) {
+	hashByName := make(map[string]crypto.Hash)
+	for _, h := range supportedHashAlgorithms() {
+		hashByName[CleanDigestAlgorithmName(h.String())] = h
+	}
+
+	var hashers []crypto.Hash
+	for _, hashStr := range names {
+		hashObj, ok := hashByName[CleanDigestAlgorithmName(hashStr)]
+		if !ok {
+			return nil, fmt.Errorf("unsupported hash algorithm: %s", hashStr)
+		}
+		hashers = append(hashers, hashObj)
+	}
+	return hashers, nil
+}
+
+func CleanDigestAlgorithmName(name string) string {
+	lower := strings.ToLower(name)
+	return strings.ReplaceAll(lower, "-", "")
+}