Feature: Adding the ability to extend configuration files (gitleaks#926)

* init

* working on default and path config extensions

* adding trace log level, consolidating some code

* cleaning things up, updating generate package

* fix tests

* formatting

* adding tests for extend

* extend not extends

* formatting

* only allow usedefault or path to be set

* update readme

* add note about allowlists

* more readme, expand env var for path

* actually dont support env var. ez attack

README.md

@@ -17,7 +17,7 @@
 		  <img src="https://img.shields.io/docker/pulls/zricethezav/gitleaks.svg" />
 	  </a>
 	  <a href="https://www.jit.io/jit-open-source-gitleaks?utm_source=github&utm_medium=badge&utm_campaign=GitleaksReadme&utm_id=oss&items=item-secret-detection">
-<img src="https://img.shields.io/badge/Secured%20by-Jit-B8287F?style=?style=plastic" /> 
+<img src="https://img.shields.io/badge/Secured%20by-Jit-B8287F?style=?style=plastic" />
 	  </a>
 	  <a href="https://github.com/zricethezav/gitleaks-action">
         	<img alt="gitleaks badge" src="https://img.shields.io/badge/protected%20by-gitleaks-blue">
@@ -28,7 +28,7 @@
   </p>
 </p>
 
-Gitleaks is a SAST tool for **detecting** and **preventing** hardcoded secrets like passwords, api keys, and tokens in git repos. Gitleaks is an **easy-to-use, all-in-one solution** for detecting secrets, past or present, in your code. 
+Gitleaks is a SAST tool for **detecting** and **preventing** hardcoded secrets like passwords, api keys, and tokens in git repos. Gitleaks is an **easy-to-use, all-in-one solution** for detecting secrets, past or present, in your code.
 
 | Demos |
 | ----------- |
@@ -258,6 +258,23 @@ Gitleaks offers a configuration format you can follow to write your own secret d
 # Title for the gitleaks configuration file.
 title = "Gitleaks title"
 
+# Extend the base (this) configuration. When you extend a configuration
+# the base rules take precendence over the extended rules. I.e, if there are
+# duplicate rules in both the base configuration and the extended configuration
+# the base rules will override the extended rules.
+# Another thing to know with extending configurations is you can chain together
+# multiple configuration files to a depth of 2. Allowlist arrays are appended
+# and can contain duplicates.
+# useDefault and path can NOT be used at the same time. Choose one.
+[extend]
+# useDefault will extend the base configuration with the default gitleaks config
+useDefault = true
+# or you can supply a path to a configuration. Path is relative to where gitleaks
+# was invoked, not the location of the base config. This is a bit clunky.
+# A better alternative would be to supply an absolute path (expanded env var
+# not supported at this time)
+path = "common_config.toml"
+
 # An array of tables that contain information that define instructions
 # on how to detect secrets
 [[rules]]

cmd/generate/config/main.go

@@ -149,16 +149,18 @@ func main() {
 	configRules = append(configRules, rules.GenericCredential())
 
 	// ensure rules have unique ids
-	ruleLookUp := make(map[string]bool)
+	ruleLookUp := make(map[string]config.Rule)
 	for _, rule := range configRules {
 		// check if rule is in ruleLookUp
 		if _, ok := ruleLookUp[rule.RuleID]; ok {
 			log.Fatal().Msgf("rule id %s is not unique", rule.RuleID)
 		}
-		ruleLookUp[rule.RuleID] = true
+		// TODO: eventually change all the signatures to get ride of this
+		// nasty dereferencing.
+		ruleLookUp[rule.RuleID] = *rule
 	}
 	config := config.Config{
-		Rules: configRules,
+		Rules: ruleLookUp,
 	}
 	tmpl, err := template.ParseFiles(templatePath)
 	if err != nil {

cmd/generate/config/rules/rule.go

@@ -62,8 +62,10 @@ func validate(r config.Rule, truePositives []string, falsePositives []string) *c
 	}
 	r.Keywords = keywords
 
+	rules := make(map[string]config.Rule)
+	rules[r.RuleID] = r
 	d := detect.NewDetector(config.Config{
-		Rules:    []*config.Rule{&r},
+		Rules:    rules,
 		Keywords: keywords,
 	})
 	for _, tp := range truePositives {

cmd/root.go

@@ -19,13 +19,13 @@ const banner = `
     │╲
     │ ○
     ○ ░
-    ░    gitleaks 
+    ░    gitleaks
 
 `
 
 const configDescription = `config file path
-order of precedence: 
-1. --config/-c 
+order of precedence:
+1. --config/-c
 2. env var GITLEAKS_CONFIG
 3. (--source/-s)/.gitleaks.toml
 If none of the three options are used, then gitleaks will use the default config`
@@ -42,7 +42,7 @@ func init() {
 	rootCmd.PersistentFlags().StringP("source", "s", ".", "path to source (default: $PWD)")
 	rootCmd.PersistentFlags().StringP("report-path", "r", "", "report file")
 	rootCmd.PersistentFlags().StringP("report-format", "f", "json", "output format (json, csv, sarif)")
-	rootCmd.PersistentFlags().StringP("log-level", "l", "info", "log level (debug, info, warn, error, fatal)")
+	rootCmd.PersistentFlags().StringP("log-level", "l", "info", "log level (trace, debug, info, warn, error, fatal)")
 	rootCmd.PersistentFlags().BoolP("verbose", "v", false, "show verbose output from scan")
 	rootCmd.PersistentFlags().Bool("redact", false, "redact secrets from logs and stdout")
 	err := viper.BindPFlag("config", rootCmd.PersistentFlags().Lookup("config"))
@@ -58,6 +58,8 @@ func initLog() {
 		log.Fatal().Msg(err.Error())
 	}
 	switch strings.ToLower(ll) {
+	case "trace":
+		zerolog.SetGlobalLevel(zerolog.TraceLevel)
 	case "debug":
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 	case "info":
@@ -81,11 +83,11 @@ func initConfig() {
 	}
 	if cfgPath != "" {
 		viper.SetConfigFile(cfgPath)
-		log.Debug().Msgf("Using gitleaks config %s from `--config`", cfgPath)
+		log.Debug().Msgf("using gitleaks config %s from `--config`", cfgPath)
 	} else if os.Getenv("GITLEAKS_CONFIG") != "" {
 		envPath := os.Getenv("GITLEAKS_CONFIG")
 		viper.SetConfigFile(envPath)
-		log.Debug().Msgf("Using gitleaks config from GITLEAKS_CONFIG env var: %s", envPath)
+		log.Debug().Msgf("using gitleaks config from GITLEAKS_CONFIG env var: %s", envPath)
 	} else {
 		source, err := rootCmd.Flags().GetString("source")
 		if err != nil {
@@ -97,7 +99,7 @@ func initConfig() {
 		}
 
 		if !fileInfo.IsDir() {
-			log.Debug().Msgf("Unable to load gitleaks config from %s since --source=%s is a file, using default config",
+			log.Debug().Msgf("unable to load gitleaks config from %s since --source=%s is a file, using default config",
 				filepath.Join(source, ".gitleaks.toml"), source)
 			viper.SetConfigType("toml")
 			if err = viper.ReadConfig(strings.NewReader(config.DefaultConfig)); err != nil {
@@ -107,22 +109,22 @@ func initConfig() {
 		}
 
 		if _, err := os.Stat(filepath.Join(source, ".gitleaks.toml")); os.IsNotExist(err) {
-			log.Debug().Msgf("No gitleaks config found in path %s, using default gitleaks config", filepath.Join(source, ".gitleaks.toml"))
+			log.Debug().Msgf("no gitleaks config found in path %s, using default gitleaks config", filepath.Join(source, ".gitleaks.toml"))
 			viper.SetConfigType("toml")
 			if err = viper.ReadConfig(strings.NewReader(config.DefaultConfig)); err != nil {
 				log.Fatal().Msgf("err reading default config toml %s", err.Error())
 			}
 			return
 		} else {
-			log.Debug().Msgf("Using existing gitleaks config %s from `(--source)/.gitleaks.toml`", filepath.Join(source, ".gitleaks.toml"))
+			log.Debug().Msgf("using existing gitleaks config %s from `(--source)/.gitleaks.toml`", filepath.Join(source, ".gitleaks.toml"))
 		}
 
 		viper.AddConfigPath(source)
 		viper.SetConfigName(".gitleaks")
 		viper.SetConfigType("toml")
 	}
 	if err := viper.ReadInConfig(); err != nil {
-		log.Fatal().Msgf("Unable to load gitleaks config, err: %s", err)
+		log.Fatal().Msgf("unable to load gitleaks config, err: %s", err)
 	}
 }
 

config/config.go

@@ -5,16 +5,26 @@ import (
 	"fmt"
 	"regexp"
 	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
 )
 
 //go:embed gitleaks.toml
 var DefaultConfig string
 
+// use to keep track of how many configs we can extend
+// yea I know, globals bad
+var extendDepth int
+
+const maxExtendDepth = 2
+
 // ViperConfig is the config struct used by the Viper config package
 // to parse the config file. This struct does not include regular expressions.
 // It is used as an intermediary to convert the Viper config to the Config struct.
 type ViperConfig struct {
 	Description string
+	Extend      Extend
 	Rules       []struct {
 		ID          string
 		Description string
@@ -42,18 +52,32 @@ type ViperConfig struct {
 
 // Config is a configuration struct that contains rules and an allowlist if present.
 type Config struct {
+	Extend      Extend
 	Path        string
 	Description string
-	Rules       []*Rule
+	Rules       map[string]Rule
 	Allowlist   Allowlist
 	Keywords    []string
+
+	// used to keep sarif results consistent
+	orderedRules []string
+}
+
+// Extend is a struct that allows users to define how they want their
+// configuration extended by other configuration files.
+type Extend struct {
+	Path       string
+	URL        string
+	UseDefault bool
 }
 
 func (vc *ViperConfig) Translate() (Config, error) {
 	var (
-		rules    []*Rule
-		keywords []string
+		keywords     []string
+		orderedRules []string
 	)
+	rulesMap := make(map[string]Rule)
+
 	for _, r := range vc.Rules {
 		var allowlistRegexes []*regexp.Regexp
 		for _, a := range r.Allowlist.Regexes {
@@ -88,7 +112,7 @@ func (vc *ViperConfig) Translate() (Config, error) {
 		} else {
 			configPathRegex = regexp.MustCompile(r.Path)
 		}
-		r := &Rule{
+		r := Rule{
 			Description: r.Description,
 			RuleID:      r.ID,
 			Regex:       configRegex,
@@ -104,10 +128,12 @@ func (vc *ViperConfig) Translate() (Config, error) {
 				StopWords: r.Allowlist.StopWords,
 			},
 		}
+		orderedRules = append(orderedRules, r.RuleID)
+
 		if r.Regex != nil && r.SecretGroup > r.Regex.NumSubexp() {
 			return Config{}, fmt.Errorf("%s invalid regex secret group %d, max regex secret group %d", r.Description, r.SecretGroup, r.Regex.NumSubexp())
 		}
-		rules = append(rules, r)
+		rulesMap[r.RuleID] = r
 	}
 	var allowlistRegexes []*regexp.Regexp
 	for _, a := range vc.Allowlist.Regexes {
@@ -117,15 +143,107 @@ func (vc *ViperConfig) Translate() (Config, error) {
 	for _, a := range vc.Allowlist.Paths {
 		allowlistPaths = append(allowlistPaths, regexp.MustCompile(a))
 	}
-	return Config{
+	c := Config{
 		Description: vc.Description,
-		Rules:       rules,
+		Extend:      vc.Extend,
+		Rules:       rulesMap,
 		Allowlist: Allowlist{
 			Regexes:   allowlistRegexes,
 			Paths:     allowlistPaths,
 			Commits:   vc.Allowlist.Commits,
 			StopWords: vc.Allowlist.StopWords,
 		},
-		Keywords: keywords,
-	}, nil
+		Keywords:     keywords,
+		orderedRules: orderedRules,
+	}
+
+	if maxExtendDepth != extendDepth {
+		// disallow both usedefault and path from being set
+		if c.Extend.Path != "" && c.Extend.UseDefault {
+			log.Fatal().Msg("unable to load config due to extend.path and extend.useDefault being set")
+		}
+		if c.Extend.UseDefault {
+			c.extendDefault()
+		} else if c.Extend.Path != "" {
+			c.extendPath()
+		}
+
+	}
+
+	return c, nil
+}
+
+func (c *Config) OrderedRules() []Rule {
+	var orderedRules []Rule
+	for _, id := range c.orderedRules {
+		if _, ok := c.Rules[id]; ok {
+			orderedRules = append(orderedRules, c.Rules[id])
+		}
+	}
+	return orderedRules
+}
+
+func (c *Config) extendDefault() {
+	extendDepth++
+	viper.SetConfigType("toml")
+	if err := viper.ReadConfig(strings.NewReader(DefaultConfig)); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	defaultViperConfig := ViperConfig{}
+	if err := viper.Unmarshal(&defaultViperConfig); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	cfg, err := defaultViperConfig.Translate()
+	if err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	log.Debug().Msg("extending config with default config")
+	c.extend(cfg)
+
+}
+
+func (c *Config) extendPath() {
+	extendDepth++
+	viper.SetConfigFile(c.Extend.Path)
+	if err := viper.ReadInConfig(); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	extensionViperConfig := ViperConfig{}
+	if err := viper.Unmarshal(&extensionViperConfig); err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	cfg, err := extensionViperConfig.Translate()
+	if err != nil {
+		log.Fatal().Msgf("failed to load extended config, err: %s", err)
+		return
+	}
+	log.Debug().Msgf("extending config with %s", c.Extend.Path)
+	c.extend(cfg)
+}
+
+func (c *Config) extendURL() {
+	// TODO
+}
+
+func (c *Config) extend(extensionConfig Config) {
+	for ruleID, rule := range extensionConfig.Rules {
+		if _, ok := c.Rules[ruleID]; !ok {
+			log.Trace().Msgf("adding %s to base config", ruleID)
+			c.Rules[ruleID] = rule
+			c.Keywords = append(c.Keywords, rule.Keywords...)
+		}
+	}
+
+	// append allowlists, not attempting to merge
+	c.Allowlist.Commits = append(c.Allowlist.Commits,
+		extensionConfig.Allowlist.Commits...)
+	c.Allowlist.Paths = append(c.Allowlist.Paths,
+		extensionConfig.Allowlist.Paths...)
+	c.Allowlist.Regexes = append(c.Allowlist.Regexes,
+		extensionConfig.Allowlist.Regexes...)
 }