feat(compute/secrets): redact common org secrets

pkg/commands/compute/secrets.go

@@ -6,13 +6,21 @@ import (
 	"strings"
 )
 
-// filterSecretsPattern attempts to capture a secret assigned in an environment
+// FilterSecretEnvVarPattern attempts to capture a secret assigned in an environment
 // variable where the key follows a common pattern.
+//
+// Example:
 // https://regex101.com/r/4GnH3r/1
-const filterSecretsPattern = `(?i)\b[^\s_]+_(?:API|CLIENTSECRET|CREDENTIALS|KEY|PASSWORD|SECRET|TOKEN)(?:[^=]+)?=(?:\s+)?"?([^\s"]+)` // #nosec G101 (CWE-798)
+const FilterSecretEnvVarPattern = `(?i)\b[^\s_]+_(?:API|CLIENTSECRET|CREDENTIALS|KEY|PASSWORD|SECRET|TOKEN)(?:[^=]+)?=(?:\s+)?"?([^\s"]+)` // #nosec G101 (CWE-798)
 
-// filterEnvVarSecrets identify environment variables containing secrets.
-var filterEnvVarSecrets = []string{
+// FilterStaticSecretEnvVar is a static list of env vars containing secrets.
+//
+// NOTE: Env Vars pulled from https://github.com/Puliczek/awesome-list-of-secrets-in-environment-variables
+//
+// The reason for not listing more environment variables is because we have a
+// generalised pattern `FilterSecretEnvVarPattern` that catches the majority of
+// formats used.
+var FilterStaticSecretEnvVar = []string{
 	"AZURE_CLIENT_ID",
 	"CI_JOB_JWT",
 	"CI_JOB_JWT_V2",
@@ -22,6 +30,44 @@ var filterEnvVarSecrets = []string{
 	"OKTA_OAUTH2_CLIENTID",
 }
 
+// AWSIDPattern is the pattern for an AWS ID.
+var AWSIDPattern = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA|ASIA)[0-9A-Z]{16})\b`)
+
+// AWSSecretPattern is the pattern for an AWS Secret.
+var AWSSecretPattern = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)
+
+// GitHubOAuthTokenPattern is the pattern for a GitHub OAuth token.
+var GitHubOAuthTokenPattern = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
+
+// GitHubOldOAuthTokenPattern is the pattern for an older GitHub OAuth token format.
+var GitHubOldOAuthTokenPattern = regexp.MustCompile(`(?i)(?:github|gh|pat|token)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
+
+// GitHubOAuth2ClientIDPattern is the pattern for a GitHub OAuth2 ClientID.
+var GitHubOAuth2ClientIDPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}\b([a-f0-9]{20})\b`)
+
+// GitHubOAuth2ClientSecretPattern is the pattern for a GitHub OAuth2 ClientID.
+var GitHubOAuth2ClientSecretPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}\b([a-f0-9]{40})\b`)
+
+// GitHubAppIDPattern is the pattern for a GitHub App ID.
+var GitHubAppIDPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}\b([0-9]{6})\b`)
+
+// GitHubAppKeyPattern is the pattern for a GitHub App Key.
+var GitHubAppKeyPattern = regexp.MustCompile(`(?i)(?:github)(?:.|[\n\r]){0,40}(-----BEGIN RSA PRIVATE KEY-----\s[A-Za-z0-9+\/\s]*\s-----END RSA PRIVATE KEY-----)`)
+
+// SecretPatterns is a collection of secret identifying patterns.
+//
+// NOTE: Patterns pulled from https://github.com/trufflesecurity/trufflehog
+var SecretPatterns = []*regexp.Regexp{
+	AWSIDPattern,
+	AWSSecretPattern,
+	GitHubOAuthTokenPattern,
+	GitHubOldOAuthTokenPattern,
+	GitHubOAuth2ClientIDPattern,
+	GitHubOAuth2ClientSecretPattern,
+	GitHubAppIDPattern,
+	GitHubAppKeyPattern,
+}
+
 // ExtendEnvVarSecretsFilter mutates filterEnvVarSecrets to include user
 // specified environment variables. The `filter` argument is comma-separated.
 func ExtendEnvVarSecretsFilter(filter string) {
@@ -31,27 +77,43 @@ func ExtendEnvVarSecretsFilter(filter string) {
 			continue
 		}
 		var found bool
-		for _, f := range filterEnvVarSecrets {
+		for _, f := range FilterStaticSecretEnvVar {
 			if f == v {
 				found = true
 				break
 			}
 		}
 		if !found {
-			filterEnvVarSecrets = append(filterEnvVarSecrets, v)
+			FilterStaticSecretEnvVar = append(FilterStaticSecretEnvVar, v)
 		}
 	}
 }
 
 // FilterEnvVarSecretsFromSlice mutates the input data such that any value
 // assigned to an environment variable (identified as containing a secret) is
 // redacted.
+//
+// Additionally, we filter based on known secret patterns (`SecretPatterns`).
 func FilterEnvVarSecretsFromSlice(data []string) {
-	for i, v := range data {
-		for _, f := range filterEnvVarSecrets {
-			k := strings.Split(v, "=")[0]
-			if strings.HasPrefix(k, f) {
+	for i, keypair := range data {
+		k, v, found := strings.Cut(keypair, "=")
+		if !found {
+			return
+		}
+		for _, f := range FilterStaticSecretEnvVar {
+			if k == f {
 				data[i] = k + "=REDACTED"
+				break
+			}
+		}
+		if strings.Contains(data[i], "REDACTED") {
+			continue
+		}
+		for _, pattern := range SecretPatterns {
+			n := pattern.ReplaceAllString(v, "REDACTED")
+			data[i] = k + "=" + n
+			if n == "REDACTED" {
+				break
 			}
 		}
 	}
@@ -63,15 +125,20 @@ func FilterEnvVarSecretsFromSlice(data []string) {
 //
 // Example:
 // https://go.dev/play/p/GYXMNc7Froz
+//
+// Additionally, we filter based on known secret patterns (`SecretPatterns`).
 func FilterEnvVarSecretsFromBytes(data []byte) []byte {
-	re := regexp.MustCompile(filterSecretsPattern)
+	re := regexp.MustCompile(FilterSecretEnvVarPattern)
 	for _, matches := range re.FindAllSubmatch(data, -1) {
 		if len(matches) == 2 {
 			o := matches[0]
 			n := bytes.ReplaceAll(matches[0], matches[1], []byte("REDACTED"))
 			data = bytes.ReplaceAll(data, o, n)
 		}
 	}
+	for _, pattern := range SecretPatterns {
+		data = pattern.ReplaceAll(data, []byte("REDACTED"))
+	}
 	return data
 }
 
@@ -81,14 +148,19 @@ func FilterEnvVarSecretsFromBytes(data []byte) []byte {
 //
 // Example:
 // https://go.dev/play/p/GYXMNc7Froz
+//
+// Additionally, we filter based on known secret patterns (`SecretPatterns`).
 func FilterEnvVarSecretsFromString(data string) string {
-	re := regexp.MustCompile(filterSecretsPattern)
+	re := regexp.MustCompile(FilterSecretEnvVarPattern)
 	for _, matches := range re.FindAllStringSubmatch(data, -1) {
 		if len(matches) == 2 {
 			o := matches[0]
 			n := strings.ReplaceAll(matches[0], matches[1], "REDACTED")
 			data = strings.ReplaceAll(data, o, n)
 		}
 	}
+	for _, pattern := range SecretPatterns {
+		data = pattern.ReplaceAllString(data, "REDACTED")
+	}
 	return data
 }