add identity search workflow

Signed-off-by: linus-sun <linussun@google.com>

.github/workflows/main.yml

@@ -90,4 +90,6 @@ jobs:
           check-latest: true
       - name: run e2e test
         run: ./pkg/test/e2e/e2e_test.sh
+      - name: run identity monitor test
+        run: ./pkg/test/identity_workflow/identity_workflow_e2e_test.sh
 

cmd/monitor/config.go

@@ -0,0 +1,38 @@
+//
+// Copyright 2024 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"time"
+
+	"github.com/sigstore/rekor-monitor/pkg/identity"
+	"github.com/sigstore/rekor-monitor/pkg/notifications"
+)
+
+type IdentityMonitorConfiguration struct {
+	StartIndex                *int                                    `yaml:"startIndex"`
+	EndIndex                  *int                                    `yaml:"endIndex"`
+	MonitoredValues           identity.MonitoredValues                `yaml:"monitoredValues"`
+	ServerURL                 string                                  `yaml:"serverURL"`
+	OutputIdentitiesFile      string                                  `yaml:"outputIdentities"`
+	LogInfoFile               string                                  `yaml:"logInfoFile"`
+	GitHubIssue               notifications.GitHubIssueInput          `yaml:"githubIssue"`
+	EmailNotificationSMTP     notifications.EmailNotificationInput    `yaml:"emailNotificationSMTP"`
+	EmailNotificationMailgun  notifications.MailgunNotificationInput  `yaml:"emailNotificationMailgun"`
+	EmailNotificationSendGrid notifications.SendGridNotificationInput `yaml:"emailNotificationSendGrid"`
+	Once                      *bool                                   `yaml:"once"`
+	Interval                  *time.Duration                          `yaml:"interval"`
+}

cmd/monitor/main.go

@@ -0,0 +1,146 @@
+//
+// Copyright 2021 The Sigstore Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/sigstore/rekor-monitor/pkg/rekor"
+	"github.com/sigstore/rekor-monitor/pkg/util/file"
+	"github.com/sigstore/rekor/pkg/client"
+	"github.com/sigstore/rekor/pkg/util"
+	"gopkg.in/yaml.v2"
+	"sigs.k8s.io/release-utils/version"
+)
+
+// Default values for monitoring job parameters
+const (
+	publicRekorServerURL     = "https://rekor.sigstore.dev"
+	logInfoFileName          = "logInfo.txt"
+	outputIdentitiesFileName = "identities.txt"
+)
+
+// This main function performs a periodic identity search.
+// Upon starting, any existing latest snapshot data is loaded and the function runs
+// indefinitely to perform identity search for every time interval that was specified.
+func main() {
+	// Command-line flags that are parameters to the verifier job
+	configFilePath := flag.String("config-file", "", "path to yaml configuration file containing identity monitor settings")
+	flag.Parse()
+
+	if configFilePath == nil {
+		log.Fatalf("empty configuration file path")
+	}
+
+	readConfig, err := os.ReadFile(*configFilePath)
+	if err != nil {
+		log.Fatalf("error reading from identity monitor configuration file: %v", err)
+	}
+
+	configString := string(readConfig)
+	var config IdentityMonitorConfiguration
+	if err := yaml.Unmarshal([]byte(configString), &config); err != nil {
+		log.Fatalf("error parsing identities: %v", err)
+	}
+
+	rekorClient, err := client.GetRekorClient(config.ServerURL, client.WithUserAgent(strings.TrimSpace(fmt.Sprintf("rekor-monitor/%s (%s; %s)", version.GetVersionInfo().GitVersion, runtime.GOOS, runtime.GOARCH))))
+	if err != nil {
+		log.Fatalf("getting Rekor client: %v", err)
+	}
+
+	if config.StartIndex == nil || config.EndIndex == nil {
+		logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient)
+		if err != nil {
+			log.Fatalf("error getting log info: %v", err)
+		}
+
+		checkpoint, err := rekor.ReadLatestCheckpoint(logInfo)
+		if err != nil {
+			log.Fatalf("error reading checkpoint: %v", err)
+		}
+
+		var prevCheckpoint *util.SignedCheckpoint
+		prevCheckpoint, err = file.ReadLatestCheckpoint(config.LogInfoFile)
+		if err != nil {
+			log.Fatalf("reading checkpoint log: %v", err)
+		}
+
+		checkpointStartIndex, checkpointEndIndex := rekor.GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
+		if config.StartIndex == nil {
+			config.StartIndex = &checkpointStartIndex
+		}
+		if config.EndIndex == nil {
+			config.EndIndex = &checkpointEndIndex
+		}
+	}
+
+	if config.ServerURL == "" {
+		config.ServerURL = publicRekorServerURL
+	}
+	if config.LogInfoFile == "" {
+		config.LogInfoFile = logInfoFileName
+	}
+	if config.OutputIdentitiesFile == "" {
+		config.OutputIdentitiesFile = outputIdentitiesFileName
+	}
+	if config.Interval == nil {
+		defaultInterval := time.Hour
+		config.Interval = &defaultInterval
+	}
+	if config.Once == nil {
+		defaultOnce := true
+		config.Once = &defaultOnce
+	}
+
+	ticker := time.NewTicker(*config.Interval)
+	defer ticker.Stop()
+
+	// To get an immediate first tick
+	for ; ; <-ticker.C {
+		err = rekor.IdentitySearch(*config.StartIndex, *config.EndIndex, rekorClient, config.MonitoredValues, config.OutputIdentitiesFile)
+		if err != nil {
+			ticker.Stop()
+			log.Fatal(err.Error())
+		}
+
+		if *config.Once {
+			return
+		}
+
+		prevCheckpoint, checkpoint, err := rekor.GetPrevCurrentCheckpoints(rekorClient, config.LogInfoFile)
+		if err != nil {
+			ticker.Stop()
+			log.Fatalf("failed to fetch previous and current checkpoints: %v", err)
+		}
+
+		logInfo, err := rekor.GetLogInfo(context.Background(), rekorClient)
+		if err != nil {
+			ticker.Stop()
+			log.Fatalf("failed to fetch log info: %v", err)
+		}
+
+		checkpointStartIndex, checkpointEndIndex := rekor.GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
+		config.StartIndex = &checkpointStartIndex
+		config.EndIndex = &checkpointEndIndex
+	}
+}

pkg/notifications/email.go

@@ -24,12 +24,12 @@ import (
 // EmailNotificationInput extends the NotificationPlatform interface to support
 // found identity notification by sending emails to a specified user.
 type EmailNotificationInput struct {
-	RecipientEmailAddress string
-	SenderEmailAddress    string
-	SenderSMTPUsername    string
-	SenderSMTPPassword    string
-	SMTPHostURL           string
-	SMTPCustomOptions     []mail.Option
+	RecipientEmailAddress string        `yaml:"recipientEmailAddress"`
+	SenderEmailAddress    string        `yaml:"senderEmailAddress"`
+	SenderSMTPUsername    string        `yaml:"senderSMTPUsername"`
+	SenderSMTPPassword    string        `yaml:"senderSMTPPassword"`
+	SMTPHostURL           string        `yaml:"SMTPHostURL"`
+	SMTPCustomOptions     []mail.Option `yaml:"SMTPCustomOptions"`
 }
 
 func GenerateEmailBody(monitoredIdentities []identity.MonitoredIdentity) (string, error) {

pkg/notifications/github_issues.go

@@ -30,15 +30,15 @@ var (
 // GitHubIssueInput extends the NotificationPlatform interface to support found identity
 // notification via creating new GitHub issues in a given repo.
 type GitHubIssueInput struct {
-	AssigneeUsername string
-	RepositoryOwner  string
-	RepositoryName   string
+	AssigneeUsername string `yaml:"assigneeUsername"`
+	RepositoryOwner  string `yaml:"repositoryOwner"`
+	RepositoryName   string `yaml:"repositoryName"`
 	// The PAT or other access token to authenticate creating an issue.
 	// The authentication token requires repo write and push access.
-	AuthenticationToken string
+	AuthenticationToken string `yaml:"authenticationToken"`
 	// For users who want to pass in a custom client.
 	// If nil, a default client with the given authentication token will be instantiated.
-	GitHubClient *github.Client
+	GitHubClient *github.Client `yaml:"githubClient"`
 }
 
 func generateGitHubIssueBody(monitoredIdentities []identity.MonitoredIdentity) (string, error) {

pkg/notifications/mailgun.go

@@ -24,10 +24,10 @@ import (
 // MailgunNotificationInput extends the NotificationPlatform interface to support
 // found identity notification by sending emails to a specified user via Mailgun.
 type MailgunNotificationInput struct {
-	RecipientEmailAddress string
-	SenderEmailAddress    string
-	MailgunAPIKey         string
-	MailgunDomainName     string
+	RecipientEmailAddress string `yaml:"recipientEmailAddress"`
+	SenderEmailAddress    string `yaml:"senderEmailAddress"`
+	MailgunAPIKey         string `yaml:"mailgunAPIKey"`
+	MailgunDomainName     string `yaml:"mailgunDomainName"`
 }
 
 // Send takes in an MailgunNotificationInput and attempts to send the

pkg/notifications/sendgrid.go

@@ -25,11 +25,11 @@ import (
 // SendGrid extends the NotificationPlatform interface to support
 // found identity notification by sending emails to a specified user via SendGrid.
 type SendGridNotificationInput struct {
-	RecipientName         string
-	RecipientEmailAddress string
-	SenderName            string
-	SenderEmailAddress    string
-	SendGridAPIKey        string
+	RecipientName         string `yaml:"recipientName"`
+	RecipientEmailAddress string `yaml:"recipientEmailAddress"`
+	SenderName            string `yaml:"senderName"`
+	SenderEmailAddress    string `yaml:"senderEmailAddress"`
+	SendGridAPIKey        string `yaml:"sendGridAPIKey"`
 }
 
 // Send takes in an SendGridNotificationInput and attempts to send the

pkg/rekor/identity.go

@@ -341,8 +341,28 @@ func oidMatchesPolicy(cert *x509.Certificate, oid asn1.ObjectIdentifier, extensi
 	return false, nil, "", nil
 }
 
-// writeIdentitiesBetweenCheckpoints monitors for given identities between two checkpoints and writes any found identities to file.
-func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint, monitoredValues identity.MonitoredValues, rekorClient *client.Rekor, outputIdentitiesFile string) error {
+func GetPrevCurrentCheckpoints(client *client.Rekor, logInfoFile string) (*util.SignedCheckpoint, *util.SignedCheckpoint, error) {
+	logInfo, err := GetLogInfo(context.Background(), client)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error getting log info: %v", err)
+	}
+
+	checkpoint, err := ReadLatestCheckpoint(logInfo)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error reading checkpoint: %v", err)
+	}
+
+	var prevCheckpoint *util.SignedCheckpoint
+	prevCheckpoint, err = file.ReadLatestCheckpoint(logInfoFile)
+	if err != nil {
+		return nil, nil, fmt.Errorf("reading checkpoint log: %v", err)
+	}
+
+	return prevCheckpoint, checkpoint, nil
+}
+
+// GetCheckpointIndices fetches the start and end indexes between two checkpoints and returns them.
+func GetCheckpointIndices(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint) (int, int) {
 	// Get log size of inactive shards
 	totalSize := 0
 	for _, s := range logInfo.InactiveShards {
@@ -351,26 +371,43 @@ func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *
 	startIndex := int(prevCheckpoint.Size) + totalSize - 1 //nolint: gosec // G115, log will never be large enough to overflow
 	endIndex := int(checkpoint.Size) + totalSize - 1       //nolint: gosec // G115
 
-	// Search for identities in the log range
-	if identity.MonitoredValuesExist(monitoredValues) {
-		entries, err := GetEntriesByIndexRange(context.Background(), rekorClient, startIndex, endIndex)
-		if err != nil {
-			return fmt.Errorf("error getting entries by index range: %v", err)
-		}
-		idEntries, err := MatchedIndices(entries, monitoredValues)
-		if err != nil {
-			return fmt.Errorf("error finding log indices: %v", err)
-		}
+	return startIndex, endIndex
+}
 
-		if len(idEntries) > 0 {
-			for _, idEntry := range idEntries {
-				fmt.Fprintf(os.Stderr, "Found %s\n", idEntry.String())
+func IdentitySearch(startIndex int, endIndex int, rekorClient *client.Rekor, monitoredValues identity.MonitoredValues, outputIdentitiesFile string) error {
 
-				if err := file.WriteIdentity(outputIdentitiesFile, idEntry); err != nil {
-					return fmt.Errorf("failed to write entry: %v", err)
-				}
+	entries, err := GetEntriesByIndexRange(context.Background(), rekorClient, startIndex, endIndex)
+	if err != nil {
+		return fmt.Errorf("error getting entries by index range: %v", err)
+	}
+	idEntries, err := MatchedIndices(entries, monitoredValues)
+	if err != nil {
+		return fmt.Errorf("error finding log indices: %v", err)
+	}
+
+	if len(idEntries) > 0 {
+		for _, idEntry := range idEntries {
+			fmt.Fprintf(os.Stderr, "Found %s\n", idEntry.String())
+
+			if err := file.WriteIdentity(outputIdentitiesFile, idEntry); err != nil {
+				return fmt.Errorf("failed to write entry: %v", err)
 			}
 		}
 	}
 	return nil
 }
+
+// writeIdentitiesBetweenCheckpoints monitors for given identities between two checkpoints and writes any found identities to file.
+func writeIdentitiesBetweenCheckpoints(logInfo *models.LogInfo, prevCheckpoint *util.SignedCheckpoint, checkpoint *util.SignedCheckpoint, monitoredValues identity.MonitoredValues, rekorClient *client.Rekor, outputIdentitiesFile string) error {
+	// Get log size of inactive shards
+	startIndex, endIndex := GetCheckpointIndices(logInfo, prevCheckpoint, checkpoint)
+
+	// Search for identities in the log range
+	if identity.MonitoredValuesExist(monitoredValues) {
+		err := IdentitySearch(startIndex, endIndex, rekorClient, monitoredValues, outputIdentitiesFile)
+		if err != nil {
+			return fmt.Errorf("error monitoring for identities: %v", err)
+		}
+	}
+	return nil
+}

pkg/rekor/verifier.go

@@ -51,11 +51,21 @@ func GetLogVerifier(ctx context.Context, rekorClient *client.Rekor) (signature.V
 	return verifier, nil
 }
 
+// ReadLatestCheckpoint fetches the latest checkpoint from log info fetched from Rekor.
+// It returns the checkpoint if it successfully fetches one; otherwise, it returns an error.
+func ReadLatestCheckpoint(logInfo *models.LogInfo) (*util.SignedCheckpoint, error) {
+	checkpoint := &util.SignedCheckpoint{}
+	if err := checkpoint.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil {
+		return nil, fmt.Errorf("unmarshalling logInfo.SignedTreeHead to Checkpoint: %v", err)
+	}
+	return checkpoint, nil
+}
+
 // verifyLatestCheckpoint fetches and verifies the signature of the latest checkpoint from log info fetched from Rekor.
 // If it successfully verifies the checkpoint's signature, it returns the checkpoint; otherwise, it returns an error.
 func verifyLatestCheckpointSignature(logInfo *models.LogInfo, verifier signature.Verifier) (*util.SignedCheckpoint, error) {
-	checkpoint := &util.SignedCheckpoint{}
-	if err := checkpoint.UnmarshalText([]byte(*logInfo.SignedTreeHead)); err != nil {
+	checkpoint, err := ReadLatestCheckpoint(logInfo)
+	if err != nil {
 		return nil, fmt.Errorf("unmarshalling logInfo.SignedTreeHead to Checkpoint: %v", err)
 	}
 	if !checkpoint.Verify(verifier) {