diff --git a/README.md b/README.md index d2639a6f6e51..4614dc4e8f1b 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ trufflehog s3 --bucket= --only-verified TruffleHog v3 is a complete rewrite in Go with many new powerful features. - We've **added over 700 credential detectors that support active verification against their respective APIs**. -- We've also added native **support for scanning GitHub, GitLab, filesystems, and S3**. +- We've also added native **support for scanning GitHub, GitLab, filesystems, S3, and Circle CI**. - **Instantly verify private keys** against millions of github users and **billions** of TLS certificates using our [Driftwood](https://trufflesecurity.com/blog/driftwood) technology. @@ -160,6 +160,7 @@ TruffleHog has a sub-command for each source of data that you may want to scan: - S3 - filesystem - syslog +- circleci - file and stdin (coming soon) Each subcommand can have options that you can see with the `--help` flag provided to the sub command: diff --git a/main.go b/main.go index f6d77ece7896..e8802abfdf86 100644 --- a/main.go +++ b/main.go @@ -95,6 +95,9 @@ var ( syslogTLSCert = syslogScan.Flag("cert", "Path to TLS cert.").String() syslogTLSKey = syslogScan.Flag("key", "Path to TLS key.").String() syslogFormat = syslogScan.Flag("format", "Log format. Can be rfc3164 or rfc5424").String() + + circleCiScan = cli.Command("circleci", "Scan CircleCI") + circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String() ) func init() { @@ -293,6 +296,10 @@ func run(state overseer.State) { if err = e.ScanSyslog(ctx, sources.NewConfig(syslog)); err != nil { logrus.WithError(err).Fatal("Failed to scan syslog.") } + case circleCiScan.FullCommand(): + if err = e.ScanCircleCI(ctx, *circleCiScanToken); err != nil { + logrus.WithError(err).Fatal("Failed to scan CircleCI.") + } } // asynchronously wait for scanning to finish and cleanup go e.Finish(ctx) diff --git a/pkg/engine/circleci.go b/pkg/engine/circleci.go new file mode 100644 index 000000000000..7c3625f8f14f --- /dev/null +++ b/pkg/engine/circleci.go @@ -0,0 +1,48 @@ +package engine + +import ( + "runtime" + + "github.com/go-errors/errors" + "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci" +) + +// ScanS3 scans CircleCI logs. +func (e *Engine) ScanCircleCI(ctx context.Context, token string) error { + connection := &sourcespb.CircleCI{ + Credential: &sourcespb.CircleCI_Token{ + Token: token, + }, + } + + var conn anypb.Any + err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) + if err != nil { + logrus.WithError(err).Error("failed to marshal Circle CI connection") + return err + } + + circleSource := circleci.Source{} + err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU()) + if err != nil { + return errors.WrapPrefix(err, "failed to init Circle CI source", 0) + } + + e.sourcesWg.Add(1) + go func() { + defer common.RecoverWithExit(ctx) + defer e.sourcesWg.Done() + err := circleSource.Chunks(ctx, e.ChunksChan()) + if err != nil { + logrus.WithError(err).Error("error scanning Circle CI") + } + }() + return nil +} diff --git a/pkg/engine/s3.go b/pkg/engine/s3.go index 98f9ca7212c2..900670e1f38a 100644 --- a/pkg/engine/s3.go +++ b/pkg/engine/s3.go @@ -42,7 +42,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error { var conn anypb.Any err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) if err != nil { - logrus.WithError(err).Error("failed to marshal github connection") + logrus.WithError(err).Error("failed to marshal S3 connection") return err } @@ -58,7 +58,7 @@ func (e *Engine) ScanS3(ctx context.Context, c sources.Config) error { defer e.sourcesWg.Done() err := s3Source.Chunks(ctx, e.ChunksChan()) if err != nil { - logrus.WithError(err).Error("error scanning s3") + logrus.WithError(err).Error("error scanning S3") } }() return nil diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.go new file mode 100644 index 000000000000..7366b90761b5 --- /dev/null +++ b/pkg/sources/circleci/circleci.go @@ -0,0 +1,256 @@ +package circleci + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/go-errors/errors" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +const ( + baseURL = "https://circleci.com/api/v1.1/" +) + +type Source struct { + name string + sourceId int64 + jobId int64 + verify bool + sources.Progress + token string + client *http.Client +} + +// Ensure the Source satisfies the interface at compile time +var _ sources.Source = (*Source)(nil) + +// Type returns the type of source. +// It is used for matching source types in configuration and job input. +func (s *Source) Type() sourcespb.SourceType { + return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI +} + +func (s *Source) SourceID() int64 { + return s.sourceId +} + +func (s *Source) JobID() int64 { + return s.jobId +} + +// Init returns an initialized Filesystem source. +func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error { + s.name = name + s.sourceId = sourceId + s.jobId = jobId + s.verify = verify + s.client = common.RetryableHttpClientTimeout(3) + + var conn sourcespb.CircleCI + if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil { + return errors.WrapPrefix(err, "error unmarshalling connection", 0) + } + + switch conn.Credential.(type) { + case *sourcespb.CircleCI_Token: + s.token = conn.GetToken() + } + + return nil +} + +// Chunks emits chunks of bytes over a channel. +func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error { + projects, err := s.projects(ctx) + if err != nil { + return err + } + + for _, proj := range projects { + builds, err := s.buildsForProject(ctx, proj) + if err != nil { + return err + } + + for _, bld := range builds { + buildSteps, err := s.stepsForBuild(ctx, proj, bld) + if err != nil { + return err + } + + for _, step := range buildSteps { + for _, action := range step.Actions { + err = s.chunkAction(ctx, proj, bld, action, step.Name, chunksChan) + if err != nil { + return err + } + } + } + } + } + + return nil +} + +type project struct { + VCS string `json:"vcs_type"` + Username string `json:"username"` + RepoName string `json:"reponame"` +} + +func (s *Source) projects(ctx context.Context) ([]project, error) { + reqURL := fmt.Sprintf("%sprojects", baseURL) + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Circle-Token", s.token) + req.Header.Set("Accept", "application/json") + res, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode > 399 && res.StatusCode < 500 { + return nil, fmt.Errorf("invalid credentials, status %d", res.StatusCode) + } + + var projects []project + if err := json.NewDecoder(res.Body).Decode(&projects); err != nil { + return nil, err + } + + return projects, nil +} + +type build struct { + BuildNum int `json:"build_num"` +} + +func (s *Source) buildsForProject(ctx context.Context, proj project) ([]build, error) { + reqURL := fmt.Sprintf("%sproject/%s/%s/%s", baseURL, proj.VCS, proj.Username, proj.RepoName) + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Circle-Token", s.token) + req.Header.Set("Accept", "application/json") + res, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + var builds []build + if err := json.NewDecoder(res.Body).Decode(&builds); err != nil { + return nil, err + } + + return builds, nil +} + +type action struct { + Index int `json:"index"` + OutputURL string `json:"output_url"` +} + +type buildStep struct { + Name string `json:"name"` + Actions []action `json:"actions"` +} + +func (s *Source) stepsForBuild(ctx context.Context, proj project, bld build) ([]buildStep, error) { + reqURL := fmt.Sprintf("%sproject/%s/%s/%s/%d", baseURL, proj.VCS, proj.Username, proj.RepoName, bld.BuildNum) + req, err := http.NewRequest("GET", reqURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("Circle-Token", s.token) + req.Header.Set("Accept", "application/json") + res, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + type buildRes struct { + Steps []buildStep `json:"steps"` + } + + var bldRes buildRes + if err := json.NewDecoder(res.Body).Decode(&bldRes); err != nil { + return nil, err + } + + return bldRes.Steps, nil +} + +func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error { + req, err := http.NewRequest("GET", act.OutputURL, nil) + if err != nil { + return err + } + res, err := s.client.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + logOutput, err := ioutil.ReadAll(res.Body) + if err != nil { + return err + } + + linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum) + + chunk := &sources.Chunk{ + SourceType: s.Type(), + SourceName: s.name, + SourceID: s.SourceID(), + Data: removeCircleSha1Line(logOutput), + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Circleci{ + Circleci: &source_metadatapb.CircleCI{ + VcsType: proj.VCS, + Username: proj.Username, + Repository: proj.RepoName, + BuildNumber: int64(bld.BuildNum), + BuildStep: stepName, + Link: linkURL, + }, + }, + }, + Verify: s.verify, + } + + chunksChan <- chunk + + return nil +} + +func removeCircleSha1Line(input []byte) []byte { + // Split the input slice into a slice of lines + lines := bytes.Split(input, []byte("\n")) + + // Iterate over the lines and add the ones that don't contain "CIRCLE_SHA1=" to the result slice + result := make([][]byte, 0, len(lines)) + for _, line := range lines { + if !bytes.Contains(line, []byte("CIRCLE_SHA1=")) { + result = append(result, line) + } + } + + // Join the lines in the result slice and return the resulting slice + return bytes.Join(result, []byte("\n")) +} diff --git a/pkg/sources/circleci/circleci_test.go b/pkg/sources/circleci/circleci_test.go new file mode 100644 index 000000000000..2be4891f02e7 --- /dev/null +++ b/pkg/sources/circleci/circleci_test.go @@ -0,0 +1,93 @@ +package circleci + +import ( + "fmt" + "testing" + "time" + + "github.com/kylelemons/godebug/pretty" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources" +) + +func TestSource_Scan(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + secret, err := common.GetTestSecret(ctx) + if err != nil { + t.Fatal(fmt.Errorf("failed to access secret: %v", err)) + } + token := secret.MustGetField("CIRCLECI_TOKEN") + + type init struct { + name string + verify bool + connection *sourcespb.CircleCI + } + tests := []struct { + name string + init init + wantSourceMetadata *source_metadatapb.MetaData + wantErr bool + }{ + { + name: "get a chunk", + init: init{ + name: "example repo", + connection: &sourcespb.CircleCI{ + Credential: &sourcespb.CircleCI_Token{ + Token: token, + }, + }, + verify: true, + }, + wantSourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Circleci{ + Circleci: &source_metadatapb.CircleCI{ + VcsType: "github", + Username: "dustin-decker", + Repository: "circle-ci", + BuildNumber: 2, + BuildStep: "Spin up environment", + Link: "https://app.circleci.com/pipelines/github/dustin-decker/circle-ci/2", + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := Source{} + + conn, err := anypb.New(tt.init.connection) + if err != nil { + t.Fatal(err) + } + + err = s.Init(ctx, tt.init.name, 0, 0, tt.init.verify, conn, 5) + if (err != nil) != tt.wantErr { + t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr) + return + } + chunksCh := make(chan *sources.Chunk, 1) + go func() { + err = s.Chunks(ctx, chunksCh) + if (err != nil) != tt.wantErr { + t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr) + return + } + }() + gotChunk := <-chunksCh + if diff := pretty.Compare(gotChunk.SourceMetadata, tt.wantSourceMetadata); diff != "" { + t.Errorf("Source.Chunks() %s diff: (-got +want)\n%s", tt.name, diff) + } + }) + } +}