forked from trufflesecurity/trufflehog
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Circle CI source (trufflesecurity#997)
* Add Circle CI source * remove SHA1 line * remove trim
- Loading branch information
1 parent
3fadec9
commit 5f6143f
Showing
6 changed files
with
408 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
package engine | ||
|
||
import ( | ||
"runtime" | ||
|
||
"github.com/go-errors/errors" | ||
"github.com/sirupsen/logrus" | ||
"google.golang.org/protobuf/proto" | ||
"google.golang.org/protobuf/types/known/anypb" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/common" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/context" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/circleci" | ||
) | ||
|
||
// ScanS3 scans CircleCI logs. | ||
func (e *Engine) ScanCircleCI(ctx context.Context, token string) error { | ||
connection := &sourcespb.CircleCI{ | ||
Credential: &sourcespb.CircleCI_Token{ | ||
Token: token, | ||
}, | ||
} | ||
|
||
var conn anypb.Any | ||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}) | ||
if err != nil { | ||
logrus.WithError(err).Error("failed to marshal Circle CI connection") | ||
return err | ||
} | ||
|
||
circleSource := circleci.Source{} | ||
err = circleSource.Init(ctx, "trufflehog - Circle CI", 0, int64(sourcespb.SourceType_SOURCE_TYPE_CIRCLECI), true, &conn, runtime.NumCPU()) | ||
if err != nil { | ||
return errors.WrapPrefix(err, "failed to init Circle CI source", 0) | ||
} | ||
|
||
e.sourcesWg.Add(1) | ||
go func() { | ||
defer common.RecoverWithExit(ctx) | ||
defer e.sourcesWg.Done() | ||
err := circleSource.Chunks(ctx, e.ChunksChan()) | ||
if err != nil { | ||
logrus.WithError(err).Error("error scanning Circle CI") | ||
} | ||
}() | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,256 @@ | ||
package circleci | ||
|
||
import ( | ||
"bytes" | ||
"encoding/json" | ||
"fmt" | ||
"io/ioutil" | ||
"net/http" | ||
|
||
"github.com/go-errors/errors" | ||
"google.golang.org/protobuf/proto" | ||
"google.golang.org/protobuf/types/known/anypb" | ||
|
||
"github.com/trufflesecurity/trufflehog/v3/pkg/common" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/context" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" | ||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources" | ||
) | ||
|
||
const ( | ||
baseURL = "https://circleci.com/api/v1.1/" | ||
) | ||
|
||
type Source struct { | ||
name string | ||
sourceId int64 | ||
jobId int64 | ||
verify bool | ||
sources.Progress | ||
token string | ||
client *http.Client | ||
} | ||
|
||
// Ensure the Source satisfies the interface at compile time | ||
var _ sources.Source = (*Source)(nil) | ||
|
||
// Type returns the type of source. | ||
// It is used for matching source types in configuration and job input. | ||
func (s *Source) Type() sourcespb.SourceType { | ||
return sourcespb.SourceType_SOURCE_TYPE_CIRCLECI | ||
} | ||
|
||
func (s *Source) SourceID() int64 { | ||
return s.sourceId | ||
} | ||
|
||
func (s *Source) JobID() int64 { | ||
return s.jobId | ||
} | ||
|
||
// Init returns an initialized Filesystem source. | ||
func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, _ int) error { | ||
s.name = name | ||
s.sourceId = sourceId | ||
s.jobId = jobId | ||
s.verify = verify | ||
s.client = common.RetryableHttpClientTimeout(3) | ||
|
||
var conn sourcespb.CircleCI | ||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil { | ||
return errors.WrapPrefix(err, "error unmarshalling connection", 0) | ||
} | ||
|
||
switch conn.Credential.(type) { | ||
case *sourcespb.CircleCI_Token: | ||
s.token = conn.GetToken() | ||
} | ||
|
||
return nil | ||
} | ||
|
||
// Chunks emits chunks of bytes over a channel. | ||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error { | ||
projects, err := s.projects(ctx) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for _, proj := range projects { | ||
builds, err := s.buildsForProject(ctx, proj) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for _, bld := range builds { | ||
buildSteps, err := s.stepsForBuild(ctx, proj, bld) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
for _, step := range buildSteps { | ||
for _, action := range step.Actions { | ||
err = s.chunkAction(ctx, proj, bld, action, step.Name, chunksChan) | ||
if err != nil { | ||
return err | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
return nil | ||
} | ||
|
||
type project struct { | ||
VCS string `json:"vcs_type"` | ||
Username string `json:"username"` | ||
RepoName string `json:"reponame"` | ||
} | ||
|
||
func (s *Source) projects(ctx context.Context) ([]project, error) { | ||
reqURL := fmt.Sprintf("%sprojects", baseURL) | ||
req, err := http.NewRequest("GET", reqURL, nil) | ||
if err != nil { | ||
return nil, err | ||
} | ||
req.Header.Set("Circle-Token", s.token) | ||
req.Header.Set("Accept", "application/json") | ||
res, err := s.client.Do(req) | ||
if err != nil { | ||
return nil, err | ||
} | ||
defer res.Body.Close() | ||
|
||
if res.StatusCode > 399 && res.StatusCode < 500 { | ||
return nil, fmt.Errorf("invalid credentials, status %d", res.StatusCode) | ||
} | ||
|
||
var projects []project | ||
if err := json.NewDecoder(res.Body).Decode(&projects); err != nil { | ||
return nil, err | ||
} | ||
|
||
return projects, nil | ||
} | ||
|
||
type build struct { | ||
BuildNum int `json:"build_num"` | ||
} | ||
|
||
func (s *Source) buildsForProject(ctx context.Context, proj project) ([]build, error) { | ||
reqURL := fmt.Sprintf("%sproject/%s/%s/%s", baseURL, proj.VCS, proj.Username, proj.RepoName) | ||
req, err := http.NewRequest("GET", reqURL, nil) | ||
if err != nil { | ||
return nil, err | ||
} | ||
req.Header.Set("Circle-Token", s.token) | ||
req.Header.Set("Accept", "application/json") | ||
res, err := s.client.Do(req) | ||
if err != nil { | ||
return nil, err | ||
} | ||
defer res.Body.Close() | ||
|
||
var builds []build | ||
if err := json.NewDecoder(res.Body).Decode(&builds); err != nil { | ||
return nil, err | ||
} | ||
|
||
return builds, nil | ||
} | ||
|
||
type action struct { | ||
Index int `json:"index"` | ||
OutputURL string `json:"output_url"` | ||
} | ||
|
||
type buildStep struct { | ||
Name string `json:"name"` | ||
Actions []action `json:"actions"` | ||
} | ||
|
||
func (s *Source) stepsForBuild(ctx context.Context, proj project, bld build) ([]buildStep, error) { | ||
reqURL := fmt.Sprintf("%sproject/%s/%s/%s/%d", baseURL, proj.VCS, proj.Username, proj.RepoName, bld.BuildNum) | ||
req, err := http.NewRequest("GET", reqURL, nil) | ||
if err != nil { | ||
return nil, err | ||
} | ||
req.Header.Set("Circle-Token", s.token) | ||
req.Header.Set("Accept", "application/json") | ||
res, err := s.client.Do(req) | ||
if err != nil { | ||
return nil, err | ||
} | ||
defer res.Body.Close() | ||
|
||
type buildRes struct { | ||
Steps []buildStep `json:"steps"` | ||
} | ||
|
||
var bldRes buildRes | ||
if err := json.NewDecoder(res.Body).Decode(&bldRes); err != nil { | ||
return nil, err | ||
} | ||
|
||
return bldRes.Steps, nil | ||
} | ||
|
||
func (s *Source) chunkAction(ctx context.Context, proj project, bld build, act action, stepName string, chunksChan chan *sources.Chunk) error { | ||
req, err := http.NewRequest("GET", act.OutputURL, nil) | ||
if err != nil { | ||
return err | ||
} | ||
res, err := s.client.Do(req) | ||
if err != nil { | ||
return err | ||
} | ||
defer res.Body.Close() | ||
logOutput, err := ioutil.ReadAll(res.Body) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
linkURL := fmt.Sprintf("https://app.circleci.com/pipelines/%s/%s/%s/%d", proj.VCS, proj.Username, proj.RepoName, bld.BuildNum) | ||
|
||
chunk := &sources.Chunk{ | ||
SourceType: s.Type(), | ||
SourceName: s.name, | ||
SourceID: s.SourceID(), | ||
Data: removeCircleSha1Line(logOutput), | ||
SourceMetadata: &source_metadatapb.MetaData{ | ||
Data: &source_metadatapb.MetaData_Circleci{ | ||
Circleci: &source_metadatapb.CircleCI{ | ||
VcsType: proj.VCS, | ||
Username: proj.Username, | ||
Repository: proj.RepoName, | ||
BuildNumber: int64(bld.BuildNum), | ||
BuildStep: stepName, | ||
Link: linkURL, | ||
}, | ||
}, | ||
}, | ||
Verify: s.verify, | ||
} | ||
|
||
chunksChan <- chunk | ||
|
||
return nil | ||
} | ||
|
||
func removeCircleSha1Line(input []byte) []byte { | ||
// Split the input slice into a slice of lines | ||
lines := bytes.Split(input, []byte("\n")) | ||
|
||
// Iterate over the lines and add the ones that don't contain "CIRCLE_SHA1=" to the result slice | ||
result := make([][]byte, 0, len(lines)) | ||
for _, line := range lines { | ||
if !bytes.Contains(line, []byte("CIRCLE_SHA1=")) { | ||
result = append(result, line) | ||
} | ||
} | ||
|
||
// Join the lines in the result slice and return the resulting slice | ||
return bytes.Join(result, []byte("\n")) | ||
} |
Oops, something went wrong.