From f49aad68bea9230eb063be06e450595d15abc301 Mon Sep 17 00:00:00 2001 From: Naveen <172697+naveensrinivasan@users.noreply.github.com> Date: Sun, 16 May 2021 20:36:46 -0500 Subject: [PATCH 1/2] :sparkles: Moved the cloudbuilds to yaml (#444) * Moved the cloudbuild configuration into YAML --- cloudbuild/README.md | 3 +++ cloudbuild/cron.yaml | 38 +++++++++++++++++++++++++++++++++++ cloudbuild/scorecard-tag.yaml | 34 +++++++++++++++++++++++++++++++ cloudbuild/scorecard.yaml | 37 ++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+) create mode 100644 cloudbuild/README.md create mode 100644 cloudbuild/cron.yaml create mode 100644 cloudbuild/scorecard-tag.yaml create mode 100644 cloudbuild/scorecard.yaml diff --git a/cloudbuild/README.md b/cloudbuild/README.md new file mode 100644 index 000000000000..6e1261e1d2fb --- /dev/null +++ b/cloudbuild/README.md @@ -0,0 +1,3 @@ +# cloud builds + +The container images for scorecard and the cron job is built using `cloud build` in `gcp`. diff --git a/cloudbuild/cron.yaml b/cloudbuild/cron.yaml new file mode 100644 index 000000000000..b43465b90a49 --- /dev/null +++ b/cloudbuild/cron.yaml @@ -0,0 +1,38 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build: + images: + - gcr.io/openssf/cron:latest + options: + diskSizeGb: '200' + steps: + - args: + - build + - -t + - gcr.io/openssf/cron:latest + - -f + - ./cron/Dockerfile + - . + name: gcr.io/cloud-builders/docker +description: builds image for scorecard cron +github: + name: scorecard + owner: ossf + push: + branch: ^main$ +name: scorecard-cron +tags: +- cron +- scorecard diff --git a/cloudbuild/scorecard-tag.yaml b/cloudbuild/scorecard-tag.yaml new file mode 100644 index 000000000000..fc26021b42ef --- /dev/null +++ b/cloudbuild/scorecard-tag.yaml @@ -0,0 +1,34 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build: + images: + - gcr.io/openssf/scorecard:$TAG_NAME + options: {} + steps: + - args: + - build + - -t + - gcr.io/openssf/scorecard:$TAG_NAME + - . + name: gcr.io/cloud-builders/docker +description: scorecard build based on tag +github: + name: scorecard + owner: ossf + push: + tag: .* +name: scorecard-tag +tags: +- v* diff --git a/cloudbuild/scorecard.yaml b/cloudbuild/scorecard.yaml new file mode 100644 index 000000000000..b3207a996834 --- /dev/null +++ b/cloudbuild/scorecard.yaml @@ -0,0 +1,37 @@ +# Copyright 2021 Security Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +build: + images: + - gcr.io/openssf/scorecard:latest + options: + diskSizeGb: '200' + steps: + - args: + - build + - -t + - gcr.io/openssf/scorecard:latest + - -f + - Dockerfile + - . + name: gcr.io/cloud-builders/docker +description: 'builds image for scorecard ' +github: + name: scorecard + owner: ossf + push: + branch: ^main$ +name: scorecard +tags: +- scorecard From 37519d967279800529b0712e4f811d6b2d7fbd4f Mon Sep 17 00:00:00 2001 From: Azeem Shaikh Date: Sun, 16 May 2021 19:38:46 -0700 Subject: [PATCH 2/2] Update RunScorecards API. (#461) Co-authored-by: Azeem Shaikh --- Makefile | 12 ++++++------ cmd/root.go | 11 ++++++++++- cmd/serve.go | 11 ++++++++++- cron/main.go | 32 +++++++++++++++++++++++--------- pkg/scorecard.go | 47 ++++++++++++++++++++++------------------------- 5 files changed, 71 insertions(+), 42 deletions(-) diff --git a/Makefile b/Makefile index acfe0afd8053..cc5340059bfb 100644 --- a/Makefile +++ b/Makefile @@ -153,15 +153,15 @@ test-disk-cache: build-scorecard | $(GINKGO) mkdir cache @echo Focusing on these tests $(FOCUS_DISK_TEST) USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" \ - ./scorecard \ - --repo=https://github.com/ossf/scorecard \ - --show-details --metadata=openssf --format json > ./$(OUTPUT)/results.json + ./scorecard \ + --repo=https://github.com/ossf/scorecard \ + --show-details --metadata=openssf --format json > ./$(OUTPUT)/results.json USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" ginkgo -p -v -cover --focus=$(FOCUS_DISK_TEST) ./e2e/... # Rerun the same test with the disk cache filled to make sure the cache is working. USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" \ - ./scorecard \ - --repo=https://github.com/ossf/scorecard --show-details \ - --metadata=openssf --format json > ./$(OUTPUT)/results.json + ./scorecard \ + --repo=https://github.com/ossf/scorecard --show-details \ + --metadata=openssf --format json > ./$(OUTPUT)/results.json USE_DISK_CACHE=1 DISK_CACHE_PATH="./cache" ginkgo -p -v -cover --focus=$(FOCUS_DISK_TEST) ./e2e/... e2e-cron: ## Runs a e2e test cron job and validates its functionality diff --git a/cmd/root.go b/cmd/root.go index d437a9653804..652a4b7eadb8 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -28,10 +28,13 @@ import ( goflag "flag" + "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checker" "github.com/ossf/scorecard/checks" "github.com/ossf/scorecard/pkg" "github.com/ossf/scorecard/repos" + "github.com/ossf/scorecard/roundtripper" + "github.com/shurcooL/githubv4" "github.com/spf13/cobra" "go.uber.org/zap" ) @@ -123,7 +126,13 @@ or ./scorecard --{npm,pypi,rubgems}= [--checks=check1,...] [--show } ctx := context.Background() - repoResult := pkg.RunScorecards(ctx, sugar, repo, enabledChecks) + rt := roundtripper.NewTransport(ctx, sugar) + httpClient := &http.Client{ + Transport: rt, + } + githubClient := github.NewClient(httpClient) + graphClient := githubv4.NewClient(httpClient) + repoResult := pkg.RunScorecards(ctx, repo, enabledChecks, httpClient, githubClient, graphClient) repoResult.Metadata = append(repoResult.Metadata, metaData...) // Sort them by name diff --git a/cmd/serve.go b/cmd/serve.go index 231144f52488..a4f3f7452553 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -22,9 +22,12 @@ import ( "os" "strings" + "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checks" "github.com/ossf/scorecard/pkg" "github.com/ossf/scorecard/repos" + "github.com/ossf/scorecard/roundtripper" + "github.com/shurcooL/githubv4" "github.com/spf13/cobra" "go.uber.org/zap" ) @@ -65,7 +68,13 @@ var serveCmd = &cobra.Command{ } sugar.Info(repoParam) ctx := r.Context() - repoResult := pkg.RunScorecards(ctx, sugar, repo, checks.AllChecks) + rt := roundtripper.NewTransport(ctx, sugar) + httpClient := &http.Client{ + Transport: rt, + } + githubClient := github.NewClient(httpClient) + graphClient := githubv4.NewClient(httpClient) + repoResult := pkg.RunScorecards(ctx, repo, checks.AllChecks, httpClient, githubClient, graphClient) if r.Header.Get("Content-Type") == "application/json" { if err := repoResult.AsJSON(showDetails, rw); err != nil { diff --git a/cron/main.go b/cron/main.go index 3e192b95c21f..669a83f3aa16 100644 --- a/cron/main.go +++ b/cron/main.go @@ -19,14 +19,18 @@ import ( "fmt" "io/ioutil" "log" + "net/http" "os" "os/exec" "time" + "github.com/google/go-github/v32/github" "github.com/jszwec/csvutil" "github.com/ossf/scorecard/checks" "github.com/ossf/scorecard/pkg" "github.com/ossf/scorecard/repos" + "github.com/ossf/scorecard/roundtripper" + "github.com/shurcooL/githubv4" "go.uber.org/zap" ) @@ -60,6 +64,23 @@ func main() { if err != nil { panic(err) } + + ctx := context.Background() + cfg := zap.NewProductionConfig() + cfg.Level.SetLevel(zap.InfoLevel) + logger, err := cfg.Build() + if err != nil { + panic(err) + } + sugar := logger.Sugar() + // Use our custom roundtripper + rt := roundtripper.NewTransport(ctx, sugar) + httpClient := &http.Client{ + Transport: rt, + } + githubClient := github.NewClient(httpClient) + graphClient := githubv4.NewClient(httpClient) + for _, r := range inputRepos { fmt.Println(r.Repo) @@ -71,21 +92,14 @@ func main() { panic(err) } - ctx := context.Background() - cfg := zap.NewProductionConfig() - cfg.Level.SetLevel(zap.InfoLevel) - logger, err := cfg.Build() - if err != nil { - panic(err) - } - sugar := logger.Sugar() //nolint // FIXME :- deleting branch-protection // The branch protection check needs an admin access to the repository. // All of the checks from cron would fail and uses another call to the API. // This will reduce usage of the API. delete(checks.AllChecks, "Branch-Protection") - repoResult := pkg.RunScorecards(ctx, sugar, repoURL, checks.AllChecks) + + repoResult := pkg.RunScorecards(ctx, repoURL, checks.AllChecks, httpClient, githubClient, graphClient) if err := repoResult.AsJSON( /*showDetails=*/ true, result); err != nil { panic(err) } diff --git a/pkg/scorecard.go b/pkg/scorecard.go index 168497595feb..128bc1f5d154 100644 --- a/pkg/scorecard.go +++ b/pkg/scorecard.go @@ -23,37 +23,28 @@ import ( "github.com/google/go-github/v32/github" "github.com/ossf/scorecard/checker" "github.com/ossf/scorecard/repos" - "github.com/ossf/scorecard/roundtripper" "github.com/shurcooL/githubv4" - "go.uber.org/zap" ) -func runEnabledChecks(ctx context.Context, logger *zap.SugaredLogger, repo repos.RepoURL, - checksToRun checker.CheckNameToFnMap, resultsCh chan checker.CheckResult) { +func runEnabledChecks(ctx context.Context, + repo repos.RepoURL, checksToRun checker.CheckNameToFnMap, + httpClient *http.Client, githubClient *github.Client, graphClient *githubv4.Client, + resultsCh chan checker.CheckResult) { + request := checker.CheckRequest{ + Ctx: ctx, + Client: githubClient, + HTTPClient: httpClient, + Owner: repo.Owner, + Repo: repo.Repo, + GraphClient: graphClient, + } wg := sync.WaitGroup{} for _, checkFn := range checksToRun { checkFn := checkFn wg.Add(1) go func() { - // Use our custom roundtripper - rt := roundtripper.NewTransport(ctx, logger) - - client := &http.Client{ - Transport: rt, - } - ghClient := github.NewClient(client) - graphClient := githubv4.NewClient(client) - - c := checker.CheckRequest{ - Ctx: ctx, - Client: ghClient, - HTTPClient: client, - Owner: repo.Owner, - Repo: repo.Repo, - GraphClient: graphClient, - } defer wg.Done() - runner := checker.Runner{CheckRequest: c} + runner := checker.Runner{CheckRequest: request} resultsCh <- runner.Run(checkFn) }() } @@ -61,14 +52,20 @@ func runEnabledChecks(ctx context.Context, logger *zap.SugaredLogger, repo repos close(resultsCh) } -func RunScorecards(ctx context.Context, logger *zap.SugaredLogger, - repo repos.RepoURL, checksToRun checker.CheckNameToFnMap) repos.RepoResult { +func RunScorecards(ctx context.Context, + repo repos.RepoURL, + checksToRun checker.CheckNameToFnMap, + httpClient *http.Client, + githubClient *github.Client, + graphClient *githubv4.Client) repos.RepoResult { ret := repos.RepoResult{ Repo: repo.URL(), Date: time.Now().Format("2006-01-02"), } resultsCh := make(chan checker.CheckResult) - go runEnabledChecks(ctx, logger, repo, checksToRun, resultsCh) + go runEnabledChecks(ctx, repo, checksToRun, + httpClient, githubClient, graphClient, + resultsCh) for result := range resultsCh { ret.Checks = append(ret.Checks, result) }