check policy against user repository files

.gitignore

@@ -13,4 +13,5 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
-tmp/
+tmp/
+testdata/tmp/

check/check.go

@@ -2,41 +2,54 @@ package check
 
 import (
 	"context"
-	"path/filepath"
-	"strings"
 
 	"github.com/kobtea/gorgo/config"
 	"github.com/kobtea/gorgo/fetch"
+	"github.com/kobtea/gorgo/storage"
 	"github.com/open-policy-agent/conftest/output"
 	"github.com/open-policy-agent/conftest/runner"
 )
 
 func Check(ctx context.Context, cfg *config.Config) error {
 	var result []output.CheckResult
+	st := storage.NewStorage(cfg.WorkingDir)
+	// metadata
 	for _, elm := range cfg.Users {
-		pat := filepath.Join(cfg.WorkingDir, fetch.MetadataDirname, elm.Name, "*", fetch.RepoFilename)
-		files, err := filepath.Glob(pat)
+		files, err := st.ListUserRepoPaths(fetch.MetadataDirname, "github.com", elm.Name, elm.Regex.Regexp, fetch.RepoFilename)
 		if err != nil {
 			return err
 		}
-		var matchFiles []string
-		for _, file := range files {
-			l := strings.Split(file, "/")
-			repoName := l[len(l)-2]
-			if elm.Regex.Match([]byte(repoName)) {
-				matchFiles = append(matchFiles, file)
-			}
-		}
+
 		r := runner.TestRunner{
 			AllNamespaces: true,
 			Policy:        elm.RepoPolicies,
 		}
-		res, err := r.Run(ctx, matchFiles)
+		res, err := r.Run(ctx, files)
 		if err != nil {
 			return err
 		}
 		result = append(result, res...)
 	}
+	// source
+	for _, elm := range cfg.Users {
+		for _, srcPolicy := range elm.SrcPolicies {
+			paths, err := st.ListUserRepoPaths("src", "github.com", elm.Name, elm.Regex.Regexp, srcPolicy.Input)
+			if err != nil {
+				return err
+			}
+			r := runner.TestRunner{
+				AllNamespaces: true,
+				Combine:       srcPolicy.Combine,
+				Policy:        srcPolicy.Policies,
+			}
+			res, err := r.Run(ctx, paths)
+			if err != nil {
+				return err
+			}
+			result = append(result, res...)
+		}
+	}
+
 	// FIXME: support multi format
 	outputter := output.Get("", output.Options{})
 	if err := outputter.Output(result); err != nil {

config/config.go

@@ -35,9 +35,10 @@ type Config struct {
 }
 
 type User struct {
-	Name         string   `yaml:"name"`
-	Regex        *Regexp  `yaml:"regex,omitempty"`
-	RepoPolicies []string `yaml:"repo_policies"`
+	Name         string         `yaml:"name"`
+	Regex        *Regexp        `yaml:"regex,omitempty"`
+	RepoPolicies []string       `yaml:"repo_policies"`
+	SrcPolicies  []SourcePolicy `yaml:"src_policies"`
 }
 
 func (s *User) UnmarshalYAML(unmarshal func(interface{}) error) error {
@@ -52,6 +53,12 @@ func (s *User) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return nil
 }
 
+type SourcePolicy struct {
+	Input    string   `yaml:"input"`
+	Combine  bool     `yaml:"combine"`
+	Policies []string `yaml:"policies"`
+}
+
 func Validate(c *Config) []error {
 	var res []error
 	for _, users := range c.Users {

example/config.yaml

@@ -1,9 +1,14 @@
-working_dir: ./tmp/
+working_dir: ./testdata/tmp/
 users:
   - name: kobtea
     regex: jsonnet
     repo_policies:
       - ./example/policy/github_repo.rego
+    src_policies:
+      - input: .github/workflows
+        combine: false
+        policies:
+          - ./example/policy/github_actions.rego
   - name: kobtea
     regex: exporter
     repo_policies:

example/policy/github_actions.rego

@@ -0,0 +1,6 @@
+package github.actions
+
+warn[msg] {
+  count({x | input.jobs[_].steps[x].name == "Install dependencies"}) == 0
+  msg := "GitHub actions should be defined `Install dependencies` step"
+}

fetch/fetch.go

@@ -7,13 +7,16 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/go-git/go-git/v5"
 	"github.com/google/go-github/v48/github"
 	"github.com/kobtea/gorgo/config"
+	"github.com/kobtea/gorgo/storage"
 	"golang.org/x/oauth2"
 )
 
 const (
 	MetadataDirname = "metadata"
+	SourceDirname   = "src"
 	RepoFilename    = "repo.json"
 )
 
@@ -31,7 +34,7 @@ func Fetch(ctx context.Context, cfg *config.Config) error {
 	return nil
 }
 
-func NewClient(ctx context.Context) (*github.Client, error) {
+func newClient(ctx context.Context) (*github.Client, error) {
 	token := os.Getenv("GITHUB_TOKEN")
 	if len(token) == 0 {
 		return nil, fmt.Errorf("require GITHUB_TOKEN env var")
@@ -45,31 +48,57 @@ func NewClient(ctx context.Context) (*github.Client, error) {
 }
 
 func fetchUserRepositories(ctx context.Context, name string, regexes []*config.Regexp, outputDir string) error {
-	cli, err := NewClient(ctx)
+	st := storage.NewStorage(outputDir)
+	cli, err := newClient(ctx)
 	if err != nil {
 		return err
 	}
 	opt := &github.RepositoryListOptions{}
 	for {
+		// TODO: support ghe domain
 		repos, resp, err := cli.Repositories.List(ctx, name, opt)
 		if err != nil {
 			return err
 		}
 		for _, repo := range repos {
 			for _, r := range regexes {
 				if r.Match([]byte(*repo.Name)) {
+					// metadata
 					j, err := json.Marshal(repo)
 					if err != nil {
 						return err
 					}
-					dir := filepath.Join(outputDir, MetadataDirname, name, *repo.Name)
+					dir := st.UserRepoPath(MetadataDirname, "github.com", name, *repo.Name)
 					if err = os.MkdirAll(dir, 0755); err != nil {
 						return err
 					}
-
 					if err = os.WriteFile(filepath.Join(dir, RepoFilename), j, 0644); err != nil {
 						return err
 					}
+
+					// source
+					srcPath := st.UserRepoPath(SourceDirname, "github.com", name, *repo.Name)
+					gitRepo, err := git.PlainOpen(srcPath)
+					if err == git.ErrRepositoryNotExists {
+						gitRepo, err = git.PlainClone(srcPath, false, &git.CloneOptions{
+							URL:   *repo.CloneURL,
+							Depth: 1,
+						})
+						if err != nil {
+							return err
+						}
+					} else if err != nil {
+						return err
+					} else {
+						wt, err := gitRepo.Worktree()
+						if err != nil {
+							return err
+						}
+						if err = wt.Pull(&git.PullOptions{Depth: 1}); err != nil {
+							return err
+
+						}
+					}
 				}
 			}
 		}

go.mod

@@ -3,10 +3,12 @@ module github.com/kobtea/gorgo
 go 1.19
 
 require (
+	github.com/go-git/go-git/v5 v5.5.1
 	github.com/google/go-github/v48 v48.2.0
 	github.com/open-policy-agent/conftest v0.37.0
 	github.com/open-policy-agent/opa v0.47.4
 	github.com/spf13/cobra v1.6.1
+	github.com/stretchr/testify v1.8.1
 	golang.org/x/oauth2 v0.3.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -22,7 +24,10 @@ require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/CycloneDX/cyclonedx-go v0.4.0 // indirect
 	github.com/KeisukeYamashita/go-vcl v0.4.0 // indirect
+	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 // indirect
+	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
@@ -31,18 +36,23 @@ require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/cloudflare/circl v1.1.0 // indirect
 	github.com/cockroachdb/apd/v2 v2.0.1 // indirect
 	github.com/containerd/containerd v1.6.10 // indirect
 	github.com/containerd/typeurl v1.0.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/cli v20.10.18+incompatible // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/docker v20.10.18+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
 	github.com/go-akka/configuration v0.0.0-20200606091224-a002c0330665 // indirect
+	github.com/go-git/gcfg v1.5.0 // indirect
+	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -61,9 +71,12 @@ require (
 	github.com/hashicorp/go-version v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.9.1 // indirect
+	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jstemmer/go-junit-report v0.9.1 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
@@ -81,28 +94,33 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc1 // indirect
+	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.14.0 // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.4.2 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/shteou/go-ignore v0.3.1 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/spdx/tools-golang v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	github.com/tchap/go-patricia/v2 v2.3.1 // indirect
 	github.com/tmccombs/hcl2json v0.3.1 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/yashtewari/glob-intersection v0.1.0 // indirect
 	github.com/zclconf/go-cty v1.8.1 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/crypto v0.3.0 // indirect
 	golang.org/x/net v0.4.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/sys v0.3.0 // indirect
@@ -113,6 +131,7 @@ require (
 	google.golang.org/genproto v0.0.0-20221202195650-67e5cbc046fd // indirect
 	google.golang.org/grpc v1.51.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	muzzammil.xyz/jsonc v1.0.0 // indirect
 	olympos.io/encoding/edn v0.0.0-20201019073823-d3554ca0b0a3 // indirect