From ff03538f1f58369e2ee9f1f52cfd94679344e7f2 Mon Sep 17 00:00:00 2001 From: Balazs Nadasdi Date: Thu, 2 Mar 2023 16:28:22 +0100 Subject: [PATCH] TODO: commit message Signed-off-by: Balazs Nadasdi --- .github/workflows/test.yaml | 6 +- cmd/clusters-service/pkg/git/git.go | 284 +++++------------- .../pkg/git/gitfakes/git_fake.go | 78 ++--- .../pkg/server/automations.go | 15 +- cmd/clusters-service/pkg/server/clusters.go | 67 +++-- .../pkg/server/clusters_test.go | 48 +-- cmd/clusters-service/pkg/server/helpers.go | 14 +- .../pkg/server/helpers_test.go | 70 ++--- cmd/clusters-service/pkg/server/templates.go | 30 +- .../pkg/server/templates_test.go | 6 +- cmd/gitops/app/create/templates/cmd.go | 12 +- cmd/gitops/app/create/templates/cmd_test.go | 4 +- go.mod | 16 +- go.sum | 42 +-- pkg/git/azure.go | 198 ++++++++++++ pkg/git/bitbucket_server.go | 119 ++++++-- pkg/git/factory.go | 95 +++++- pkg/git/github.go | 110 +++++-- pkg/git/gitlab.go | 150 +++++---- pkg/git/gitprovider.go | 217 +++++++++++++ pkg/git/jenkins_scm.go | 180 +++++++++++ pkg/git/provider.go | 25 +- pkg/git/pull_request.go | 28 ++ pkg/git/repository.go | 16 + pkg/git/utils.go | 63 ---- pkg/pipelines/server/list_prs.go | 6 +- pkg/pipelines/server/list_prs_test.go | 4 +- tools/kind-cluster-with-extramounts.yaml | 10 + 28 files changed, 1289 insertions(+), 624 deletions(-) create mode 100644 pkg/git/azure.go create mode 100644 pkg/git/gitprovider.go create mode 100644 pkg/git/jenkins_scm.go create mode 100644 pkg/git/pull_request.go create mode 100644 pkg/git/repository.go diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fca43c9eac..03072b6871 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -147,10 +147,8 @@ jobs: run: | go version make dependencies - cd cmd/clusters-service - go test -v ./... -tags=integration - cd ../../pkg/git - go test -v ./... -tags=integration + go test -v ./cmd/clusters-service/... -tags=integration + go test -v ./pkg/git/... -tags=integration ui-unit-tests: runs-on: ubuntu-latest diff --git a/cmd/clusters-service/pkg/git/git.go b/cmd/clusters-service/pkg/git/git.go index b9c0d2fff7..dead37de8d 100644 --- a/cmd/clusters-service/pkg/git/git.go +++ b/cmd/clusters-service/pkg/git/git.go @@ -2,26 +2,20 @@ package git import ( "context" - "errors" "fmt" "path" "path/filepath" - "regexp" "strings" "time" - "github.com/fluxcd/go-git-providers/github" - "github.com/fluxcd/go-git-providers/gitlab" "github.com/fluxcd/go-git-providers/gitprovider" - "github.com/fluxcd/go-git-providers/stash" go_git "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-logr/logr" "github.com/spf13/viper" - "github.com/weaveworks/weave-gitops/pkg/gitproviders" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/util/retry" ) const deleteFilesCommitMessage = "Delete old files for resources" @@ -35,9 +29,9 @@ var DefaultBackoff = wait.Backoff{ type Provider interface { WriteFilesToBranchAndCreatePullRequest(ctx context.Context, req WriteFilesToBranchAndCreatePullRequestRequest) (*WriteFilesToBranchAndCreatePullRequestResponse, error) - GetRepository(ctx context.Context, gp GitProvider, url string) (gitprovider.OrgRepository, error) - GetTreeList(ctx context.Context, gp GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) - ListPullRequests(ctx context.Context, gp GitProvider, url string) ([]gitprovider.PullRequest, error) + GetRepository(ctx context.Context, gp GitProvider, url string) (*git.Repository, error) + GetTreeList(ctx context.Context, gp GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*git.TreeEntry, error) + ListPullRequests(ctx context.Context, gp GitProvider, url string) ([]*git.PullRequest, error) } type GitProviderService struct { @@ -66,7 +60,7 @@ type WriteFilesToBranchAndCreatePullRequestRequest struct { Title string Description string CommitMessage string - Files []gitprovider.CommitFile + Files []git.CommitFile } type WriteFilesToBranchAndCreatePullRequestResponse struct { @@ -76,20 +70,17 @@ type WriteFilesToBranchAndCreatePullRequestResponse struct { // WriteFilesToBranchAndCreatePullRequest writes a set of provided files // to a new branch and creates a new pull request for that branch. // It returns the URL of the pull request. -func (s *GitProviderService) WriteFilesToBranchAndCreatePullRequest(ctx context.Context, - req WriteFilesToBranchAndCreatePullRequestRequest) (*WriteFilesToBranchAndCreatePullRequestResponse, error) { - commits := []Commit{} +func (s *GitProviderService) WriteFilesToBranchAndCreatePullRequest( + ctx context.Context, + req WriteFilesToBranchAndCreatePullRequestRequest, +) (*WriteFilesToBranchAndCreatePullRequestResponse, error) { + commits := []git.Commit{} repoURL, err := GetGitProviderUrl(req.RepositoryURL) if err != nil { return nil, fmt.Errorf("unable to get git provider url: %w", err) } - repo, err := s.GetRepository(ctx, req.GitProvider, repoURL) - if err != nil { - return nil, fmt.Errorf("unable to get repo: %w", err) - } - // Gitlab doesn't support createOrUpdate, so we need to check if the file exists // and if it does, we need to create a commit to delete the file. if req.GitProvider.Type == "gitlab" { @@ -100,7 +91,7 @@ func (s *GitProviderService) WriteFilesToBranchAndCreatePullRequest(ctx context. // If there are files to delete, append them to the map of changes to be deleted. if len(deletedFiles) > 0 { - commits = append(commits, Commit{ + commits = append(commits, git.Commit{ CommitMessage: deleteFilesCommitMessage, Files: deletedFiles, }) @@ -108,33 +99,30 @@ func (s *GitProviderService) WriteFilesToBranchAndCreatePullRequest(ctx context. } // Add the files to be created to the map of changes. - commits = append(commits, Commit{ + commits = append(commits, git.Commit{ CommitMessage: req.CommitMessage, Files: req.Files, }) - if err := s.writeFilesToBranch(ctx, writeFilesToBranchRequest{ - Repository: repo, - HeadBranch: req.HeadBranch, - BaseBranch: req.BaseBranch, - Commits: commits, - }); err != nil { - return nil, fmt.Errorf("unable to write files to branch %q: %w", req.HeadBranch, err) + provider, err := getGitProviderClient(s.log, req.GitProvider) + if err != nil { + return nil, fmt.Errorf("unable to get a git provider client for %q: %w", req.GitProvider.Type, err) } - res, err := s.createPullRequest(ctx, createPullRequestRequest{ - Repository: repo, - HeadBranch: req.HeadBranch, - BaseBranch: req.BaseBranch, - Title: req.Title, - Description: req.Description, + pr, err := provider.CreatePullRequest(ctx, git.PullRequestInput{ + RepositoryURL: req.RepositoryURL, + Title: req.Title, + Body: req.Description, + Head: req.HeadBranch, + Base: req.BaseBranch, + Commits: commits, }) if err != nil { return nil, fmt.Errorf("unable to create pull request for branch %q: %w", req.HeadBranch, err) } return &WriteFilesToBranchAndCreatePullRequestResponse{ - WebURL: res.WebURL, + WebURL: pr.Link, }, nil } @@ -144,66 +132,13 @@ type GitRepo struct { Auth *http.BasicAuth } -func (s *GitProviderService) GetRepository(ctx context.Context, gp GitProvider, url string) (gitprovider.OrgRepository, error) { - c, err := getGitProviderClient(gp) +func (s *GitProviderService) GetRepository(ctx context.Context, gp GitProvider, url string) (*git.Repository, error) { + provider, err := getGitProviderClient(s.log, gp) if err != nil { return nil, fmt.Errorf("unable to get a git provider client for %q: %w", gp.Type, err) } - var ref *gitprovider.OrgRepositoryRef - if gp.Type == string(gitproviders.GitProviderGitHub) || gp.Type == string(gitproviders.GitProviderGitLab) { - ref, err = gitprovider.ParseOrgRepositoryURL(url) - if err != nil { - return nil, fmt.Errorf("unable to parse repository URL %q: %w", url, err) - } - ref.Domain = addSchemeToDomain(ref.Domain) - ref = WithCombinedSubOrgs(*ref) - } else if gp.Type == string(gitproviders.GitProviderBitBucketServer) { - // The ParseOrgRepositoryURL function used for other providers - // fails to parse BitBucket Server URLs correctly - re := regexp.MustCompile(`://(?P[^/]+)/(.+/)?(?P[^/]+)/(?P[^/]+)\.git`) - match := re.FindStringSubmatch(url) - result := make(map[string]string) - for i, name := range re.SubexpNames() { - if i != 0 && name != "" { - result[name] = match[i] - } - } - if len(result) != 3 { - return nil, fmt.Errorf("unable to parse repository URL %q using regex %q", url, re.String()) - } - - orgRef := &gitprovider.OrganizationRef{ - Domain: result["host"], - Organization: result["key"], - } - ref = &gitprovider.OrgRepositoryRef{ - OrganizationRef: *orgRef, - RepositoryName: result["repo"], - } - ref.SetKey(result["key"]) - ref.Domain = addSchemeToDomain(ref.Domain) - } else { - return nil, fmt.Errorf("unsupported git provider") - } - - var repo gitprovider.OrgRepository - err = retry.OnError(DefaultBackoff, - func(err error) bool { return errors.Is(err, gitprovider.ErrNotFound) }, - func() error { - var err error - repo, err = c.OrgRepositories().Get(ctx, *ref) - if err != nil { - s.log.Info("Retrying getting the repository") - return err - } - return nil - }) - if err != nil { - return nil, fmt.Errorf("unable to get repository %q: %w, (client domain: %s)", url, err, c.SupportedDomain()) - } - - return repo, nil + return provider.GetRepository(ctx, url) } // WithCombinedSubOrgs combines the subgroups into the organization field of the reference @@ -217,39 +152,22 @@ func WithCombinedSubOrgs(ref gitprovider.OrgRepositoryRef) *gitprovider.OrgRepos } // GetTreeList retrieves list of tree files from gitprovider given the sha/branch -func (s *GitProviderService) GetTreeList(ctx context.Context, gp GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { - repo, err := s.GetRepository(ctx, gp, repoUrl) - if err != nil { - return nil, err - } - - treePaths, err := repo.Trees().List(ctx, sha, path, recursive) +func (s *GitProviderService) GetTreeList(ctx context.Context, gp GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*git.TreeEntry, error) { + provider, err := getGitProviderClient(s.log, gp) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to get a git provider client for %q: %w", gp.Type, err) } - return treePaths, nil + return provider.GetTreeList(ctx, repoUrl, sha, path) } -func (s *GitProviderService) ListPullRequests(ctx context.Context, gp GitProvider, repoURL string) ([]gitprovider.PullRequest, error) { - repo, err := s.GetRepository(ctx, gp, repoURL) - if err != nil { - return nil, fmt.Errorf("unable to get repo: %w", err) - } - - prs, err := repo.PullRequests().List(ctx) +func (s *GitProviderService) ListPullRequests(ctx context.Context, gp GitProvider, repoURL string) ([]*git.PullRequest, error) { + provider, err := getGitProviderClient(s.log, gp) if err != nil { - return nil, fmt.Errorf("unable to list pull requests for %q: %w", repoURL, err) + return nil, fmt.Errorf("unable to get a git provider client for %q: %w", gp.Type, err) } - return prs, nil -} - -type writeFilesToBranchRequest struct { - Repository gitprovider.OrgRepository - HeadBranch string - BaseBranch string - Commits []Commit + return provider.ListPullRequests(ctx, repoURL) } type Commit struct { @@ -257,110 +175,44 @@ type Commit struct { Files []gitprovider.CommitFile } -func (s *GitProviderService) writeFilesToBranch(ctx context.Context, req writeFilesToBranchRequest) error { - - var commits []gitprovider.Commit - err := retry.OnError(DefaultBackoff, - func(err error) bool { - // Ideally this should return true only for 404 (gitprovider.ErrNotFound) and 409 errors - return true - }, func() error { - var err error - commits, err = req.Repository.Commits().ListPage(ctx, req.BaseBranch, 1, 1) - if err != nil { - s.log.Info("Retrying getting the repository") - return err - } - return nil - }) - if err != nil { - return fmt.Errorf("unable to get most recent commit for branch %q: %w", req.BaseBranch, err) - } - if len(commits) == 0 { - return fmt.Errorf("no commits were found for branch %q, is the repository empty?", req.BaseBranch) - } - - err = req.Repository.Branches().Create(ctx, req.HeadBranch, commits[0].Get().Sha) - if err != nil { - return fmt.Errorf("unable to create new branch %q from commit %q in branch %q: %w", req.HeadBranch, commits[0].Get().Sha, req.BaseBranch, err) - } - - // Loop through all the commits and write the files. - for _, c := range req.Commits { - commit, err := req.Repository.Commits().Create(ctx, req.HeadBranch, c.CommitMessage, c.Files) - if err != nil { - return fmt.Errorf("unable to commit changes to %q: %w", req.HeadBranch, err) - } - s.log.WithValues("sha", commit.Get().Sha, "branch", req.HeadBranch).Info("Files committed") - } - - return nil -} - -type createPullRequestRequest struct { - Repository gitprovider.OrgRepository - HeadBranch string - BaseBranch string - Title string - Description string -} - -type createPullRequestResponse struct { - WebURL string -} - -func (s *GitProviderService) createPullRequest(ctx context.Context, req createPullRequestRequest) (*createPullRequestResponse, error) { - pr, err := req.Repository.PullRequests().Create(ctx, req.Title, req.HeadBranch, req.BaseBranch, req.Description) - if err != nil { - return nil, fmt.Errorf("unable to create new pull request for branch %q: %w", req.HeadBranch, err) - } - s.log.WithValues("pullRequestURL", pr.Get().WebURL).Info("Created pull request") - - return &createPullRequestResponse{ - WebURL: pr.Get().WebURL, - }, nil -} - -func getGitProviderClient(gpi GitProvider) (gitprovider.Client, error) { - var client gitprovider.Client - var err error - +func getGitProviderClient(log logr.Logger, gpi GitProvider) (git.Provider, error) { // quirk of ggp hostname := addSchemeToDomain(gpi.Hostname) + providerFactory := git.NewFactory(log) + providerOpts := []git.ProviderWithFn{} + switch gpi.Type { - case "github": + case git.GitHubProviderName: + providerOpts = append(providerOpts, git.WithOAuth2Token(gpi.Token)) + if gpi.Hostname != "github.com" { - client, err = github.NewClient( - gitprovider.WithOAuth2Token(gpi.Token), - gitprovider.WithDomain(hostname), - ) - } else { - client, err = github.NewClient( - gitprovider.WithOAuth2Token(gpi.Token), - ) - } - if err != nil { - return nil, err + providerOpts = append(providerOpts, git.WithDomain(hostname)) } - case "gitlab": + case git.GitLabProviderName: + providerOpts = append(providerOpts, git.WithConditionalRequests()) + providerOpts = append(providerOpts, git.WithToken(gpi.Token, gpi.TokenType)) + if gpi.Hostname != "gitlab.com" { - client, err = gitlab.NewClient(gpi.Token, gpi.TokenType, gitprovider.WithDomain(hostname), gitprovider.WithConditionalRequests(true)) - } else { - client, err = gitlab.NewClient(gpi.Token, gpi.TokenType, gitprovider.WithConditionalRequests(true)) - } - if err != nil { - return nil, err - } - case "bitbucket-server": - client, err = stash.NewStashClient("git", gpi.Token, gitprovider.WithDomain(hostname), gitprovider.WithConditionalRequests(true)) - if err != nil { - return nil, err + providerOpts = append(providerOpts, git.WithDomain(hostname)) } + case git.BitBucketServerProviderName: + providerOpts = append(providerOpts, git.WithUsername("git")) + providerOpts = append(providerOpts, git.WithToken(gpi.TokenType, gpi.Token)) + providerOpts = append(providerOpts, git.WithDomain(hostname)) + providerOpts = append(providerOpts, git.WithConditionalRequests()) + case git.AzureGitOpsProviderName: + providerOpts = append(providerOpts, git.WithToken(gpi.TokenType, gpi.Token)) default: return nil, fmt.Errorf("the Git provider %q is not supported", gpi.Type) } - return client, err + + provider, err := providerFactory.Create( + gpi.Type, + providerOpts..., + ) + + return provider, err } func GetGitProviderUrl(giturl string) (string, error) { @@ -392,11 +244,11 @@ func addSchemeToDomain(domain string) string { func (s *GitProviderService) getUpdatedFiles( ctx context.Context, - reqFiles []gitprovider.CommitFile, + reqFiles []git.CommitFile, gp GitProvider, repoURL, - branch string) ([]gitprovider.CommitFile, error) { - var updatedFiles []gitprovider.CommitFile + branch string) ([]git.CommitFile, error) { + var updatedFiles []git.CommitFile for _, file := range reqFiles { // if file content is empty, then it's a delete operation @@ -405,7 +257,7 @@ func (s *GitProviderService) getUpdatedFiles( continue } - dirPath, _ := filepath.Split(*file.Path) + dirPath, _ := filepath.Split(file.Path) treeEntries, err := s.GetTreeList(ctx, gp, repoURL, branch, dirPath, true) if err != nil { @@ -413,9 +265,9 @@ func (s *GitProviderService) getUpdatedFiles( } for _, treeEntry := range treeEntries { - if treeEntry.Path == *file.Path { - updatedFiles = append(updatedFiles, gitprovider.CommitFile{ - Path: &treeEntry.Path, + if treeEntry.Path == file.Path { + updatedFiles = append(updatedFiles, git.CommitFile{ + Path: treeEntry.Path, Content: nil, }) } diff --git a/cmd/clusters-service/pkg/git/gitfakes/git_fake.go b/cmd/clusters-service/pkg/git/gitfakes/git_fake.go index e79eaa1c18..9249ecccaa 100644 --- a/cmd/clusters-service/pkg/git/gitfakes/git_fake.go +++ b/cmd/clusters-service/pkg/git/gitfakes/git_fake.go @@ -5,12 +5,12 @@ import ( "sort" "strings" - "github.com/fluxcd/go-git-providers/gitprovider" - "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" + csgit "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" capiv1_protos "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" ) -func NewFakeGitProvider(url string, repo *git.GitRepo, err error, originalFilesPaths []string, prs []gitprovider.PullRequest) git.Provider { +func NewFakeGitProvider(url string, repo *csgit.GitRepo, err error, originalFilesPaths []string, prs []*git.PullRequest) csgit.Provider { return &FakeGitProvider{ url: url, repo: repo, @@ -22,22 +22,22 @@ func NewFakeGitProvider(url string, repo *git.GitRepo, err error, originalFilesP type FakeGitProvider struct { url string - repo *git.GitRepo + repo *csgit.GitRepo err error - CommittedFiles []gitprovider.CommitFile + CommittedFiles []git.CommitFile OriginalFiles []string - pullRequests []gitprovider.PullRequest + pullRequests []*git.PullRequest } -func (p *FakeGitProvider) WriteFilesToBranchAndCreatePullRequest(ctx context.Context, req git.WriteFilesToBranchAndCreatePullRequestRequest) (*git.WriteFilesToBranchAndCreatePullRequestResponse, error) { +func (p *FakeGitProvider) WriteFilesToBranchAndCreatePullRequest(ctx context.Context, req csgit.WriteFilesToBranchAndCreatePullRequestRequest) (*csgit.WriteFilesToBranchAndCreatePullRequestResponse, error) { if p.err != nil { return nil, p.err } p.CommittedFiles = append(p.CommittedFiles, req.Files...) - return &git.WriteFilesToBranchAndCreatePullRequestResponse{WebURL: p.url}, nil + return &csgit.WriteFilesToBranchAndCreatePullRequestResponse{WebURL: p.url}, nil } -func (p *FakeGitProvider) GetRepository(ctx context.Context, gp git.GitProvider, url string) (gitprovider.OrgRepository, error) { +func (p *FakeGitProvider) GetRepository(ctx context.Context, gp csgit.GitProvider, url string) (*git.Repository, error) { if p.err != nil { return nil, p.err } @@ -53,7 +53,7 @@ func (p *FakeGitProvider) GetCommittedFiles() []*capiv1_protos.CommitFile { } committedFiles = append(committedFiles, &capiv1_protos.CommitFile{ - Path: *f.Path, + Path: f.Path, Content: content, }) } @@ -61,22 +61,20 @@ func (p *FakeGitProvider) GetCommittedFiles() []*capiv1_protos.CommitFile { return committedFiles } -func (p *FakeGitProvider) GetTreeList(ctx context.Context, gp git.GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { +func (p *FakeGitProvider) GetTreeList(ctx context.Context, gp csgit.GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*git.TreeEntry, error) { if p.err != nil { return nil, p.err } - var treeEntries []*gitprovider.TreeEntry + var treeEntries []*git.TreeEntry for _, filePath := range p.OriginalFiles { if path == "" || (path != "" && strings.HasPrefix(filePath, path)) { - treeEntries = append(treeEntries, &gitprovider.TreeEntry{ - Path: filePath, - Mode: "", - Type: "", - Size: 0, - SHA: "", - Content: "", - URL: "", + treeEntries = append(treeEntries, &git.TreeEntry{ + Path: filePath, + Type: "", + Size: 0, + SHA: "", + Link: "", }) } @@ -84,45 +82,19 @@ func (p *FakeGitProvider) GetTreeList(ctx context.Context, gp git.GitProvider, r return treeEntries, nil } -func (p *FakeGitProvider) ListPullRequests(ctx context.Context, gp git.GitProvider, url string) ([]gitprovider.PullRequest, error) { +func (p *FakeGitProvider) ListPullRequests(ctx context.Context, gp csgit.GitProvider, url string) ([]*git.PullRequest, error) { return p.pullRequests, nil } -func NewPullRequest(id int, title string, description string, url string, merged bool, sourceBranch string) gitprovider.PullRequest { - return &pullrequest{ - id: id, - title: title, - description: description, - url: url, - merged: merged, - sourceBranch: sourceBranch, +func NewPullRequest(id int, title string, description string, url string, merged bool, sourceBranch string) *git.PullRequest { + return &git.PullRequest{ + Title: title, + Description: description, + Link: url, + Merged: merged, } } -type pullrequest struct { - id int - title string - description string - url string - merged bool - sourceBranch string -} - -func (pr *pullrequest) Get() gitprovider.PullRequestInfo { - return gitprovider.PullRequestInfo{ - Title: pr.title, - Description: pr.description, - WebURL: pr.url, - Number: pr.id, - Merged: pr.merged, - SourceBranch: pr.sourceBranch, - } -} - -func (pr *pullrequest) APIObject() interface{} { - return &pr -} - func sortCommitFiles(files []*capiv1_protos.CommitFile) { sort.Slice(files, func(i, j int) bool { return files[i].Path < files[j].Path diff --git a/cmd/clusters-service/pkg/server/automations.go b/cmd/clusters-service/pkg/server/automations.go index 66460a8c9d..5302c3589c 100644 --- a/cmd/clusters-service/pkg/server/automations.go +++ b/cmd/clusters-service/pkg/server/automations.go @@ -23,8 +23,9 @@ import ( esv1beta1 "github.com/external-secrets/external-secrets/apis/externalsecrets/v1beta1" pacv2beta2 "github.com/weaveworks/policy-agent/api/v2beta2" - "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" + csgit "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" capiv1_proto "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ) @@ -36,9 +37,9 @@ type GetAutomations struct { PolicyConfigsFiles []*capiv1_proto.CommitFile } -func toGitCommitFile(file *capiv1_proto.CommitFile) gitprovider.CommitFile { - return gitprovider.CommitFile{ - Path: &file.Path, +func toGitCommitFile(file *capiv1_proto.CommitFile) git.CommitFile { + return git.CommitFile{ + Path: file.Path, Content: &file.Content, } } @@ -67,7 +68,7 @@ func (s *server) CreateAutomationsPullRequest(ctx context.Context, msg *capiv1_p baseBranch = msg.BaseBranch } - var files []gitprovider.CommitFile + var files []git.CommitFile if len(automations.KustomizationFiles) > 0 { for _, f := range automations.KustomizationFiles { @@ -117,7 +118,7 @@ func (s *server) CreateAutomationsPullRequest(ctx context.Context, msg *capiv1_p return nil, grpcStatus.Errorf(codes.Unauthenticated, "failed to access repo %s: %s", repositoryURL, err) } - res, err := s.provider.WriteFilesToBranchAndCreatePullRequest(ctx, git.WriteFilesToBranchAndCreatePullRequestRequest{ + res, err := s.provider.WriteFilesToBranchAndCreatePullRequest(ctx, csgit.WriteFilesToBranchAndCreatePullRequestRequest{ GitProvider: *gp, RepositoryURL: repositoryURL, ReposistoryAPIURL: msg.RepositoryApiUrl, @@ -193,7 +194,7 @@ func getAutomations(ctx context.Context, client client.Client, ca []*capiv1_prot } kustomizationFiles = append(kustomizationFiles, &capiv1_proto.CommitFile{ - Path: *kustomization.Path, + Path: kustomization.Path, Content: *kustomization.Content, }) } diff --git a/cmd/clusters-service/pkg/server/clusters.go b/cmd/clusters-service/pkg/server/clusters.go index 0b57acf461..321ef44d08 100644 --- a/cmd/clusters-service/pkg/server/clusters.go +++ b/cmd/clusters-service/pkg/server/clusters.go @@ -13,13 +13,13 @@ import ( "strings" "time" - "github.com/fluxcd/go-git-providers/gitprovider" helmv2 "github.com/fluxcd/helm-controller/api/v2beta1" kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/mkmik/multierror" "github.com/spf13/viper" gitopsv1alpha1 "github.com/weaveworks/cluster-controller/api/v1alpha1" + csgit "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" "github.com/weaveworks/weave-gitops-enterprise/pkg/helm" "github.com/weaveworks/weave-gitops-enterprise/pkg/services/profiles" "github.com/weaveworks/weave-gitops/pkg/gitproviders" @@ -37,9 +37,11 @@ import ( capiv1 "github.com/weaveworks/templates-controller/apis/capi/v1alpha2" templatesv1 "github.com/weaveworks/templates-controller/apis/core" "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/charts" - "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" + + // "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" capiv1_proto "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos" "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/templates" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/validation" ) @@ -214,7 +216,7 @@ func (s *server) CreatePullRequest(ctx context.Context, msg *capiv1_proto.Create return nil, err } - files := []gitprovider.CommitFile{} + files := []git.CommitFile{} files = append(files, git_files.RenderedTemplate...) files = append(files, git_files.ProfileFiles...) files = append(files, git_files.KustomizationFiles...) @@ -243,12 +245,13 @@ func (s *server) CreatePullRequest(ctx context.Context, msg *capiv1_proto.Create if msg.CommitMessage == "" { msg.CommitMessage = "Add Cluster Manifests" } + _, err = s.provider.GetRepository(ctx, *gp, repositoryURL) if err != nil { return nil, grpcStatus.Errorf(codes.Unauthenticated, "failed to access repo %s: %s", repositoryURL, err) } - res, err := s.provider.WriteFilesToBranchAndCreatePullRequest(ctx, git.WriteFilesToBranchAndCreatePullRequestRequest{ + res, err := s.provider.WriteFilesToBranchAndCreatePullRequest(ctx, csgit.WriteFilesToBranchAndCreatePullRequestRequest{ GitProvider: *gp, RepositoryURL: repositoryURL, ReposistoryAPIURL: msg.RepositoryApiUrl, @@ -289,7 +292,7 @@ func (s *server) DeleteClustersPullRequest(ctx context.Context, msg *capiv1_prot baseBranch = msg.BaseBranch } - var filesList []gitprovider.CommitFile + var filesList []git.CommitFile if len(msg.ClusterNamespacedNames) > 0 { for _, clusterNamespacedName := range msg.ClusterNamespacedNames { // Files in manifest path @@ -298,8 +301,8 @@ func (s *server) DeleteClustersPullRequest(ctx context.Context, msg *capiv1_prot clusterNamespacedName.Name, getClusterNamespace(clusterNamespacedName.Namespace)), ) - filesList = append(filesList, gitprovider.CommitFile{ - Path: &path, + filesList = append(filesList, git.CommitFile{ + Path: path, Content: nil, }) @@ -315,8 +318,8 @@ func (s *server) DeleteClustersPullRequest(ctx context.Context, msg *capiv1_prot } for _, treeEntry := range treeEntries { - filesList = append(filesList, gitprovider.CommitFile{ - Path: &treeEntry.Path, + filesList = append(filesList, git.CommitFile{ + Path: treeEntry.Path, Content: nil, }) } @@ -327,8 +330,8 @@ func (s *server) DeleteClustersPullRequest(ctx context.Context, msg *capiv1_prot path := getClusterManifestPath( createNamespacedName(clusterName, getClusterNamespace("")), ) - filesList = append(filesList, gitprovider.CommitFile{ - Path: &path, + filesList = append(filesList, git.CommitFile{ + Path: path, Content: nil, }) @@ -344,8 +347,8 @@ func (s *server) DeleteClustersPullRequest(ctx context.Context, msg *capiv1_prot } for _, treeEntry := range treeEntries { - filesList = append(filesList, gitprovider.CommitFile{ - Path: &treeEntry.Path, + filesList = append(filesList, git.CommitFile{ + Path: treeEntry.Path, Content: nil, }) } @@ -370,7 +373,7 @@ func (s *server) DeleteClustersPullRequest(ctx context.Context, msg *capiv1_prot return nil, grpcStatus.Errorf(codes.Unauthenticated, "failed to get repo %s: %s", repositoryURL, err) } - res, err := s.provider.WriteFilesToBranchAndCreatePullRequest(ctx, git.WriteFilesToBranchAndCreatePullRequestRequest{ + res, err := s.provider.WriteFilesToBranchAndCreatePullRequest(ctx, csgit.WriteFilesToBranchAndCreatePullRequestRequest{ GitProvider: *gp, RepositoryURL: repositoryURL, ReposistoryAPIURL: msg.RepositoryApiUrl, @@ -515,7 +518,7 @@ func getToken(ctx context.Context) (string, string, error) { return providerToken.AccessToken, "oauth2", nil } -func getCommonKustomization(cluster types.NamespacedName) (*gitprovider.CommitFile, error) { +func getCommonKustomization(cluster types.NamespacedName) (*git.CommitFile, error) { commonKustomizationPath := getCommonKustomizationPath(cluster) commonKustomization := createKustomizationObject(&capiv1_proto.Kustomization{ Metadata: &capiv1_proto.Metadata{ @@ -538,15 +541,15 @@ func getCommonKustomization(cluster types.NamespacedName) (*gitprovider.CommitFi return nil, fmt.Errorf("error marshalling common kustomization, %w", err) } commonKustomizationString := string(b) - file := &gitprovider.CommitFile{ - Path: &commonKustomizationPath, + file := &git.CommitFile{ + Path: commonKustomizationPath, Content: &commonKustomizationString, } return file, nil } -func getGitProvider(ctx context.Context, repositoryURL string) (*git.GitProvider, error) { +func getGitProvider(ctx context.Context, repositoryURL string) (*csgit.GitProvider, error) { token, tokenType, err := getToken(ctx) if err != nil { return nil, err @@ -568,7 +571,7 @@ func getGitProvider(ctx context.Context, repositoryURL string) (*git.GitProvider repoHostname = repoURL.URL().Host } - return &git.GitProvider{ + return &csgit.GitProvider{ Type: repoType, TokenType: tokenType, Token: token, @@ -620,7 +623,7 @@ func createProfileYAML(helmRepo *sourcev1.HelmRepository, helmReleases []*helmv2 // profileValues is what the client will provide to the API. // It may have > 1 and its values parameter may be empty. // Assumption: each profile should have a values.yaml that we can treat as the default. -func generateProfileFiles(ctx context.Context, tmpl templatesv1.Template, cluster types.NamespacedName, helmRepo *sourcev1.HelmRepository, args generateProfileFilesParams) ([]gitprovider.CommitFile, error) { +func generateProfileFiles(ctx context.Context, tmpl templatesv1.Template, cluster types.NamespacedName, helmRepo *sourcev1.HelmRepository, args generateProfileFilesParams) ([]git.CommitFile, error) { tmplProcessor, err := templates.NewProcessorForTemplate(tmpl) if err != nil { return nil, err @@ -730,7 +733,7 @@ func generateProfileFiles(ctx context.Context, tmpl templatesv1.Template, cluste return nil, err } - commitFiles := []gitprovider.CommitFile{} + commitFiles := []git.CommitFile{} // For each path, we join the content of relative profiles and add to a commit file for path := range profilesByPath { profileContent := string(bytes.Join(profilesByPath[path], []byte("---\n"))) @@ -739,15 +742,15 @@ func generateProfileFiles(ctx context.Context, tmpl templatesv1.Template, cluste return nil, fmt.Errorf("cannot render path %s: %w", path, err) } renderedPathStr := string(renderedPath) - file := &gitprovider.CommitFile{ - Path: &renderedPathStr, + file := git.CommitFile{ + Path: renderedPathStr, Content: &profileContent, } - commitFiles = append(commitFiles, *file) + commitFiles = append(commitFiles, file) } sort.Slice(commitFiles, func(i, j int) bool { - return *commitFiles[i].Path < *commitFiles[j].Path + return commitFiles[i].Path < commitFiles[j].Path }) return commitFiles, nil @@ -971,12 +974,12 @@ func generateKustomizationFile( isControlPlane bool, cluster types.NamespacedName, kustomization *capiv1_proto.Kustomization, - filePath string) (gitprovider.CommitFile, error) { + filePath string) (git.CommitFile, error) { kustomizationYAML := createKustomizationObject(kustomization) b, err := yaml.Marshal(kustomizationYAML) if err != nil { - return gitprovider.CommitFile{}, fmt.Errorf("error marshalling %s kustomization, %w", kustomization.Metadata.Name, err) + return git.CommitFile{}, fmt.Errorf("error marshalling %s kustomization, %w", kustomization.Metadata.Name, err) } k := createNamespacedName(kustomization.Metadata.Name, kustomization.Metadata.Namespace) @@ -988,8 +991,8 @@ func generateKustomizationFile( kustomizationContent := string(b) - file := &gitprovider.CommitFile{ - Path: &kustomizationPath, + file := &git.CommitFile{ + Path: kustomizationPath, Content: &kustomizationContent, } @@ -1086,11 +1089,11 @@ func createNamespacedName(name, namespace string) types.NamespacedName { } } -// Get list of gitprovider.CommitFile objects of files that should be deleted with empty content +// Get list of git.CommitFile objects of files that should be deleted with empty content // Kustomizations and Profiles removed during an edit are added to the deleted list // Old files with changed paths are added to the deleted list -func getDeletedFiles(prevFiles *GetFilesReturn, newFiles *GetFilesReturn) []gitprovider.CommitFile { - deletedFiles := []gitprovider.CommitFile{} +func getDeletedFiles(prevFiles *GetFilesReturn, newFiles *GetFilesReturn) []git.CommitFile { + deletedFiles := []git.CommitFile{} removedKustomizations := getMissingFiles(prevFiles.KustomizationFiles, newFiles.KustomizationFiles) removedProfiles := getMissingFiles(prevFiles.ProfileFiles, newFiles.ProfileFiles) diff --git a/cmd/clusters-service/pkg/server/clusters_test.go b/cmd/clusters-service/pkg/server/clusters_test.go index fe926f158d..427d994e4e 100644 --- a/cmd/clusters-service/pkg/server/clusters_test.go +++ b/cmd/clusters-service/pkg/server/clusters_test.go @@ -17,10 +17,10 @@ import ( "time" capiv1 "github.com/weaveworks/templates-controller/apis/capi/v1alpha2" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" "github.com/weaveworks/weave-gitops-enterprise/pkg/helm" "github.com/weaveworks/weave-gitops/pkg/server/auth" - "github.com/fluxcd/go-git-providers/gitprovider" "github.com/fluxcd/pkg/apis/meta" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/google/go-cmp/cmp" @@ -42,7 +42,7 @@ import ( templatesv1 "github.com/weaveworks/templates-controller/apis/core" gapiv1 "github.com/weaveworks/templates-controller/apis/gitops/v1alpha2" "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/charts" - "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" + csgit "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git" "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git/gitfakes" capiv1_protos "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos" ) @@ -397,7 +397,7 @@ func TestCreatePullRequest(t *testing.T) { testCases := []struct { name string clusterState []runtime.Object - provider git.Provider + provider csgit.Provider pruneEnvVar string req *capiv1_protos.CreatePullRequestRequest expected string @@ -1561,7 +1561,7 @@ func TestDeleteClustersPullRequest(t *testing.T) { testCases := []struct { name string - provider git.Provider + provider csgit.Provider req *capiv1_protos.DeleteClustersPullRequestRequest CommittedFiles []*capiv1_protos.CommitFile expected string @@ -1673,8 +1673,8 @@ func TestDeleteClustersPullRequest(t *testing.T) { if fakeGitProvider.OriginalFiles != nil { // sort CommittedFiles and OriginalFiles for comparison sort.Slice(fakeGitProvider.CommittedFiles[:], func(i, j int) bool { - currFile := *fakeGitProvider.CommittedFiles[i].Path - nextFile := *fakeGitProvider.CommittedFiles[j].Path + currFile := fakeGitProvider.CommittedFiles[i].Path + nextFile := fakeGitProvider.CommittedFiles[j].Path return currFile < nextFile }) sort.Strings(fakeGitProvider.OriginalFiles) @@ -1683,8 +1683,8 @@ func TestDeleteClustersPullRequest(t *testing.T) { t.Fatalf("number of committed files (%d) do not match number of expected files (%d)\n", len(fakeGitProvider.CommittedFiles), len(fakeGitProvider.OriginalFiles)) } for ind, committedFile := range fakeGitProvider.CommittedFiles { - if *committedFile.Path != fakeGitProvider.OriginalFiles[ind] { - t.Fatalf("committed file does not match expected file\n%v\n%v", *committedFile.Path, fakeGitProvider.OriginalFiles[ind]) + if committedFile.Path != fakeGitProvider.OriginalFiles[ind] { + t.Fatalf("committed file does not match expected file\n%v\n%v", committedFile.Path, fakeGitProvider.OriginalFiles[ind]) } } @@ -1811,7 +1811,7 @@ func TestGenerateProfileFiles(t *testing.T) { }, ) assert.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -1887,7 +1887,7 @@ func TestGenerateProfileFiles_without_editable_flag(t *testing.T) { }, ) require.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -1962,7 +1962,7 @@ func TestGenerateProfileFiles_with_editable_flag(t *testing.T) { }, ) require.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "management/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -2038,7 +2038,7 @@ func TestGenerateProfileFiles_with_templates(t *testing.T) { }, ) assert.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -2115,7 +2115,7 @@ func TestGenerateProfileFilesWithLayers(t *testing.T) { }, ) assert.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -2221,7 +2221,7 @@ func TestGenerateProfileFiles_with_text_templates(t *testing.T) { }, ) assert.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -2294,7 +2294,7 @@ func TestGenerateProfileFiles_with_required_profiles_only(t *testing.T) { }, ) require.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -2377,7 +2377,7 @@ func TestGenerateProfileFiles_reading_layer_from_cache(t *testing.T) { }, ) assert.NoError(t, err) - expected := []gitprovider.CommitFile{ + expected := []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", `apiVersion: source.toolkit.fluxcd.io/v1beta2 @@ -2539,13 +2539,13 @@ status: {} var tests = []struct { name string template *gapiv1.GitOpsTemplate - expected []gitprovider.CommitFile + expected []git.CommitFile params map[string]string }{ { name: "generate profile paths", template: makeTestTemplateWithPaths(templatesv1.RenderTypeEnvsubst, "", "", ""), - expected: []gitprovider.CommitFile{ + expected: []git.CommitFile{ makeCommitFile( "ns-foo/cluster-foo/profiles.yaml", concatYaml(expectedHelmRelease, expectedBarHelmRelease, expectedFooHelmRelease), @@ -2555,7 +2555,7 @@ status: {} { name: "generate profile paths with custom paths", template: makeTestTemplateWithPaths(templatesv1.RenderTypeEnvsubst, "repo.yaml", "foo.yaml", "bar.yaml"), - expected: []gitprovider.CommitFile{ + expected: []git.CommitFile{ makeCommitFile( "bar.yaml", concatYaml(expectedBarHelmRelease), @@ -2576,7 +2576,7 @@ status: {} params: map[string]string{ "BAR_PATH": "special-bar.yaml", }, - expected: []gitprovider.CommitFile{ + expected: []git.CommitFile{ makeCommitFile( "foo.yaml", concatYaml(expectedFooHelmRelease), @@ -2597,7 +2597,7 @@ status: {} params: map[string]string{ "BAR_PATH": "special-bar.yaml", }, - expected: []gitprovider.CommitFile{ + expected: []git.CommitFile{ makeCommitFile( "foo.yaml", concatYaml(expectedFooHelmRelease), @@ -2659,11 +2659,11 @@ func makeTestHelmRepositoryTemplate(base string) *sourcev1.HelmRepository { }) } -func makeCommitFile(path, content string) gitprovider.CommitFile { +func makeCommitFile(path, content string) git.CommitFile { p := path c := content - return gitprovider.CommitFile{ - Path: &p, + return git.CommitFile{ + Path: p, Content: &c, } } diff --git a/cmd/clusters-service/pkg/server/helpers.go b/cmd/clusters-service/pkg/server/helpers.go index b2ef225cc4..06d0f77a2c 100644 --- a/cmd/clusters-service/pkg/server/helpers.go +++ b/cmd/clusters-service/pkg/server/helpers.go @@ -4,13 +4,13 @@ import ( "fmt" "strings" - "github.com/fluxcd/go-git-providers/gitprovider" "github.com/spf13/viper" "k8s.io/apimachinery/pkg/util/sets" capiv1 "github.com/weaveworks/templates-controller/apis/capi/v1alpha2" apitemplates "github.com/weaveworks/templates-controller/apis/core" "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/templates" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" ) func renderTemplateWithValues(t apitemplates.Template, name, namespace string, values map[string]string) ([]templates.RenderedTemplate, error) { @@ -97,25 +97,25 @@ func isCAPITemplate(t apitemplates.Template) bool { return t.GetObjectKind().GroupVersionKind().Kind == capiv1.Kind } -func filePaths(files []gitprovider.CommitFile) []string { +func filePaths(files []git.CommitFile) []string { names := []string{} for _, f := range files { - names = append(names, *f.Path) + names = append(names, f.Path) } return names } // Check if there are files in originalFiles that are missing from extraFiles and returns them -func getMissingFiles(originalFiles []gitprovider.CommitFile, extraFiles []gitprovider.CommitFile) []gitprovider.CommitFile { +func getMissingFiles(originalFiles []git.CommitFile, extraFiles []git.CommitFile) []git.CommitFile { originalFilePaths := filePaths(originalFiles) extraFilePaths := filePaths(extraFiles) diffPaths := sets.NewString(originalFilePaths...).Difference(sets.NewString(extraFilePaths...)).List() - removedFilenames := []gitprovider.CommitFile{} + removedFilenames := []git.CommitFile{} for i := range diffPaths { - removedFilenames = append(removedFilenames, gitprovider.CommitFile{ - Path: &diffPaths[i], + removedFilenames = append(removedFilenames, git.CommitFile{ + Path: diffPaths[i], Content: nil, }) } diff --git a/cmd/clusters-service/pkg/server/helpers_test.go b/cmd/clusters-service/pkg/server/helpers_test.go index 0e9ecc88a1..b02ac4bce3 100644 --- a/cmd/clusters-service/pkg/server/helpers_test.go +++ b/cmd/clusters-service/pkg/server/helpers_test.go @@ -5,10 +5,10 @@ import ( "sort" "testing" - "github.com/fluxcd/go-git-providers/gitprovider" capiv1 "github.com/weaveworks/templates-controller/apis/capi/v1alpha2" templatesv1 "github.com/weaveworks/templates-controller/apis/core" apitemplate "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/templates" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" ) func TestGetProvider(t *testing.T) { @@ -252,104 +252,104 @@ func TestGetMissingFiles(t *testing.T) { tests := []struct { name string - originalFiles []gitprovider.CommitFile - extraFiles []gitprovider.CommitFile - expected []gitprovider.CommitFile + originalFiles []git.CommitFile + extraFiles []git.CommitFile + expected []git.CommitFile }{ { name: "original files with empty extra files", - originalFiles: []gitprovider.CommitFile{ + originalFiles: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr("dummy content"), }, { - Path: strPtr("testdata/cluster-template-1.yaml"), + Path: "testdata/cluster-template-1.yaml", Content: strPtr("dummy content"), }, }, - extraFiles: []gitprovider.CommitFile{}, - expected: []gitprovider.CommitFile{ + extraFiles: []git.CommitFile{}, + expected: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr(""), }, { - Path: strPtr("testdata/cluster-template-1.yaml"), + Path: "testdata/cluster-template-1.yaml", Content: strPtr(""), }, }, }, { name: "original files with files not in extra files", - originalFiles: []gitprovider.CommitFile{ + originalFiles: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr("dummy content"), }, { - Path: strPtr("testdata/cluster-template-1.yaml"), + Path: "testdata/cluster-template-1.yaml", Content: strPtr("dummy content"), }, }, - extraFiles: []gitprovider.CommitFile{ + extraFiles: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template-2.yaml"), + Path: "testdata/cluster-template-2.yaml", Content: strPtr("dummy content"), }, }, - expected: []gitprovider.CommitFile{ + expected: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr(""), }, { - Path: strPtr("testdata/cluster-template-1.yaml"), + Path: "testdata/cluster-template-1.yaml", Content: strPtr(""), }, }, }, { name: "no original files", - originalFiles: []gitprovider.CommitFile{}, - extraFiles: []gitprovider.CommitFile{ + originalFiles: []git.CommitFile{}, + extraFiles: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr("dummy content"), }, { - Path: strPtr("testdata/cluster-template-1.yaml"), + Path: "testdata/cluster-template-1.yaml", Content: strPtr("dummy content"), }, }, - expected: []gitprovider.CommitFile{}, + expected: []git.CommitFile{}, }, { name: "original with 1 file and extra with 2 files not in original", - originalFiles: []gitprovider.CommitFile{ + originalFiles: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr("dummy content"), }, }, - extraFiles: []gitprovider.CommitFile{ + extraFiles: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template-2.yaml"), + Path: "testdata/cluster-template-2.yaml", Content: strPtr("dummy content"), }, { - Path: strPtr("testdata/cluster-template-1.yaml"), + Path: "testdata/cluster-template-1.yaml", Content: strPtr(""), }, }, - expected: []gitprovider.CommitFile{ + expected: []git.CommitFile{ { - Path: strPtr("testdata/cluster-template.yaml"), + Path: "testdata/cluster-template.yaml", Content: strPtr(""), }, }, @@ -359,7 +359,7 @@ func TestGetMissingFiles(t *testing.T) { t.Run(tt.name, func(t *testing.T) { sortFiles(tt.expected) - var expectedPaths []*string + var expectedPaths []string expectedContents := make([]*string, len(tt.expected)) for i := range tt.expected { expectedPaths = append(expectedPaths, tt.expected[i].Path) @@ -367,7 +367,7 @@ func TestGetMissingFiles(t *testing.T) { difference := getMissingFiles(tt.originalFiles, tt.extraFiles) sortFiles(difference) - var differencePaths []*string + var differencePaths []string var differenceContents []*string for i := range difference { differencePaths = append(differencePaths, difference[i].Path) @@ -388,9 +388,9 @@ func TestGetMissingFiles(t *testing.T) { } -func sortFiles(files []gitprovider.CommitFile) { +func sortFiles(files []git.CommitFile) { sort.Slice(files, func(i, j int) bool { - return *files[i].Path < *files[j].Path + return files[i].Path < files[j].Path }) } diff --git a/cmd/clusters-service/pkg/server/templates.go b/cmd/clusters-service/pkg/server/templates.go index 6f6b14d46f..ef15a9aadf 100644 --- a/cmd/clusters-service/pkg/server/templates.go +++ b/cmd/clusters-service/pkg/server/templates.go @@ -8,7 +8,6 @@ import ( "sort" "strings" - "github.com/fluxcd/go-git-providers/gitprovider" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/go-logr/logr" "github.com/spf13/viper" @@ -19,6 +18,7 @@ import ( capiv1_proto "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos" "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/templates" "github.com/weaveworks/weave-gitops-enterprise/pkg/estimation" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" "github.com/weaveworks/weave-gitops-enterprise/pkg/helm" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -37,11 +37,11 @@ type GetFilesRequest struct { } type GetFilesReturn struct { - RenderedTemplate []gitprovider.CommitFile - ProfileFiles []gitprovider.CommitFile - KustomizationFiles []gitprovider.CommitFile + RenderedTemplate []git.CommitFile + ProfileFiles []git.CommitFile + KustomizationFiles []git.CommitFile CostEstimate *capiv1_proto.CostEstimate - ExternalSecretsFiles []gitprovider.CommitFile + ExternalSecretsFiles []git.CommitFile } func (s *server) getTemplate(ctx context.Context, name, namespace, templateKind string) (templatesv1.Template, error) { @@ -205,11 +205,11 @@ func (s *server) ListTemplateProfiles(ctx context.Context, msg *capiv1_proto.Lis return &capiv1_proto.ListTemplateProfilesResponse{Profiles: profiles, Objects: t.Objects}, err } -func toCommitFileProtos(file []gitprovider.CommitFile) []*capiv1_proto.CommitFile { +func toCommitFileProtos(file []git.CommitFile) []*capiv1_proto.CommitFile { var files []*capiv1_proto.CommitFile for _, f := range file { files = append(files, &capiv1_proto.CommitFile{ - Path: *f.Path, + Path: f.Path, Content: *f.Content, }) } @@ -284,7 +284,7 @@ func GetFiles( return nil, fmt.Errorf("failed to render template with parameter values: %w", err) } - var files []gitprovider.CommitFile + var files []git.CommitFile for _, renderedTemplate := range renderedTemplates { tmplWithValues := renderedTemplate.Data if createRequestMessage != nil { @@ -317,8 +317,8 @@ func GetFiles( } content := string(bytes.Join(tmplWithValues, []byte("\n---\n"))) - files = append(files, gitprovider.CommitFile{ - Path: &path, + files = append(files, git.CommitFile{ + Path: path, Content: &content, }) } @@ -326,9 +326,9 @@ func GetFiles( // if this feature is not enabled the Nil estimator will be invoked returning a nil estimate costEstimate := getCostEstimate(ctx, estimator, renderedTemplates) - var profileFiles []gitprovider.CommitFile - var kustomizationFiles []gitprovider.CommitFile - var externalSecretFiles []gitprovider.CommitFile + var profileFiles []git.CommitFile + var kustomizationFiles []git.CommitFile + var externalSecretFiles []git.CommitFile if shouldAddCommonBases(tmpl) { cluster, err := getCluster(resourcesNamespace, msg) @@ -395,8 +395,8 @@ func GetFiles( if err != nil { return nil, err } - kustomizationFiles = append(kustomizationFiles, gitprovider.CommitFile{ - Path: namespace.Path, + kustomizationFiles = append(kustomizationFiles, git.CommitFile{ + Path: *namespace.Path, Content: namespace.Content, }) } diff --git a/cmd/clusters-service/pkg/server/templates_test.go b/cmd/clusters-service/pkg/server/templates_test.go index 77c461cd8a..e16fdaaf7d 100644 --- a/cmd/clusters-service/pkg/server/templates_test.go +++ b/cmd/clusters-service/pkg/server/templates_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "testing" - "github.com/fluxcd/go-git-providers/gitprovider" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/go-logr/logr" "github.com/google/go-cmp/cmp" @@ -25,6 +24,7 @@ import ( gapiv1 "github.com/weaveworks/templates-controller/apis/gitops/v1alpha2" capiv1_protos "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos" "github.com/weaveworks/weave-gitops-enterprise/pkg/estimation" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" "github.com/weaveworks/weave-gitops-enterprise/pkg/helm" "github.com/weaveworks/weave-gitops/pkg/server/auth" ) @@ -1439,9 +1439,9 @@ status: {} expectedContent := simpleTemplate(t, expectedTemplateContent, struct{ URL string }{URL: ts.URL}) expected := &GetFilesReturn{ RenderedTemplate: nil, - ProfileFiles: []gitprovider.CommitFile{ + ProfileFiles: []git.CommitFile{ { - Path: &expectedPath, + Path: expectedPath, Content: &expectedContent, }, }, diff --git a/cmd/gitops/app/create/templates/cmd.go b/cmd/gitops/app/create/templates/cmd.go index 8c3fb020da..236e1119c5 100644 --- a/cmd/gitops/app/create/templates/cmd.go +++ b/cmd/gitops/app/create/templates/cmd.go @@ -11,7 +11,6 @@ import ( "time" securejoin "github.com/cyphar/filepath-securejoin" - "github.com/fluxcd/go-git-providers/gitprovider" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" "github.com/go-logr/logr" "github.com/spf13/cobra" @@ -22,6 +21,7 @@ import ( "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/templates" clitemplates "github.com/weaveworks/weave-gitops-enterprise/cmd/gitops/pkg/templates" "github.com/weaveworks/weave-gitops-enterprise/pkg/estimation" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" "github.com/weaveworks/weave-gitops-enterprise/pkg/helm" "github.com/weaveworks/weave-gitops/core/logger" "helm.sh/helm/v3/pkg/cli" @@ -109,7 +109,7 @@ func initializeConfig(cmd *cobra.Command) error { } func templatesCmdRunE() func(*cobra.Command, []string) error { - return func(cmd *cobra.Command, args []string) error { + return func(_ *cobra.Command, args []string) error { log, err := logger.New(logger.DefaultLogLevel, true) if err != nil { return fmt.Errorf("failed to create logger: %w", err) @@ -162,7 +162,7 @@ func templatesCmdRunE() func(*cobra.Command, []string) error { if config.Export { renderedTemplate := "" for _, file := range files { - renderedTemplate += fmt.Sprintf("# path: %s\n---\n%s\n\n", *file.Path, *file.Content) + renderedTemplate += fmt.Sprintf("# path: %s\n---\n%s\n\n", file.Path, *file.Content) } err := export(renderedTemplate, os.Stdout) @@ -175,9 +175,9 @@ func templatesCmdRunE() func(*cobra.Command, []string) error { if config.OutputDir != "" { for _, res := range files { - filePath, err := securejoin.SecureJoin(config.OutputDir, *res.Path) + filePath, err := securejoin.SecureJoin(config.OutputDir, res.Path) if err != nil { - return fmt.Errorf("failed to join %s to %s: %w", config.OutputDir, *res.Path, err) + return fmt.Errorf("failed to join %s to %s: %w", config.OutputDir, res.Path, err) } directoryPath := filepath.Dir(filePath) @@ -239,7 +239,7 @@ func export(template string, out io.Writer) error { return nil } -func generateFilesLocally(tmpl *gapiv1.GitOpsTemplate, params map[string]string, helmRepoName string, profiles []*capiv1_proto.ProfileValues, settings *cli.EnvSettings, log logr.Logger) ([]gitprovider.CommitFile, error) { +func generateFilesLocally(tmpl *gapiv1.GitOpsTemplate, params map[string]string, helmRepoName string, profiles []*capiv1_proto.ProfileValues, settings *cli.EnvSettings, log logr.Logger) ([]git.CommitFile, error) { templateHasRequiredProfiles, err := templates.TemplateHasRequiredProfiles(tmpl) if err != nil { return nil, fmt.Errorf("failed to check if template has required profiles: %w", err) diff --git a/cmd/gitops/app/create/templates/cmd_test.go b/cmd/gitops/app/create/templates/cmd_test.go index 7f3dbb600a..8430040d40 100644 --- a/cmd/gitops/app/create/templates/cmd_test.go +++ b/cmd/gitops/app/create/templates/cmd_test.go @@ -156,7 +156,7 @@ func TestGenerateFilesLocally(t *testing.T) { actualFilenames := []string{} for _, file := range files { - actualFilenames = append(actualFilenames, *file.Path) + actualFilenames = append(actualFilenames, file.Path) } if diff := cmp.Diff(expectedFiles, actualFilenames); diff != "" { @@ -186,7 +186,7 @@ func TestGenerateFilesLocallyWithCharts(t *testing.T) { actualFilenames := []string{} for _, file := range files { - actualFilenames = append(actualFilenames, *file.Path) + actualFilenames = append(actualFilenames, file.Path) } assert.Contains(t, *files[1].Content, "version: 0.0.8") diff --git a/go.mod b/go.mod index 079adcaac5..e22efa6e7c 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/sclevine/agouti v3.0.0+incompatible github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.6.1 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.2 github.com/weaveworks/weave-gitops v0.18.0 github.com/weaveworks/weave-gitops-enterprise-credentials v0.0.2 github.com/weaveworks/weave-gitops-enterprise/common v0.0.0 @@ -42,6 +42,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.16.16 github.com/aws/aws-sdk-go-v2/config v1.17.8 github.com/aws/aws-sdk-go-v2/service/pricing v1.17.1 + github.com/drone/go-scm v1.29.1 github.com/external-secrets/external-secrets v0.7.0 github.com/fluxcd/flagger v1.21.0 github.com/fluxcd/go-git-providers v0.14.0 @@ -58,6 +59,7 @@ require ( github.com/grpc-ecosystem/protoc-gen-grpc-gateway-ts v1.1.1 github.com/hashicorp/go-multierror v1.1.1 github.com/jarcoal/httpmock v1.0.8 + github.com/jenkins-x/go-scm v1.13.7 github.com/loft-sh/vcluster v0.12.0 github.com/mattn/go-sqlite3 v1.14.16 github.com/maxbrunsfeld/counterfeiter/v6 v6.5.0 @@ -74,7 +76,7 @@ require ( github.com/weaveworks/templates-controller v0.1.3 github.com/xanzy/go-gitlab v0.78.0 golang.org/x/crypto v0.3.0 - golang.org/x/oauth2 v0.3.0 + golang.org/x/oauth2 v0.5.0 google.golang.org/genproto v0.0.0-20221027153422-115e99e71e1c google.golang.org/grpc v1.51.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 @@ -115,16 +117,12 @@ require ( github.com/fluxcd/image-automation-controller/api v0.27.0 // indirect github.com/fluxcd/image-reflector-controller/api v0.23.0 // indirect github.com/fluxcd/notification-controller/api v0.29.1 // indirect - github.com/fluxcd/pkg/http/fetch v0.3.0 // indirect - github.com/fluxcd/pkg/tar v0.2.0 // indirect github.com/fvbommel/sortorder v1.0.1 // indirect - github.com/gitops-tools/pkg v0.1.0 // indirect github.com/gobuffalo/flect v0.3.0 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-github/v49 v49.1.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf // indirect - github.com/jenkins-x/go-scm v1.13.4 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect @@ -240,9 +238,9 @@ require ( github.com/go-asset/generics v0.0.0-20220317100214-d5f632c68060 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.0 // indirect - github.com/go-git/go-billy/v5 v5.4.0 // indirect + github.com/go-git/go-billy/v5 v5.4.1 // indirect github.com/go-git/go-git/v5 v5.5.2 - github.com/go-gorp/gorp/v3 v3.0.2 // indirect + github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect @@ -277,7 +275,7 @@ require ( github.com/klauspost/compress v1.15.11 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect - github.com/lib/pq v1.10.6 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 5e14a192b1..3243c36b09 100644 --- a/go.sum +++ b/go.sum @@ -355,6 +355,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNE github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 h1:7QPwrLT79GlD5sizHf27aoY2RTvw62mO6x7mxkScNk0= github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46/go.mod h1:esf2rsHFNlZlxsqsZDojNBcnNs5REqIvRrWRHqX0vEU= +github.com/drone/go-scm v1.29.1 h1:9ZrokpIAX+BANYGahqJp6vJqgYiD5c8v5A4iVk+oyZo= +github.com/drone/go-scm v1.29.1/go.mod h1:DFIJJjhMj0TSXPz+0ni4nyZ9gtTtC40Vh/TGRugtyWw= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -419,15 +421,11 @@ github.com/fluxcd/pkg/apis/kustomize v0.7.0 h1:X2htBmJ91nGYv4d93gin665MFWKNGiNwU github.com/fluxcd/pkg/apis/kustomize v0.7.0/go.mod h1:Mu+KdktsEKWA4l/33CZdY5lB4hz51mqfcLzBZSwAqVg= github.com/fluxcd/pkg/apis/meta v0.18.0 h1:s0LeulWcQ4DxVX6805vgDTxlA6bAYk+Lq1QHSnNdqLM= github.com/fluxcd/pkg/apis/meta v0.18.0/go.mod h1:pYvXRFi1UKNNrGR34jw3uqOnMXw9X6dTkML8j5Z7tis= -github.com/fluxcd/pkg/http/fetch v0.3.0 h1:/mLj0IzTx+GhR09etzMJsBoNQ0qeOx9cSdeUgRB+oqM= -github.com/fluxcd/pkg/http/fetch v0.3.0/go.mod h1:dHTDYIeL0ZAQ9mHM6ZS4VProxho+Atm73MHJ55yj0Sg= github.com/fluxcd/pkg/kustomize v0.10.0 h1:EG5MbYrLtxeCiZxeFUgvyBhFZaXnKfeqqpg7O+J7o3s= github.com/fluxcd/pkg/runtime v0.25.0 h1:Lk5WrKDJKsayymLnnSCY/Zn77/mrlIf+skYz64suoww= github.com/fluxcd/pkg/runtime v0.25.0/go.mod h1:I2T+HWVNzX0cxm9TgH+SVNHTwqlmEDiSke43JXsq9iY= github.com/fluxcd/pkg/ssa v0.22.0 h1:HvJTuiYLZMxCjin7bAqBgnc2RjSqEfYrMbV5yINoM64= github.com/fluxcd/pkg/ssa v0.22.0/go.mod h1:QND0ZNOQ5EzFxoNKfjUxE9J46AbRK3WKF8YkURwbVg0= -github.com/fluxcd/pkg/tar v0.2.0 h1:HEUHgONQYsJGeZZ4x6h5nQU9Aox1I4T3bOp1faWTqf8= -github.com/fluxcd/pkg/tar v0.2.0/go.mod h1:w0/TOC7kwBJhnSJn7TCABkc/I7ib1f2Yz6vOsbLBnhw= github.com/fluxcd/pkg/untar v0.2.0 h1:sJXU+FbJcNUb2ffLJNjeR3hwt3X2loVpOMlCUjyFw6E= github.com/fluxcd/pkg/untar v0.2.0/go.mod h1:33AyoWaPpjX/xXpczcfhQh2AkB63TFwiR2YwROtv23E= github.com/fluxcd/source-controller/api v0.33.0 h1:NZYU3+MNf9puyrTbBa7AJbBDlN7tmt0uw8lyye++5fE= @@ -448,8 +446,6 @@ github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSy github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gitops-tools/pkg v0.1.0 h1:atKTGUjGEEvkSX+HGCzI76rHRB84+nr77ll8kyJY3Nk= -github.com/gitops-tools/pkg v0.1.0/go.mod h1:c+ZMQS6qVn3+HfJ3Hl04ARo7zxD30ackJnV60UlLC5s= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-asset/generics v0.0.0-20220317100214-d5f632c68060 h1:NFuFDuTToogr9xQDZ7w7Z6TuiziNzTEkL49BW5yrH9M= @@ -462,8 +458,9 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= -github.com/go-git/go-billy/v5 v5.4.0 h1:Vaw7LaSTRJOUric7pe4vnzBSgyuf2KrLsu2Y4ZpQBDE= github.com/go-git/go-billy/v5 v5.4.0/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= +github.com/go-git/go-billy/v5 v5.4.1 h1:Uwp5tDRkPr+l/TnbHOQzp+tmJfLceOlbVucgpTz8ix4= +github.com/go-git/go-billy/v5 v5.4.1/go.mod h1:vjbugF6Fz7JIflbVpl1hJsGjSHNltrSw45YK/ukIvQg= github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlKnXHuzrfjTQ= github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw= @@ -471,8 +468,9 @@ github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4B github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gorp/gorp/v3 v3.0.2 h1:ULqJXIekoqMx29FI5ekXXFoH1dT2Vc8UhnRzBg+Emz4= github.com/go-gorp/gorp/v3 v3.0.2/go.mod h1:BJ3q1ejpV8cVALtcXvXaXyTOlMmJhWDxTmncaR6rwBY= +github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= +github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -725,6 +723,9 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.2 h1:ERKrevVTnCw3Wu4I3mtR15QU3gt github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.2/go.mod h1:chrfS3YoLAlKTRE5cFWvCbt8uGAjshktT4PveTUpsFQ= github.com/grpc-ecosystem/protoc-gen-grpc-gateway-ts v1.1.1 h1:arwcA21RFb6T+jDlx8webgjG5mxVOVUk/L4/sxRLisY= github.com/grpc-ecosystem/protoc-gen-grpc-gateway-ts v1.1.1/go.mod h1:eirF820XKj2zKB4hFfdsOqlAVq45EpWH2WzR2WxBua4= +github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= +github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -800,8 +801,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a h1:d4+I1YEKVmWZrgkt6jpXBnLgV2ZjO0YxEtLDdfIZfH4= github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a/go.mod h1:Zi/ZFkEqFHTm7qkjyNJjaWH4LQA9LQhGJyF0lTYGpxw= -github.com/jenkins-x/go-scm v1.13.4 h1:uljVQaOKh/K+quGli0a10YLUPr4x1ouDORa/sZWhvok= -github.com/jenkins-x/go-scm v1.13.4/go.mod h1:avNeJtE6yddfN4JQXGXug8YoKaAsWWWpw0wO84Iezkc= +github.com/jenkins-x/go-scm v1.13.7 h1:UfrPt8kmj3Et7S3frZT6Fqf2pIVPCVX4tmf11F9FfWk= +github.com/jenkins-x/go-scm v1.13.7/go.mod h1:peylsiJ/pKuAW/p6/jUsSR64k8I3lWsjySmAqA8nxA8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/jgautheron/goconst v1.5.1/go.mod h1:aAosetZ5zaeC/2EfMeRswtxUFBpe2Hr7HzkgX4fanO4= @@ -877,7 +878,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -899,8 +900,8 @@ github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/loft-sh/loft-util v0.0.9-alpha h1:kGcyTQWxWHWy7bbjhS8Hsq/JRdlSztAU++anV6P+sqk= @@ -1069,6 +1070,7 @@ github.com/mwitkow/go-proto-validators v0.0.0-20180403085117-0950a7990007/go.mod github.com/mwitkow/go-proto-validators v0.2.0/go.mod h1:ZfA1hW+UH/2ZHOWvQ3HnQaU0DtnpXu850MZiy+YUgcc= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354/go.mod h1:KSVJerMDfblTH7p5MZaTt+8zaT2iEk3AkVb9PQdZuE8= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -1153,8 +1155,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/polyfloyd/go-errorlint v0.0.0-20210722154253-910bb7978349/go.mod h1:wi9BfjxjF/bwiZ701TzmfKu6UKC357IOAtNr0Td0Lvw= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8C/OCy0zs9906d/VUru+bqg= github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= +github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -1324,8 +1326,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -1365,8 +1367,6 @@ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7Fw github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/weaveworks/cluster-controller v1.4.1 h1:vnbHPI+0GkcSOp986WvO5D6grb2IMgDvveKblmGm3vk= github.com/weaveworks/cluster-controller v1.4.1/go.mod h1:NrgkuiyejE2nizsiXNeqaNWfVhpvG2SlAiWHnxKIS4U= -github.com/weaveworks/gitopssets-controller v0.4.0 h1:Ozhajnszl5B/Gjsm5B69jkNteuTHeXwg2ZqwOSsIukA= -github.com/weaveworks/gitopssets-controller v0.4.0/go.mod h1:FF4+ie44tnxSgFv+juV3b4f8ScGRxMckJGqwbYU7mfk= github.com/weaveworks/gitopssets-controller v0.5.2-0.20230306134151-79578e0349c2 h1:NOQ0n63xgh7SrRrmOwK0XavGqNymTXLN+ny0K8PhNcM= github.com/weaveworks/gitopssets-controller v0.5.2-0.20230306134151-79578e0349c2/go.mod h1:FF4+ie44tnxSgFv+juV3b4f8ScGRxMckJGqwbYU7mfk= github.com/weaveworks/go-checkpoint v0.0.0-20220223124739-fd9899e2b4f2 h1:EWUmjQdHzmBimPxGIus5JOvNPu+tWxOTC+Q4w9fJOok= @@ -1421,7 +1421,6 @@ github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMzt github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1649,8 +1648,8 @@ golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= -golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2097,6 +2096,7 @@ gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qS gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/pkg/git/azure.go b/pkg/git/azure.go new file mode 100644 index 0000000000..52a2de816e --- /dev/null +++ b/pkg/git/azure.go @@ -0,0 +1,198 @@ +package git + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + + "github.com/drone/go-scm/scm/driver/azure" + "github.com/go-logr/logr" + "github.com/jenkins-x/go-scm/scm" + "github.com/jenkins-x/go-scm/scm/factory" +) + +const ( + AzureGitOpsProviderName string = "azure-gitops" +) + +// AzureGitOpsProvider is used to interact with the AzureGitOps API. +type AzureGitOpsProvider struct { + log logr.Logger + client *scm.Client +} + +func NewAzureGitOpsProvider(log logr.Logger) (Provider, error) { + return &AzureGitOpsProvider{ + log: log, + }, nil +} + +func (p *AzureGitOpsProvider) Setup(opts ProviderOption) error { + if opts.Token == "" { + return fmt.Errorf("missing required option: Token") + } + + if opts.Hostname == "" { + opts.Hostname = "dev.azure.com" + } + + // ggpOpts := []gitprovider.ClientOption{ + // gitprovider.WithDomain(opts.Hostname), + // } + + var err error + + p.client, err = factory.NewClient("azure", fmt.Sprintf("https://%s", opts.Hostname), opts.Token) + + return err +} + +func (p *AzureGitOpsProvider) GetRepository(ctx context.Context, repoURL string) (*Repository, error) { + u, err := url.Parse(repoURL) + if err != nil { + return nil, fmt.Errorf("unbale to parse url %q: %w", repoURL, err) + } + + dscm := jenkinsSCM{} + + repo, err := dscm.GetRepository(ctx, p.log, p.client, u) + if err != nil { + return nil, err + } + + return &Repository{ + Domain: u.Host, + Org: repo.Namespace, + Name: repo.Name, + }, nil +} + +func (p *AzureGitOpsProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { + jsmc := jenkinsSCM{} + + u, err := url.Parse(input.RepositoryURL) + if err != nil { + return nil, fmt.Errorf("unbale to parse url %q: %w", input.RepositoryURL, err) + } + + repo, err := jsmc.GetRepository(ctx, p.log, p.client, u) + if err != nil { + return nil, err + } + + headCommit, _ := jsmc.GetCurrentCommitOfBranch(ctx, p.client, repo, input.Head, "") + + if headCommit == "" { + headCommit, err = jsmc.GetCurrentCommitOfBranch(ctx, p.client, repo, input.Base, "") + if err != nil { + return nil, err + } + _, _, err = p.client.Git.CreateRef(ctx, repo.FullName, input.Head, headCommit) + if err != nil { + return nil, fmt.Errorf("failed to create new branch: %w", err) + } + } + + // Note: commits has to be split into a separate update and add commits, Azure + // does not support updates and additions in the same commit, at least it gave + // me back an error when I tried. + for _, commit := range input.Commits { + request := jsmc.CommitFilesRequest( + headCommit, + input.RepositoryURL, + input.Head, + commit.CommitMessage, + commit.Files, + ) + + if _, err := p.sendRawRequest(ctx, request); err != nil { + return nil, err + } + + // Fetch the new head commit + headCommit, _ = jsmc.GetCurrentCommitOfBranch(ctx, p.client, repo, input.Head, "") + } + + pr, _, err := p.client.PullRequests.Create(ctx, repo.FullName, &scm.PullRequestInput{ + Title: input.Title, + Head: input.Head, + Base: input.Base, + Body: input.Body, + }) + if err != nil { + return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) + } + + return &PullRequest{Link: pr.Link}, nil +} + +func (p *AzureGitOpsProvider) GetTreeList(ctx context.Context, repoUrl string, sha string, path string) ([]*TreeEntry, error) { + url, err := GetGitProviderUrl(repoUrl) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + files := []*TreeEntry{} + + fileList, _, err := p.client.Contents.List(ctx, url, path, sha) + if err != nil { + return nil, err + } + + for _, file := range fileList { + files = append(files, &TreeEntry{ + Name: file.Name, + Path: file.Path, + Type: file.Type, + Size: file.Size, + SHA: file.Sha, + Link: file.Link, + }) + } + + return files, nil +} + +func (p *AzureGitOpsProvider) ListPullRequests(ctx context.Context, repoURL string) ([]*PullRequest, error) { + url, err := GetGitProviderUrl(repoURL) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + prList, _, err := p.client.PullRequests.List(ctx, url, &scm.PullRequestListOptions{}) + if err != nil { + return nil, err + } + + prs := []*PullRequest{} + for _, pr := range prList { + prs = append(prs, &PullRequest{ + Title: pr.Title, + Description: pr.Body, + Link: pr.Link, + Merged: pr.Merged, + }) + } + + return prs, nil +} + +func (p *AzureGitOpsProvider) sendRawRequest(ctx context.Context, request *scm.Request) (*scm.Response, error) { + resp, err := p.client.Do(ctx, request) + if err != nil { + return nil, fmt.Errorf("failed to commit files: %w", err) + } + + defer resp.Body.Close() + + if resp.Status > 300 { + err := new(azure.Error) + + _ = json.NewDecoder(resp.Body).Decode(err) + + return resp, err + } + + return resp, nil +} diff --git a/pkg/git/bitbucket_server.go b/pkg/git/bitbucket_server.go index c69c163d4d..855dbb6fe9 100644 --- a/pkg/git/bitbucket_server.go +++ b/pkg/git/bitbucket_server.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/fluxcd/go-git-providers/gitprovider" + "github.com/fluxcd/go-git-providers/stash" "github.com/go-logr/logr" ) @@ -11,7 +13,8 @@ const BitBucketServerProviderName string = "bitbucket-server" // BitBucketServerProvider is used to interact with the BitBucket Server (stash) API. type BitBucketServerProvider struct { - log logr.Logger + log logr.Logger + client gitprovider.Client } func NewBitBucketServerProvider(log logr.Logger) (Provider, error) { @@ -20,40 +23,76 @@ func NewBitBucketServerProvider(log logr.Logger) (Provider, error) { }, nil } +func (p *BitBucketServerProvider) Setup(opts ProviderOption) error { + if opts.Username == "" { + opts.Username = "git" + } + + if opts.Token == "" { + return fmt.Errorf("missing required option: Token") + } + + if opts.Hostname == "" { + return fmt.Errorf("missing required option: Hostname") + } + + ggpOpts := []gitprovider.ClientOption{ + gitprovider.WithDomain(opts.Hostname), + gitprovider.WithConditionalRequests(opts.ConditionalRequests), + } + + if opts.Hostname != "" { + ggpOpts = append(ggpOpts, gitprovider.WithDomain(opts.Hostname)) + } + + var err error + + p.client, err = stash.NewStashClient(opts.Username, opts.Token, ggpOpts...) + + return err +} + +func (p *BitBucketServerProvider) GetRepository(ctx context.Context, url string) (*Repository, error) { + ggp := goGitProvider{} + + repo, err := ggp.GetBitbucketRepository(ctx, p.log, p.client, url) + if err != nil { + return nil, err + } + + return &Repository{ + Org: repo.Repository().GetIdentity(), + Name: repo.Repository().GetRepository(), + }, nil +} + func (p *BitBucketServerProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { - repoURL, err := GetGitProviderUrl(input.RepositoryURL) + url, err := GetGitProviderUrl(input.RepositoryURL) if err != nil { return nil, fmt.Errorf("unable to get git provider url: %w", err) } - repo, err := GetRepository(ctx, p.log, input.GitProvider, repoURL) + ggp := goGitProvider{} + + repo, err := ggp.GetBitbucketRepository(ctx, p.log, p.client, url) if err != nil { - return nil, fmt.Errorf("unable to get repo: %w", err) + return nil, err } - // Add the files to be created to the map of changes. - commits := []Commit{} - commits = append(commits, Commit{ - CommitMessage: input.CommitMessage, - Files: input.Files, - }) - - if err := writeFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ - Repository: repo, + if err := ggp.WriteFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ HeadBranch: input.Head, BaseBranch: input.Base, - Commits: commits, - }); err != nil { + Commits: input.Commits, + }, repo); err != nil { return nil, fmt.Errorf("unable to write files to branch %q: %w", input.Head, err) } - res, err := createPullRequest(ctx, p.log, createPullRequestRequest{ - Repository: repo, + res, err := ggp.CreatePullRequest(ctx, p.log, createPullRequestRequest{ HeadBranch: input.Head, BaseBranch: input.Base, Title: input.Title, Description: input.Body, - }) + }, repo) if err != nil { return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) } @@ -62,3 +101,47 @@ func (p *BitBucketServerProvider) CreatePullRequest(ctx context.Context, input P Link: res.WebURL, }, nil } + +func (p *BitBucketServerProvider) GetTreeList(ctx context.Context, repoUrl string, sha string, path string) ([]*TreeEntry, error) { + url, err := GetGitProviderUrl(repoUrl) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) + if err != nil { + return nil, err + } + + files := []*TreeEntry{} + + treePaths, err := repo.Trees().List(ctx, sha, path, true) + if err != nil { + return nil, err + } + + for _, file := range treePaths { + files = append(files, &TreeEntry{ + Path: file.Path, + Type: file.Type, + Size: file.Size, + SHA: file.SHA, + Link: file.URL, + }) + } + + return files, nil +} + +func (p *BitBucketServerProvider) ListPullRequests(ctx context.Context, repoURL string) ([]*PullRequest, error) { + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, repoURL) + if err != nil { + return nil, err + } + + return ggp.ListPullRequests(ctx, repo) +} diff --git a/pkg/git/factory.go b/pkg/git/factory.go index 5ec07c85bf..eb17d6158c 100644 --- a/pkg/git/factory.go +++ b/pkg/git/factory.go @@ -20,15 +20,98 @@ func NewFactory(log logr.Logger) *ProviderFactory { } // Create creates and returns a new git provider. -func (f *ProviderFactory) Create(provider string) (Provider, error) { - switch provider { +func (f *ProviderFactory) Create(providerName string, opts ...ProviderWithFn) (Provider, error) { + var ( + provider Provider + err error + ) + + switch providerName { case GitHubProviderName: - return NewGitHubProvider(f.log) + provider, err = NewGitHubProvider(f.log) case GitLabProviderName: - return NewGitLabProvider(f.log) + provider, err = NewGitLabProvider(f.log) case BitBucketServerProviderName: - return NewBitBucketServerProvider(f.log) + provider, err = NewBitBucketServerProvider(f.log) + case AzureGitOpsProviderName: + provider, err = NewAzureGitOpsProvider(f.log) default: - return nil, fmt.Errorf("provider %q is not supported", provider) + return nil, fmt.Errorf("provider %q is not supported", providerName) + } + + if err != nil { + return nil, fmt.Errorf("unable to use provider %q: %w", providerName, err) + } + + option := ProviderOption{} + + for _, opt := range opts { + if err = opt(&option); err != nil { + return nil, fmt.Errorf("unable to gather options on provider %q: %w", provider, err) + } + } + + if err := provider.Setup(option); err != nil { + return nil, fmt.Errorf("unable to apply options on provider %q: %w", provider, err) + } + + return provider, nil +} + +type ProviderOption struct { + Hostname string + OAuth2Token string + TokenType string + Token string + Username string + ConditionalRequests bool +} + +type ProviderWithFn func(o *ProviderOption) error + +func WithDomain(domain string) ProviderWithFn { + return func(p *ProviderOption) error { + return nil + } +} + +func WithOAuth2Token(token string) ProviderWithFn { + return func(p *ProviderOption) error { + p.OAuth2Token = token + + return nil + } +} + +func WithToken(tokenType, token string) ProviderWithFn { + return func(p *ProviderOption) error { + p.TokenType = tokenType + p.Token = token + + return nil + } +} + +func WithConditionalRequests() ProviderWithFn { + return func(p *ProviderOption) error { + p.ConditionalRequests = true + + return nil + } +} + +func WithoutConditionalRequests() ProviderWithFn { + return func(p *ProviderOption) error { + p.ConditionalRequests = false + + return nil + } +} + +func WithUsername(username string) ProviderWithFn { + return func(p *ProviderOption) error { + p.Username = username + + return nil } } diff --git a/pkg/git/github.go b/pkg/git/github.go index a063f1279c..cf8fa9b4bf 100644 --- a/pkg/git/github.go +++ b/pkg/git/github.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/fluxcd/go-git-providers/github" + "github.com/fluxcd/go-git-providers/gitprovider" "github.com/go-logr/logr" ) @@ -13,7 +15,8 @@ const GitHubProviderName string = "github" // This implementation delegates most of the work to the // fluxcd/go-git-providers library. type GitHubProvider struct { - log logr.Logger + log logr.Logger + client gitprovider.Client } func NewGitHubProvider(log logr.Logger) (Provider, error) { @@ -22,40 +25,67 @@ func NewGitHubProvider(log logr.Logger) (Provider, error) { }, nil } +func (p *GitHubProvider) Setup(opts ProviderOption) error { + if opts.OAuth2Token == "" { + return fmt.Errorf("missing required option: OAuth2Token") + } + + ggpOpts := []gitprovider.ClientOption{ + gitprovider.WithOAuth2Token(opts.OAuth2Token), + } + + if opts.Hostname != "" { + ggpOpts = append(ggpOpts, gitprovider.WithDomain(opts.Hostname)) + } + + var err error + + p.client, err = github.NewClient(ggpOpts...) + + return err +} + +func (p *GitHubProvider) GetRepository(ctx context.Context, url string) (*Repository, error) { + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) + if err != nil { + return nil, err + } + + return &Repository{ + Org: repo.Repository().GetIdentity(), + Name: repo.Repository().GetRepository(), + }, nil +} + func (p *GitHubProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { - repoURL, err := GetGitProviderUrl(input.RepositoryURL) + url, err := GetGitProviderUrl(input.RepositoryURL) if err != nil { return nil, fmt.Errorf("unable to get git provider url: %w", err) } - repo, err := GetRepository(ctx, p.log, input.GitProvider, repoURL) + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) if err != nil { - return nil, fmt.Errorf("unable to get repo: %w", err) + return nil, err } - // Add the files to be created to the map of changes. - commits := []Commit{} - commits = append(commits, Commit{ - CommitMessage: input.CommitMessage, - Files: input.Files, - }) - - if err := writeFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ - Repository: repo, + if err := ggp.WriteFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ HeadBranch: input.Head, BaseBranch: input.Base, - Commits: commits, - }); err != nil { + Commits: input.Commits, + }, repo); err != nil { return nil, fmt.Errorf("unable to write files to branch %q: %w", input.Head, err) } - res, err := createPullRequest(ctx, p.log, createPullRequestRequest{ - Repository: repo, + res, err := ggp.CreatePullRequest(ctx, p.log, createPullRequestRequest{ HeadBranch: input.Head, BaseBranch: input.Base, Title: input.Title, Description: input.Body, - }) + }, repo) if err != nil { return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) } @@ -64,3 +94,47 @@ func (p *GitHubProvider) CreatePullRequest(ctx context.Context, input PullReques Link: res.WebURL, }, nil } + +func (p *GitHubProvider) GetTreeList(ctx context.Context, repoUrl string, sha string, path string) ([]*TreeEntry, error) { + url, err := GetGitProviderUrl(repoUrl) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) + if err != nil { + return nil, err + } + + files := []*TreeEntry{} + + treePaths, err := repo.Trees().List(ctx, sha, path, true) + if err != nil { + return nil, err + } + + for _, file := range treePaths { + files = append(files, &TreeEntry{ + Path: file.Path, + Type: file.Type, + Size: file.Size, + SHA: file.SHA, + Link: file.URL, + }) + } + + return files, nil +} + +func (p *GitHubProvider) ListPullRequests(ctx context.Context, repoURL string) ([]*PullRequest, error) { + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, repoURL) + if err != nil { + return nil, err + } + + return ggp.ListPullRequests(ctx, repo) +} diff --git a/pkg/git/gitlab.go b/pkg/git/gitlab.go index 3ffb1ae6ad..77f3a25b74 100644 --- a/pkg/git/gitlab.go +++ b/pkg/git/gitlab.go @@ -3,8 +3,8 @@ package git import ( "context" "fmt" - "path/filepath" + "github.com/fluxcd/go-git-providers/gitlab" "github.com/fluxcd/go-git-providers/gitprovider" "github.com/go-logr/logr" ) @@ -16,7 +16,8 @@ const ( // GitLabProvider is used to interact with the GitLab API. type GitLabProvider struct { - log logr.Logger + log logr.Logger + client gitprovider.Client } func NewGitLabProvider(log logr.Logger) (Provider, error) { @@ -25,55 +26,92 @@ func NewGitLabProvider(log logr.Logger) (Provider, error) { }, nil } +func (p *GitLabProvider) Setup(opts ProviderOption) error { + if opts.Token == "" { + return fmt.Errorf("missing required option: Token") + } + + ggpOpts := []gitprovider.ClientOption{ + gitprovider.WithConditionalRequests(opts.ConditionalRequests), + } + + if opts.Hostname != "" { + ggpOpts = append(ggpOpts, gitprovider.WithDomain(opts.Hostname)) + } + + var err error + + p.client, err = gitlab.NewClient(opts.Token, opts.TokenType, ggpOpts...) + + return err +} + +func (p *GitLabProvider) GetRepository(ctx context.Context, url string) (*Repository, error) { + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) + if err != nil { + return nil, err + } + + return &Repository{ + Org: repo.Repository().GetIdentity(), + Name: repo.Repository().GetRepository(), + }, nil +} + func (p *GitLabProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { - repoURL, err := GetGitProviderUrl(input.RepositoryURL) + url, err := GetGitProviderUrl(input.RepositoryURL) if err != nil { return nil, fmt.Errorf("unable to get git provider url: %w", err) } - repo, err := GetRepository(ctx, p.log, input.GitProvider, repoURL) + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) if err != nil { - return nil, fmt.Errorf("unable to get repo: %w", err) + return nil, err } - // Gitlab doesn't support createOrUpdate, so we need to check if the file exists - // and if it does, we need to create a commit to delete the file. - commits := []Commit{} - deletedFiles, err := getUpdatedFiles(ctx, input.Files, input.GitProvider, repoURL, input.Base) + files := []CommitFile{} + for _, commit := range input.Commits { + files = append(files, commit.Files...) + } + + updatedFiles, err := ggp.GetUpdatedFiles(ctx, files, p.client, input.RepositoryURL, input.Head) if err != nil { - return nil, fmt.Errorf("unable to get files from tree list: %w", err) + return nil, err } - // If there are files to delete, append them to the map of changes to be deleted. - if len(deletedFiles) > 0 { + commits := []Commit{} + + if len(updatedFiles) > 0 { + for idx := range updatedFiles { + updatedFiles[idx].Content = nil + } + commits = append(commits, Commit{ CommitMessage: deleteFilesCommitMessage, - Files: deletedFiles, + Files: updatedFiles, }) } - // Add the files to be created to the map of changes. - commits = append(commits, Commit{ - CommitMessage: input.CommitMessage, - Files: input.Files, - }) + commits = append(commits, input.Commits...) - if err := writeFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ - Repository: repo, + if err := ggp.WriteFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ HeadBranch: input.Head, BaseBranch: input.Base, Commits: commits, - }); err != nil { + }, repo); err != nil { return nil, fmt.Errorf("unable to write files to branch %q: %w", input.Head, err) } - res, err := createPullRequest(ctx, p.log, createPullRequestRequest{ - Repository: repo, + res, err := ggp.CreatePullRequest(ctx, p.log, createPullRequestRequest{ HeadBranch: input.Head, BaseBranch: input.Base, Title: input.Title, Description: input.Body, - }) + }, repo) if err != nil { return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) } @@ -83,52 +121,46 @@ func (p *GitLabProvider) CreatePullRequest(ctx context.Context, input PullReques }, nil } -func getUpdatedFiles( - ctx context.Context, - reqFiles []CommitFile, - gp GitProvider, - repoURL, - branch string) ([]CommitFile, error) { - var updatedFiles []CommitFile - - for _, file := range reqFiles { - // if file content is empty, then it's a delete operation - // so we don't need to check if the file exists - if file.Content == nil { - continue - } - - dirPath, _ := filepath.Split(file.Path) +func (p *GitLabProvider) GetTreeList(ctx context.Context, repoUrl string, sha string, path string) ([]*TreeEntry, error) { + url, err := GetGitProviderUrl(repoUrl) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } - treeEntries, err := GetTreeList(ctx, gp, repoURL, branch, dirPath, true) - if err != nil { - return nil, fmt.Errorf("error getting list of trees in repo: %s@%s: %w", repoURL, branch, err) - } + ggp := goGitProvider{} - for _, treeEntry := range treeEntries { - if treeEntry.Path == file.Path { - updatedFiles = append(updatedFiles, CommitFile{ - Path: treeEntry.Path, - Content: nil, - }) - } - } + repo, err := ggp.GetRepository(ctx, p.log, p.client, url) + if err != nil { + return nil, err } - return updatedFiles, nil -} + files := []*TreeEntry{} -// GetTreeList retrieves list of tree files from gitprovider given the sha/branch -func GetTreeList(ctx context.Context, gp GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { - repo, err := GetRepository(ctx, logr.Discard(), gp, repoUrl) + treePaths, err := repo.Trees().List(ctx, sha, path, true) if err != nil { return nil, err } - treePaths, err := repo.Trees().List(ctx, sha, path, recursive) + for _, file := range treePaths { + files = append(files, &TreeEntry{ + Path: file.Path, + Type: file.Type, + Size: file.Size, + SHA: file.SHA, + Link: file.URL, + }) + } + + return files, nil +} + +func (p *GitLabProvider) ListPullRequests(ctx context.Context, repoURL string) ([]*PullRequest, error) { + ggp := goGitProvider{} + + repo, err := ggp.GetRepository(ctx, p.log, p.client, repoURL) if err != nil { return nil, err } - return treePaths, nil + return ggp.ListPullRequests(ctx, repo) } diff --git a/pkg/git/gitprovider.go b/pkg/git/gitprovider.go new file mode 100644 index 0000000000..c1e11afdc5 --- /dev/null +++ b/pkg/git/gitprovider.go @@ -0,0 +1,217 @@ +package git + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "regexp" + + "github.com/fluxcd/go-git-providers/gitprovider" + "github.com/go-logr/logr" + "k8s.io/client-go/util/retry" +) + +type goGitProvider struct{} + +func (g goGitProvider) WriteFilesToBranch(ctx context.Context, log logr.Logger, req writeFilesToBranchRequest, repo gitprovider.OrgRepository) error { + + var commits []gitprovider.Commit + err := retry.OnError(DefaultBackoff, + func(err error) bool { + // Ideally this should return true only for 404 (gitprovider.ErrNotFound) and 409 errors + return true + }, func() error { + var err error + commits, err = repo.Commits().ListPage(ctx, req.BaseBranch, 1, 1) + if err != nil { + log.Info("Retrying getting the repository") + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("unable to get most recent commit for branch %q: %w", req.BaseBranch, err) + } + if len(commits) == 0 { + return fmt.Errorf("no commits were found for branch %q, is the repository empty?", req.BaseBranch) + } + + err = repo.Branches().Create(ctx, req.HeadBranch, commits[0].Get().Sha) + if err != nil { + return fmt.Errorf("unable to create new branch %q from commit %q in branch %q: %w", req.HeadBranch, commits[0].Get().Sha, req.BaseBranch, err) + } + + // Loop through all the commits and write the files. + for _, c := range req.Commits { + // Adapt for go-git-providers + adapted := make([]gitprovider.CommitFile, 0) + for _, f := range c.Files { + adapted = append(adapted, gitprovider.CommitFile{ + Path: &f.Path, + Content: f.Content, + }) + } + + commit, err := repo.Commits().Create(ctx, req.HeadBranch, c.CommitMessage, adapted) + if err != nil { + return fmt.Errorf("unable to commit changes to %q: %w", req.HeadBranch, err) + } + log.WithValues("sha", commit.Get().Sha, "branch", req.HeadBranch).Info("Files committed") + } + + return nil +} + +func (g goGitProvider) CreatePullRequest(ctx context.Context, log logr.Logger, req createPullRequestRequest, repo gitprovider.OrgRepository) (*createPullRequestResponse, error) { + pr, err := repo.PullRequests().Create(ctx, req.Title, req.HeadBranch, req.BaseBranch, req.Description) + if err != nil { + return nil, fmt.Errorf("unable to create new pull request for branch %q: %w", req.HeadBranch, err) + } + log.WithValues("pullRequestURL", pr.Get().WebURL).Info("Created pull request") + + return &createPullRequestResponse{ + WebURL: pr.Get().WebURL, + }, nil +} + +func (g goGitProvider) GetUpdatedFiles( + ctx context.Context, + reqFiles []CommitFile, + client gitprovider.Client, + repoURL, + branch string) ([]CommitFile, error) { + var updatedFiles []CommitFile + + for _, file := range reqFiles { + // if file content is empty, then it's a delete operation + // so we don't need to check if the file exists + if file.Content == nil { + continue + } + + dirPath, _ := filepath.Split(file.Path) + + treeEntries, err := g.GetTreeList(ctx, client, repoURL, branch, dirPath, true) + if err != nil { + return nil, fmt.Errorf("error getting list of trees in repo: %s@%s: %w", repoURL, branch, err) + } + + for _, treeEntry := range treeEntries { + if treeEntry.Path == file.Path { + updatedFiles = append(updatedFiles, CommitFile{ + Path: treeEntry.Path, + Content: nil, + }) + } + } + } + + return updatedFiles, nil +} + +// GetTreeList retrieves list of tree files from gitprovider given the sha/branch +func (g goGitProvider) GetTreeList(ctx context.Context, client gitprovider.Client, repoUrl string, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { + repo, err := g.GetRepository(ctx, logr.Discard(), client, repoUrl) + if err != nil { + return nil, err + } + + treePaths, err := repo.Trees().List(ctx, sha, path, recursive) + if err != nil { + return nil, err + } + + return treePaths, nil +} + +func (g goGitProvider) GetRepository(ctx context.Context, log logr.Logger, client gitprovider.Client, url string) (gitprovider.OrgRepository, error) { + ref, err := gitprovider.ParseOrgRepositoryURL(url) + if err != nil { + return nil, fmt.Errorf("unbale to parse url %q: %w", url, err) + } + + ref.Domain = addSchemeToDomain(ref.Domain) + ref = WithCombinedSubOrgs(*ref) + + var repo gitprovider.OrgRepository + err = retry.OnError(DefaultBackoff, + func(err error) bool { return errors.Is(err, gitprovider.ErrNotFound) }, + func() error { + var err error + repo, err = client.OrgRepositories().Get(ctx, *ref) + if err != nil { + log.Info("Retrying getting the repository") + return err + } + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("unable to get repository %q: %w, (client domain: %s)", url, err, client.SupportedDomain()) + } + + return repo, nil +} + +func (g goGitProvider) GetBitbucketRepository(ctx context.Context, log logr.Logger, client gitprovider.Client, url string) (gitprovider.OrgRepository, error) { + re := regexp.MustCompile(`://(?P[^/]+)/(.+/)?(?P[^/]+)/(?P[^/]+)\.git`) + match := re.FindStringSubmatch(url) + result := make(map[string]string) + for i, name := range re.SubexpNames() { + if i != 0 && name != "" { + result[name] = match[i] + } + } + if len(result) != 3 { + return nil, fmt.Errorf("unable to parse repository URL %q using regex %q", url, re.String()) + } + + orgRef := &gitprovider.OrganizationRef{ + Domain: result["host"], + Organization: result["key"], + } + ref := &gitprovider.OrgRepositoryRef{ + OrganizationRef: *orgRef, + RepositoryName: result["repo"], + } + ref.SetKey(result["key"]) + ref.Domain = addSchemeToDomain(ref.Domain) + + var repo gitprovider.OrgRepository + err := retry.OnError(DefaultBackoff, + func(err error) bool { return errors.Is(err, gitprovider.ErrNotFound) }, + func() error { + var err error + repo, err = client.OrgRepositories().Get(ctx, *ref) + if err != nil { + log.Info("Retrying getting the repository") + return err + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("unable to get repository %q: %w, (client domain: %s)", url, err, client.SupportedDomain()) + } + + return repo, nil +} + +func (g goGitProvider) ListPullRequests(ctx context.Context, repo gitprovider.OrgRepository) ([]*PullRequest, error) { + prList, err := repo.PullRequests().List(ctx) + if err != nil { + return nil, err + } + + prs := []*PullRequest{} + for _, pr := range prList { + prs = append(prs, &PullRequest{ + Title: pr.Get().Title, + Description: pr.Get().Description, + Link: pr.Get().WebURL, + Merged: pr.Get().Merged, + }) + } + + return prs, nil +} diff --git a/pkg/git/jenkins_scm.go b/pkg/git/jenkins_scm.go new file mode 100644 index 0000000000..bac907ae49 --- /dev/null +++ b/pkg/git/jenkins_scm.go @@ -0,0 +1,180 @@ +package git + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/go-logr/logr" + "github.com/jenkins-x/go-scm/scm" + "k8s.io/client-go/util/retry" +) + +type jenkinsSCM struct{} + +func (p *jenkinsSCM) GetRepository(ctx context.Context, log logr.Logger, client *scm.Client, repoURL *url.URL) (*scm.Repository, error) { + pathParts := strings.Split(strings.Trim(repoURL.Path, "/"), "/") + if len(pathParts) != 4 { + return nil, fmt.Errorf("unbale to parse url %+v", repoURL) + } + + org := pathParts[0] + project := pathParts[1] + name := pathParts[3] + + var repo *scm.Repository + err := retry.OnError(DefaultBackoff, + func(err error) bool { return errors.Is(err, scm.ErrNotFound) }, + func() error { + var err error + repo, _, err = client.Repositories.Find(ctx, fmt.Sprintf("%s/%s/%s", org, project, name)) + if err != nil { + log.Info("Retrying getting the repository") + return err + } + return nil + }, + ) + if err != nil { + return nil, fmt.Errorf("unable to get repository %q: %w", repoURL, err) + } + + return repo, nil +} + +func (p *jenkinsSCM) GetCurrentCommitOfBranch(ctx context.Context, client *scm.Client, repo *scm.Repository, branch, path string) (string, error) { + commits, _, err := client.Git.ListCommits(ctx, repo.FullName, scm.CommitListOptions{Ref: branch, Path: path}) + if err != nil { + return "", fmt.Errorf("failed to list commits: %w", err) + } + + return commits[0].Sha, nil +} + +func (p *jenkinsSCM) CreateFile(ctx context.Context, client *scm.Client, repo *scm.Repository, path string, params *scm.ContentParams) error { + _, err := client.Contents.Create(ctx, repo.FullName, path, params) + if err != nil { + return fmt.Errorf("unable to write files to branch %q: %w", params.Branch, err) + } + + return nil +} + +func (p *jenkinsSCM) UpdateFile(ctx context.Context, client *scm.Client, repo *scm.Repository, path string, params *scm.ContentParams) error { + _, err := client.Contents.Update(ctx, repo.FullName, path, params) + if err != nil { + return fmt.Errorf("unable to write files to branch %q: %w", params.Branch, err) + } + + return nil +} + +func (p *jenkinsSCM) Endpoint(repoURL, path string) (string, error) { + u, err := url.Parse(repoURL) + if err != nil { + return "", fmt.Errorf("unbale to parse url %q: %w", repoURL, err) + } + + pathParts := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(pathParts) != 4 { + return "", fmt.Errorf("unbale to parse url %+v", u) + } + + org := pathParts[0] + project := pathParts[1] + name := pathParts[3] + + return fmt.Sprintf( + "%s/%s/_apis/git/repositories/%s/%s?api-version=6.0", + org, + project, + name, + path, + ), nil +} + +// Here comes a tiny hack. +// +// jenkins-x/go-scm commits changes file by file, which is a pain. So here is a +// function that does the same as the library does but with multiple files per +// commit. +// +// See: +// https://github.com/jenkins-x/go-scm/blob/main/scm/driver/azure/content.go#L91 + +func (p *jenkinsSCM) CommitFilesRequest(sha, url, head, message string, files []CommitFile) *scm.Request { + endpoint, _ := p.Endpoint(url, "pushes") + + ref := refUpdate{ + Name: fmt.Sprintf("refs/heads/%s", head), + OldObjectID: sha, + } + + com := commitRef{ + Comment: message, + Changes: []change{}, + } + + for _, file := range files { + cha := change{} + if file.Content == nil { + cha.ChangeType = "edit" + } else { + cha.ChangeType = "add" + cha.NewContent.Content = base64.StdEncoding.EncodeToString([]byte(*file.Content)) + cha.NewContent.ContentType = "base64encoded" + } + + cha.Item.Path = file.Path + + com.Changes = append(com.Changes, cha) + } + + buf := new(bytes.Buffer) + _ = json.NewEncoder(buf).Encode(&contentCreateUpdate{ + RefUpdates: []refUpdate{ref}, + Commits: []commitRef{com}, + }) + + return &scm.Request{ + Method: "POST", + Path: endpoint, + Header: map[string][]string{ + "Content-Type": {"application/json"}, + }, + Body: buf, + } +} + +type refUpdate struct { + Name string `json:"name"` + OldObjectID string `json:"oldObjectId,omitempty"` +} + +type change struct { + ChangeType string `json:"changeType"` + Item struct { + Path string `json:"path"` + } `json:"item"` + NewContent struct { + Content string `json:"content,omitempty"` + ContentType string `json:"contentType,omitempty"` + } `json:"newContent,omitempty"` +} + +type commitRef struct { + Comment string `json:"comment,omitempty"` + Changes []change `json:"changes,omitempty"` + CommitID string `json:"commitId"` + URL string `json:"url"` +} + +type contentCreateUpdate struct { + RefUpdates []refUpdate `json:"refUpdates"` + Commits []commitRef `json:"commits"` +} diff --git a/pkg/git/provider.go b/pkg/git/provider.go index 6331ddde14..c27f94d8c2 100644 --- a/pkg/git/provider.go +++ b/pkg/git/provider.go @@ -12,28 +12,13 @@ type Provider interface { // is a two-step process that involves making multiple API // requests. CreatePullRequest(context.Context, PullRequestInput) (*PullRequest, error) -} -// PullRequestInput represents the input data when creating a -// pull request. -type PullRequestInput struct { - GitProvider GitProvider - RepositoryURL string - Title string - Body string - // Name of the branch that implements the changes. - Head string - // Name of the branch that will receive the changes. - Base string - CommitMessage string - Files []CommitFile -} + // Setup configures the provider from ProverOption. + Setup(ProviderOption) error -// PullRequest represents the result after successfully -// creating a pull request. -type PullRequest struct { - // Link links to the pull request page. - Link string + GetRepository(ctx context.Context, repoURL string) (*Repository, error) + GetTreeList(ctx context.Context, repoUrl, sha, path string) ([]*TreeEntry, error) + ListPullRequests(ctx context.Context, repoURL string) ([]*PullRequest, error) } // CommitFile represents the contents of file in the repository. diff --git a/pkg/git/pull_request.go b/pkg/git/pull_request.go new file mode 100644 index 0000000000..d832b70c5a --- /dev/null +++ b/pkg/git/pull_request.go @@ -0,0 +1,28 @@ +package git + +// PullRequestInput represents the input data when creating a +// pull request. +type PullRequestInput struct { + GitProvider GitProvider + RepositoryURL string + Title string + Body string + // Name of the branch that implements the changes. + Head string + // Name of the branch that will receive the changes. + Base string + Commits []Commit +} + +// PullRequest represents the result after successfully +// creating a pull request. +type PullRequest struct { + // Title is the title of the pull request. + Title string + // Description is the description of the pull request. + Description string + // Link links to the pull request page. + Link string + // Merge shows if the pull request is merged or not. + Merged bool +} diff --git a/pkg/git/repository.go b/pkg/git/repository.go new file mode 100644 index 0000000000..08bec32a3b --- /dev/null +++ b/pkg/git/repository.go @@ -0,0 +1,16 @@ +package git + +type Repository struct { + Domain string + Org string + Name string +} + +type TreeEntry struct { + Name string + Path string + Type string + Size int + SHA string + Link string +} diff --git a/pkg/git/utils.go b/pkg/git/utils.go index 182c27af51..c684fc8d08 100644 --- a/pkg/git/utils.go +++ b/pkg/git/utils.go @@ -170,63 +170,12 @@ func addSchemeToDomain(domain string) string { } type writeFilesToBranchRequest struct { - Repository gitprovider.OrgRepository HeadBranch string BaseBranch string Commits []Commit } -func writeFilesToBranch(ctx context.Context, log logr.Logger, req writeFilesToBranchRequest) error { - - var commits []gitprovider.Commit - err := retry.OnError(DefaultBackoff, - func(err error) bool { - // Ideally this should return true only for 404 (gitprovider.ErrNotFound) and 409 errors - return true - }, func() error { - var err error - commits, err = req.Repository.Commits().ListPage(ctx, req.BaseBranch, 1, 1) - if err != nil { - log.Info("Retrying getting the repository") - return err - } - return nil - }) - if err != nil { - return fmt.Errorf("unable to get most recent commit for branch %q: %w", req.BaseBranch, err) - } - if len(commits) == 0 { - return fmt.Errorf("no commits were found for branch %q, is the repository empty?", req.BaseBranch) - } - - err = req.Repository.Branches().Create(ctx, req.HeadBranch, commits[0].Get().Sha) - if err != nil { - return fmt.Errorf("unable to create new branch %q from commit %q in branch %q: %w", req.HeadBranch, commits[0].Get().Sha, req.BaseBranch, err) - } - - // Loop through all the commits and write the files. - for _, c := range req.Commits { - // Adapt for go-git-providers - adapted := make([]gitprovider.CommitFile, 0) - for _, f := range c.Files { - adapted = append(adapted, gitprovider.CommitFile{ - Path: &f.Path, - Content: f.Content, - }) - } - - commit, err := req.Repository.Commits().Create(ctx, req.HeadBranch, c.CommitMessage, adapted) - if err != nil { - return fmt.Errorf("unable to commit changes to %q: %w", req.HeadBranch, err) - } - log.WithValues("sha", commit.Get().Sha, "branch", req.HeadBranch).Info("Files committed") - } - - return nil -} - type createPullRequestRequest struct { - Repository gitprovider.OrgRepository HeadBranch string BaseBranch string Title string @@ -236,15 +185,3 @@ type createPullRequestRequest struct { type createPullRequestResponse struct { WebURL string } - -func createPullRequest(ctx context.Context, log logr.Logger, req createPullRequestRequest) (*createPullRequestResponse, error) { - pr, err := req.Repository.PullRequests().Create(ctx, req.Title, req.HeadBranch, req.BaseBranch, req.Description) - if err != nil { - return nil, fmt.Errorf("unable to create new pull request for branch %q: %w", req.HeadBranch, err) - } - log.WithValues("pullRequestURL", pr.Get().WebURL).Info("Created pull request") - - return &createPullRequestResponse{ - WebURL: pr.Get().WebURL, - }, nil -} diff --git a/pkg/pipelines/server/list_prs.go b/pkg/pipelines/server/list_prs.go index eb752ed69d..80a7e5b320 100644 --- a/pkg/pipelines/server/list_prs.go +++ b/pkg/pipelines/server/list_prs.go @@ -74,15 +74,13 @@ func (s *server) ListPullRequests(ctx context.Context, msg *pb.ListPullRequestsR return nil, fmt.Errorf("failed listing pull requests: %w", err) } - for _, pr := range allPrs { - prInfo := pr.Get() - + for _, prInfo := range allPrs { if prInfo.Merged { continue } if strings.Contains(prInfo.Description, fmt.Sprintf("%s/%s/%s", p.Namespace, p.Name, e.Name)) { - openPrs[e.Name] = prInfo.WebURL + openPrs[e.Name] = prInfo.Link } } } diff --git a/pkg/pipelines/server/list_prs_test.go b/pkg/pipelines/server/list_prs_test.go index 56db62545c..bf0048b1d4 100644 --- a/pkg/pipelines/server/list_prs_test.go +++ b/pkg/pipelines/server/list_prs_test.go @@ -5,7 +5,6 @@ import ( "fmt" "testing" - "github.com/fluxcd/go-git-providers/gitprovider" "github.com/fluxcd/pkg/apis/meta" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,6 +13,7 @@ import ( "github.com/weaveworks/weave-gitops-enterprise/internal/grpctesting" "github.com/weaveworks/weave-gitops-enterprise/internal/pipetesting" pb "github.com/weaveworks/weave-gitops-enterprise/pkg/api/pipelines" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,7 +44,7 @@ func TestGetListPullRequest(t *testing.T) { } require.NoError(t, kclient.Create(ctx, p)) - prs := []gitprovider.PullRequest{gitfakes.NewPullRequest(0, "testing", pipelineNamespace.Name+"/pipe-2/env-1", "http://example.com/foo/bar/pulls/1", false, "main")} + prs := []*git.PullRequest{gitfakes.NewPullRequest(0, "testing", pipelineNamespace.Name+"/pipe-2/env-1", "http://example.com/foo/bar/pulls/1", false, "main")} fakeGitProvider := gitfakes.NewFakeGitProvider("", nil, nil, nil, prs) factory := grpctesting.MakeClustersManager(kclient, "management", fmt.Sprintf("%s/cluster-1", pipelineNamespace.Name)) diff --git a/tools/kind-cluster-with-extramounts.yaml b/tools/kind-cluster-with-extramounts.yaml index 91a14307d4..b6e633698b 100644 --- a/tools/kind-cluster-with-extramounts.yaml +++ b/tools/kind-cluster-with-extramounts.yaml @@ -5,3 +5,13 @@ nodes: extraMounts: - hostPath: /var/run/docker.sock containerPath: /var/run/docker.sock + extraPortMappings: + - containerPort: 30000 + hostPort: 80 + listenAddress: "0.0.0.0" + protocol: tcp + - containerPort: 30001 + hostPort: 443 + listenAddress: "0.0.0.0" + protocol: tcp +- role: worker