diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7e63fd313e..fca43c9eac 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -149,6 +149,8 @@ jobs: make dependencies cd cmd/clusters-service go test -v ./... -tags=integration + cd ../../pkg/git + go test -v ./... -tags=integration ui-unit-tests: runs-on: ubuntu-latest diff --git a/pkg/git/bitbucket_server.go b/pkg/git/bitbucket_server.go new file mode 100644 index 0000000000..c69c163d4d --- /dev/null +++ b/pkg/git/bitbucket_server.go @@ -0,0 +1,64 @@ +package git + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" +) + +const BitBucketServerProviderName string = "bitbucket-server" + +// BitBucketServerProvider is used to interact with the BitBucket Server (stash) API. +type BitBucketServerProvider struct { + log logr.Logger +} + +func NewBitBucketServerProvider(log logr.Logger) (Provider, error) { + return &BitBucketServerProvider{ + log: log, + }, nil +} + +func (p *BitBucketServerProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { + repoURL, err := GetGitProviderUrl(input.RepositoryURL) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + repo, err := GetRepository(ctx, p.log, input.GitProvider, repoURL) + if err != nil { + return nil, fmt.Errorf("unable to get repo: %w", err) + } + + // Add the files to be created to the map of changes. + commits := []Commit{} + commits = append(commits, Commit{ + CommitMessage: input.CommitMessage, + Files: input.Files, + }) + + if err := writeFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ + Repository: repo, + HeadBranch: input.Head, + BaseBranch: input.Base, + Commits: commits, + }); err != nil { + return nil, fmt.Errorf("unable to write files to branch %q: %w", input.Head, err) + } + + res, err := createPullRequest(ctx, p.log, createPullRequestRequest{ + Repository: repo, + HeadBranch: input.Head, + BaseBranch: input.Base, + Title: input.Title, + Description: input.Body, + }) + if err != nil { + return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) + } + + return &PullRequest{ + Link: res.WebURL, + }, nil +} diff --git a/pkg/git/factory.go b/pkg/git/factory.go new file mode 100644 index 0000000000..5ec07c85bf --- /dev/null +++ b/pkg/git/factory.go @@ -0,0 +1,34 @@ +package git + +import ( + "fmt" + + "github.com/go-logr/logr" +) + +// ProviderFactory is used to create and return a +// concrete git provider. +type ProviderFactory struct { + log logr.Logger +} + +// NewFactory creates a new factory for git providers. +func NewFactory(log logr.Logger) *ProviderFactory { + return &ProviderFactory{ + log: log, + } +} + +// Create creates and returns a new git provider. +func (f *ProviderFactory) Create(provider string) (Provider, error) { + switch provider { + case GitHubProviderName: + return NewGitHubProvider(f.log) + case GitLabProviderName: + return NewGitLabProvider(f.log) + case BitBucketServerProviderName: + return NewBitBucketServerProvider(f.log) + default: + return nil, fmt.Errorf("provider %q is not supported", provider) + } +} diff --git a/pkg/git/github.go b/pkg/git/github.go new file mode 100644 index 0000000000..a063f1279c --- /dev/null +++ b/pkg/git/github.go @@ -0,0 +1,66 @@ +package git + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" +) + +const GitHubProviderName string = "github" + +// GitHubProvider is used to interact with the GitHub API. +// This implementation delegates most of the work to the +// fluxcd/go-git-providers library. +type GitHubProvider struct { + log logr.Logger +} + +func NewGitHubProvider(log logr.Logger) (Provider, error) { + return &GitHubProvider{ + log: log, + }, nil +} + +func (p *GitHubProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { + repoURL, err := GetGitProviderUrl(input.RepositoryURL) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + repo, err := GetRepository(ctx, p.log, input.GitProvider, repoURL) + if err != nil { + return nil, fmt.Errorf("unable to get repo: %w", err) + } + + // Add the files to be created to the map of changes. + commits := []Commit{} + commits = append(commits, Commit{ + CommitMessage: input.CommitMessage, + Files: input.Files, + }) + + if err := writeFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ + Repository: repo, + HeadBranch: input.Head, + BaseBranch: input.Base, + Commits: commits, + }); err != nil { + return nil, fmt.Errorf("unable to write files to branch %q: %w", input.Head, err) + } + + res, err := createPullRequest(ctx, p.log, createPullRequestRequest{ + Repository: repo, + HeadBranch: input.Head, + BaseBranch: input.Base, + Title: input.Title, + Description: input.Body, + }) + if err != nil { + return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) + } + + return &PullRequest{ + Link: res.WebURL, + }, nil +} diff --git a/pkg/git/github_test.go b/pkg/git/github_test.go new file mode 100644 index 0000000000..a1e3cb0730 --- /dev/null +++ b/pkg/git/github_test.go @@ -0,0 +1,189 @@ +//go:build integration +// +build integration + +package git_test + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/google/go-github/v32/github" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" + "golang.org/x/oauth2" +) + +const ( + TestRepositoryNamePrefix = "wge-integration-test-repo" +) + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +func TestCreatePullRequestInGitHubOrganization(t *testing.T) { + // Create a client + ctx := context.Background() + client := github.NewClient( + oauth2.NewClient(ctx, + oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ), + ), + ) + + // Create a repository using a name that doesn't exist already + repoName := fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + repos, _, err := client.Repositories.ListByOrg(ctx, os.Getenv("GITHUB_ORG"), nil) + assert.NoError(t, err) + for findGitHubRepo(repos, repoName) != nil { + repoName = fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + } + repo, _, err := client.Repositories.Create(ctx, os.Getenv("GITHUB_ORG"), &github.Repository{ + Name: github.String(repoName), + Private: github.Bool(true), + AutoInit: github.Bool(true), + }) + require.NoError(t, err) + defer func() { + _, err = client.Repositories.Delete(ctx, os.Getenv("GITHUB_ORG"), repo.GetName()) + require.NoError(t, err) + }() + + p, err := git.NewFactory(logr.Discard()).Create(git.GitHubProviderName) + require.NoError(t, err) + content := "---\n" + res, err := p.CreatePullRequest(ctx, git.PullRequestInput{ + GitProvider: git.GitProvider{ + Token: os.Getenv("GITHUB_TOKEN"), + Type: git.GitHubProviderName, + Hostname: "github.com", + }, + RepositoryURL: repo.GetCloneURL(), + Head: "feature-01", + Base: repo.GetDefaultBranch(), + Title: "New cluster", + Body: "Creates a cluster through a CAPI template", + CommitMessage: "Add cluster manifest", + Files: []git.CommitFile{ + { + Path: "management/cluster-01.yaml", + Content: &content, + }, + }, + }) + require.NoError(t, err) + + pr, _, err := client.PullRequests.Get(ctx, os.Getenv("GITHUB_ORG"), repo.GetName(), 1) // #PR is 1 because it is a new repo + require.NoError(t, err) + assert.Equal(t, pr.GetHTMLURL(), res.Link) + assert.Equal(t, pr.GetTitle(), "New cluster") + assert.Equal(t, pr.GetBody(), "Creates a cluster through a CAPI template") + assert.Equal(t, pr.GetChangedFiles(), 1) +} + +func TestCreatePullRequestInGitHubUser(t *testing.T) { + // Create a client + ctx := context.Background() + client := github.NewClient( + oauth2.NewClient(ctx, + oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: os.Getenv("GITHUB_TOKEN")}, + ), + ), + ) + // Create a repository using a name that doesn't exist already + repoName := fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + repos, _, err := client.Repositories.List(ctx, os.Getenv("GITHUB_USER"), nil) + assert.NoError(t, err) + for findGitHubRepo(repos, repoName) != nil { + repoName = fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + } + repo, _, err := client.Repositories.Create(ctx, "", &github.Repository{ + Name: github.String(repoName), + Private: github.Bool(true), + AutoInit: github.Bool(true), + }) + require.NoError(t, err) + defer func() { + _, err = client.Repositories.Delete(ctx, os.Getenv("GITHUB_USER"), repo.GetName()) + require.NoError(t, err) + }() + + p, err := git.NewFactory(logr.Discard()).Create(git.GitHubProviderName) + require.NoError(t, err) + content := "---\n" + res, err := p.CreatePullRequest(ctx, git.PullRequestInput{ + GitProvider: git.GitProvider{ + Token: os.Getenv("GITHUB_TOKEN"), + Type: git.GitHubProviderName, + Hostname: "github.com", + }, + RepositoryURL: repo.GetCloneURL(), + Head: "feature-01", + Base: repo.GetDefaultBranch(), + Title: "New cluster", + Body: "Creates a cluster through a CAPI template", + CommitMessage: "Add cluster manifest", + Files: []git.CommitFile{ + { + Path: "management/cluster-01.yaml", + Content: &content, + }, + }, + }) + require.NoError(t, err) + + pr, _, err := client.PullRequests.Get(ctx, os.Getenv("GITHUB_USER"), repo.GetName(), 1) // #PR is 1 because it is a new repo + require.NoError(t, err) + assert.Equal(t, pr.GetHTMLURL(), res.Link) + assert.Equal(t, pr.GetTitle(), "New cluster") + assert.Equal(t, pr.GetBody(), "Creates a cluster through a CAPI template") + assert.Equal(t, pr.GetAdditions(), 1) + + res, err = p.CreatePullRequest(ctx, git.PullRequestInput{ + GitProvider: git.GitProvider{ + Token: os.Getenv("GITHUB_TOKEN"), + Type: git.GitHubProviderName, + Hostname: "github.com", + }, + RepositoryURL: repo.GetCloneURL(), + Head: "feature-02", + Base: "feature-01", + Title: "Delete cluster", + Body: "Deletes a cluster via gitops", + CommitMessage: "Remove cluster manifest", + Files: []git.CommitFile{ + { + Path: "management/cluster-01.yaml", + Content: nil, + }, + }, + }) + require.NoError(t, err) + + pr, _, err = client.PullRequests.Get(ctx, os.Getenv("GITHUB_USER"), repo.GetName(), 2) // #PR is 2 because it is the 2nd PR for a new repo + require.NoError(t, err) + assert.Equal(t, pr.GetHTMLURL(), res.Link) + assert.Equal(t, pr.GetTitle(), "Delete cluster") + assert.Equal(t, pr.GetBody(), "Deletes a cluster via gitops") + assert.Equal(t, pr.GetDeletions(), 1) +} + +func findGitHubRepo(repos []*github.Repository, name string) *github.Repository { + if name == "" { + return nil + } + for _, repo := range repos { + if repo.GetName() == name { + return repo + } + } + return nil +} diff --git a/pkg/git/gitlab.go b/pkg/git/gitlab.go new file mode 100644 index 0000000000..3ffb1ae6ad --- /dev/null +++ b/pkg/git/gitlab.go @@ -0,0 +1,134 @@ +package git + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/fluxcd/go-git-providers/gitprovider" + "github.com/go-logr/logr" +) + +const ( + GitLabProviderName string = "gitlab" + deleteFilesCommitMessage string = "Delete old files for resources" +) + +// GitLabProvider is used to interact with the GitLab API. +type GitLabProvider struct { + log logr.Logger +} + +func NewGitLabProvider(log logr.Logger) (Provider, error) { + return &GitLabProvider{ + log: log, + }, nil +} + +func (p *GitLabProvider) CreatePullRequest(ctx context.Context, input PullRequestInput) (*PullRequest, error) { + repoURL, err := GetGitProviderUrl(input.RepositoryURL) + if err != nil { + return nil, fmt.Errorf("unable to get git provider url: %w", err) + } + + repo, err := GetRepository(ctx, p.log, input.GitProvider, repoURL) + if err != nil { + return nil, fmt.Errorf("unable to get repo: %w", err) + } + + // Gitlab doesn't support createOrUpdate, so we need to check if the file exists + // and if it does, we need to create a commit to delete the file. + commits := []Commit{} + deletedFiles, err := getUpdatedFiles(ctx, input.Files, input.GitProvider, repoURL, input.Base) + if err != nil { + return nil, fmt.Errorf("unable to get files from tree list: %w", err) + } + + // If there are files to delete, append them to the map of changes to be deleted. + if len(deletedFiles) > 0 { + commits = append(commits, Commit{ + CommitMessage: deleteFilesCommitMessage, + Files: deletedFiles, + }) + } + + // Add the files to be created to the map of changes. + commits = append(commits, Commit{ + CommitMessage: input.CommitMessage, + Files: input.Files, + }) + + if err := writeFilesToBranch(ctx, p.log, writeFilesToBranchRequest{ + Repository: repo, + HeadBranch: input.Head, + BaseBranch: input.Base, + Commits: commits, + }); err != nil { + return nil, fmt.Errorf("unable to write files to branch %q: %w", input.Head, err) + } + + res, err := createPullRequest(ctx, p.log, createPullRequestRequest{ + Repository: repo, + HeadBranch: input.Head, + BaseBranch: input.Base, + Title: input.Title, + Description: input.Body, + }) + if err != nil { + return nil, fmt.Errorf("unable to create pull request for branch %q: %w", input.Head, err) + } + + return &PullRequest{ + Link: res.WebURL, + }, nil +} + +func getUpdatedFiles( + ctx context.Context, + reqFiles []CommitFile, + gp GitProvider, + repoURL, + branch string) ([]CommitFile, error) { + var updatedFiles []CommitFile + + for _, file := range reqFiles { + // if file content is empty, then it's a delete operation + // so we don't need to check if the file exists + if file.Content == nil { + continue + } + + dirPath, _ := filepath.Split(file.Path) + + treeEntries, err := GetTreeList(ctx, gp, repoURL, branch, dirPath, true) + if err != nil { + return nil, fmt.Errorf("error getting list of trees in repo: %s@%s: %w", repoURL, branch, err) + } + + for _, treeEntry := range treeEntries { + if treeEntry.Path == file.Path { + updatedFiles = append(updatedFiles, CommitFile{ + Path: treeEntry.Path, + Content: nil, + }) + } + } + } + + return updatedFiles, nil +} + +// GetTreeList retrieves list of tree files from gitprovider given the sha/branch +func GetTreeList(ctx context.Context, gp GitProvider, repoUrl string, sha string, path string, recursive bool) ([]*gitprovider.TreeEntry, error) { + repo, err := GetRepository(ctx, logr.Discard(), gp, repoUrl) + if err != nil { + return nil, err + } + + treePaths, err := repo.Trees().List(ctx, sha, path, recursive) + if err != nil { + return nil, err + } + + return treePaths, nil +} diff --git a/pkg/git/gitlab_test.go b/pkg/git/gitlab_test.go new file mode 100644 index 0000000000..6364d07147 --- /dev/null +++ b/pkg/git/gitlab_test.go @@ -0,0 +1,255 @@ +//go:build integration +// +build integration + +package git_test + +import ( + "context" + "fmt" + "math/rand" + "os" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/weaveworks/weave-gitops-enterprise/pkg/git" + "github.com/xanzy/go-gitlab" +) + +func TestCreatePullRequestInGitLab(t *testing.T) { + // Create a client + ctx := context.Background() + gitlabHost := fmt.Sprintf("https://%s", os.Getenv("GIT_PROVIDER_HOSTNAME")) + client, err := gitlab.NewClient(os.Getenv("GITLAB_TOKEN"), gitlab.WithBaseURL(gitlabHost)) + require.NoError(t, err) + + repoName := TestRepositoryNamePrefix + "-group-test" + + // Create a group using a name that doesn't exist already + groupName := fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + groups, _, err := client.Groups.ListGroups(&gitlab.ListGroupsOptions{}) + assert.NoError(t, err) + for findGitLabGroup(groups, groupName) != nil { + groupName = fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + } + + parentGroup, _, err := client.Groups.CreateGroup(&gitlab.CreateGroupOptions{ + Path: gitlab.String(groupName), + Name: gitlab.String(groupName), + Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + }) + require.NoError(t, err) + t.Cleanup(func() { + _, err = client.Groups.DeleteGroup(parentGroup.ID) + require.NoError(t, err) + }) + + fooGroup, _, err := client.Groups.CreateGroup(&gitlab.CreateGroupOptions{ + Path: gitlab.String("foo"), + Name: gitlab.String("foo group"), + ParentID: gitlab.Int(parentGroup.ID), + Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + }) + require.NoError(t, err) + t.Cleanup(func() { + _, err = client.Groups.DeleteGroup(fooGroup.ID) + require.NoError(t, err) + }) + + repo, _, err := client.Projects.CreateProject(&gitlab.CreateProjectOptions{ + Name: gitlab.String(repoName), + MergeRequestsEnabled: gitlab.Bool(true), + Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + InitializeWithReadme: gitlab.Bool(true), + NamespaceID: gitlab.Int(fooGroup.ID), + }) + require.NoError(t, err) + t.Cleanup(func() { + _, err = client.Projects.DeleteProject(repo.ID) + require.NoError(t, err) + }) + + p, err := git.NewFactory(logr.Discard()).Create(git.GitLabProviderName) + require.NoError(t, err) + content := "---\n" + res, err := p.CreatePullRequest(ctx, git.PullRequestInput{ + GitProvider: git.GitProvider{ + Token: os.Getenv("GITLAB_TOKEN"), + Type: git.GitLabProviderName, + Hostname: os.Getenv("GIT_PROVIDER_HOSTNAME"), + }, + RepositoryURL: repo.HTTPURLToRepo, + Head: "feature-01", + Base: repo.DefaultBranch, + Title: "New cluster", + Body: "Creates a cluster through a CAPI template", + CommitMessage: "Add cluster manifest", + Files: []git.CommitFile{ + { + Path: "management/cluster-01.yaml", + Content: &content, + }, + }, + }) + assert.NoError(t, err) + + pr, _, err := client.MergeRequests.GetMergeRequest(repo.ID, 1, nil) // #PR is 1 because it is a new repo + require.NoError(t, err) + assert.Equal(t, pr.WebURL, res.Link) + assert.Equal(t, pr.Title, "New cluster") + assert.Equal(t, pr.Description, "Creates a cluster through a CAPI template") +} + +func TestCreatePullRequestInGitLab_UpdateFiles(t *testing.T) { + // Create a client + ctx := context.Background() + gitlabHost := fmt.Sprintf("https://%s", os.Getenv("GIT_PROVIDER_HOSTNAME")) + client, err := gitlab.NewClient(os.Getenv("GITLAB_TOKEN"), gitlab.WithBaseURL(gitlabHost)) + require.NoError(t, err) + // Create a repository using a name that doesn't exist already + repoName := fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + repos, _, err := client.Projects.ListProjects(&gitlab.ListProjectsOptions{ + Owned: gitlab.Bool(true), + }) + assert.NoError(t, err) + for findGitLabRepo(repos, repoName) != nil { + repoName = fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + } + repo, _, err := client.Projects.CreateProject(&gitlab.CreateProjectOptions{ + Name: gitlab.String(repoName), + MergeRequestsEnabled: gitlab.Bool(true), + Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + InitializeWithReadme: gitlab.Bool(true), + }) + require.NoError(t, err) + defer func() { + _, err = client.Projects.DeleteProject(repo.ID) + require.NoError(t, err) + }() + + _, _, err = client.RepositoryFiles.CreateFile(repo.ID, "management/cluster-01.yaml", &gitlab.CreateFileOptions{ + Branch: gitlab.String(repo.DefaultBranch), + Content: gitlab.String("---\n"), + CommitMessage: gitlab.String("Add cluster manifest"), + }) + require.NoError(t, err) + + p, err := git.NewFactory(logr.Discard()).Create(git.GitLabProviderName) + require.NoError(t, err) + content := "---\n\n" + res, err := p.CreatePullRequest(ctx, git.PullRequestInput{ + GitProvider: git.GitProvider{ + Token: os.Getenv("GITLAB_TOKEN"), + Type: git.GitLabProviderName, + Hostname: os.Getenv("GIT_PROVIDER_HOSTNAME"), + }, + RepositoryURL: repo.HTTPURLToRepo, + Head: "feature-01", + Base: repo.DefaultBranch, + Title: "Update cluster", + Body: "Updates a cluster through a CAPI template", + CommitMessage: "Update cluster manifest", + Files: []git.CommitFile{ + { + Path: "management/cluster-01.yaml", + Content: &content, + }, + }, + }) + assert.NoError(t, err) + + pr, _, err := client.MergeRequests.GetMergeRequest(repo.ID, 1, nil) + require.NoError(t, err) + assert.Equal(t, pr.WebURL, res.Link) + assert.Equal(t, pr.Title, "Update cluster") + assert.Equal(t, pr.Description, "Updates a cluster through a CAPI template") +} + +func TestCreatePullRequestInGitLab_DeleteFiles(t *testing.T) { + // Create a client + ctx := context.Background() + gitlabHost := fmt.Sprintf("https://%s", os.Getenv("GIT_PROVIDER_HOSTNAME")) + client, err := gitlab.NewClient(os.Getenv("GITLAB_TOKEN"), gitlab.WithBaseURL(gitlabHost)) + require.NoError(t, err) + // Create a repository using a name that doesn't exist already + repoName := fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + repos, _, err := client.Projects.ListProjects(&gitlab.ListProjectsOptions{ + Owned: gitlab.Bool(true), + }) + assert.NoError(t, err) + for findGitLabRepo(repos, repoName) != nil { + repoName = fmt.Sprintf("%s-%03d", TestRepositoryNamePrefix, rand.Intn(1000)) + } + repo, _, err := client.Projects.CreateProject(&gitlab.CreateProjectOptions{ + Name: gitlab.String(repoName), + MergeRequestsEnabled: gitlab.Bool(true), + Visibility: gitlab.Visibility(gitlab.PrivateVisibility), + InitializeWithReadme: gitlab.Bool(true), + }) + require.NoError(t, err) + defer func() { + _, err = client.Projects.DeleteProject(repo.ID) + require.NoError(t, err) + }() + + _, _, err = client.RepositoryFiles.CreateFile(repo.ID, "management/cluster-01.yaml", &gitlab.CreateFileOptions{ + Branch: gitlab.String(repo.DefaultBranch), + Content: gitlab.String("---\n"), + CommitMessage: gitlab.String("Add cluster manifest"), + }) + require.NoError(t, err) + + p, err := git.NewFactory(logr.Discard()).Create(git.GitLabProviderName) + require.NoError(t, err) + res, err := p.CreatePullRequest(ctx, git.PullRequestInput{ + GitProvider: git.GitProvider{ + Token: os.Getenv("GITLAB_TOKEN"), + Type: git.GitLabProviderName, + Hostname: os.Getenv("GIT_PROVIDER_HOSTNAME"), + }, + RepositoryURL: repo.HTTPURLToRepo, + Head: "feature-01", + Base: repo.DefaultBranch, + Title: "Delete cluster", + Body: "Deletes a cluster", + CommitMessage: "Delete cluster manifest", + Files: []git.CommitFile{ + { + Path: "management/cluster-01.yaml", + Content: nil, + }, + }, + }) + assert.NoError(t, err) + + pr, _, err := client.MergeRequests.GetMergeRequest(repo.ID, 1, nil) + require.NoError(t, err) + assert.Equal(t, pr.WebURL, res.Link) + assert.Equal(t, pr.Title, "Delete cluster") + assert.Equal(t, pr.Description, "Deletes a cluster") +} + +func findGitLabGroup(groups []*gitlab.Group, name string) *gitlab.Group { + if name == "" { + return nil + } + for _, group := range groups { + if group.Name == name { + return group + } + } + return nil +} + +func findGitLabRepo(repos []*gitlab.Project, name string) *gitlab.Project { + if name == "" { + return nil + } + for _, repo := range repos { + if repo.Name == name { + return repo + } + } + return nil +} diff --git a/pkg/git/provider.go b/pkg/git/provider.go new file mode 100644 index 0000000000..6331ddde14 --- /dev/null +++ b/pkg/git/provider.go @@ -0,0 +1,59 @@ +package git + +import ( + "context" +) + +// Provider defines the interface that WGE will use to interact +// with a git provider. +type Provider interface { + // CreatePullRequest pushes a set of changes to a branch + // and then creates a pull request from it. Typically this + // is a two-step process that involves making multiple API + // requests. + CreatePullRequest(context.Context, PullRequestInput) (*PullRequest, error) +} + +// PullRequestInput represents the input data when creating a +// pull request. +type PullRequestInput struct { + GitProvider GitProvider + RepositoryURL string + Title string + Body string + // Name of the branch that implements the changes. + Head string + // Name of the branch that will receive the changes. + Base string + CommitMessage string + Files []CommitFile +} + +// PullRequest represents the result after successfully +// creating a pull request. +type PullRequest struct { + // Link links to the pull request page. + Link string +} + +// CommitFile represents the contents of file in the repository. +type CommitFile struct { + Path string + // Content represents the content of the change. If nil, + // this results in a deletion. + Content *string +} + +// TODO get rid of this, intermediate structure +type GitProvider struct { + Token string + TokenType string + Type string + Hostname string +} + +// TODO get rid of this, intermediate structure +type Commit struct { + CommitMessage string + Files []CommitFile +} diff --git a/pkg/git/utils.go b/pkg/git/utils.go new file mode 100644 index 0000000000..182c27af51 --- /dev/null +++ b/pkg/git/utils.go @@ -0,0 +1,250 @@ +package git + +import ( + "context" + "errors" + "fmt" + "path" + "regexp" + "strings" + "time" + + "github.com/fluxcd/go-git-providers/github" + "github.com/fluxcd/go-git-providers/gitlab" + "github.com/fluxcd/go-git-providers/gitprovider" + "github.com/fluxcd/go-git-providers/stash" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-logr/logr" + "github.com/spf13/viper" + "github.com/weaveworks/weave-gitops/pkg/gitproviders" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" +) + +var DefaultBackoff = wait.Backoff{ + Steps: 4, + Duration: 20 * time.Millisecond, + Factor: 5.0, + Jitter: 0.1, +} + +func GetGitProviderUrl(giturl string) (string, error) { + repositoryAPIURL := viper.GetString("capi-templates-repository-api-url") + if repositoryAPIURL != "" { + return repositoryAPIURL, nil + } + + ep, err := transport.NewEndpoint(giturl) + if err != nil { + return "", err + } + if ep.Protocol == "http" || ep.Protocol == "https" { + return giturl, nil + } + + httpsEp := transport.Endpoint{Protocol: "https", Host: ep.Host, Path: ep.Path} + + return httpsEp.String(), nil +} + +func GetRepository(ctx context.Context, log logr.Logger, gp GitProvider, url string) (gitprovider.OrgRepository, error) { + c, err := getGitProviderClient(gp) + if err != nil { + return nil, fmt.Errorf("unable to get a git provider client for %q: %w", gp.Type, err) + } + + var ref *gitprovider.OrgRepositoryRef + if gp.Type == string(gitproviders.GitProviderGitHub) || gp.Type == string(gitproviders.GitProviderGitLab) { + ref, err = gitprovider.ParseOrgRepositoryURL(url) + if err != nil { + return nil, fmt.Errorf("unable to parse repository URL %q: %w", url, err) + } + ref.Domain = addSchemeToDomain(ref.Domain) + ref = WithCombinedSubOrgs(*ref) + } else if gp.Type == string(gitproviders.GitProviderBitBucketServer) { + // The ParseOrgRepositoryURL function used for other providers + // fails to parse BitBucket Server URLs correctly + re := regexp.MustCompile(`://(?P[^/]+)/(.+/)?(?P[^/]+)/(?P[^/]+)\.git`) + match := re.FindStringSubmatch(url) + result := make(map[string]string) + for i, name := range re.SubexpNames() { + if i != 0 && name != "" { + result[name] = match[i] + } + } + if len(result) != 3 { + return nil, fmt.Errorf("unable to parse repository URL %q using regex %q", url, re.String()) + } + + orgRef := &gitprovider.OrganizationRef{ + Domain: result["host"], + Organization: result["key"], + } + ref = &gitprovider.OrgRepositoryRef{ + OrganizationRef: *orgRef, + RepositoryName: result["repo"], + } + ref.SetKey(result["key"]) + ref.Domain = addSchemeToDomain(ref.Domain) + } else { + return nil, fmt.Errorf("unsupported git provider") + } + + var repo gitprovider.OrgRepository + err = retry.OnError(DefaultBackoff, + func(err error) bool { return errors.Is(err, gitprovider.ErrNotFound) }, + func() error { + var err error + repo, err = c.OrgRepositories().Get(ctx, *ref) + if err != nil { + log.Info("Retrying getting the repository") + return err + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("unable to get repository %q: %w, (client domain: %s)", url, err, c.SupportedDomain()) + } + + return repo, nil +} + +// WithCombinedSubOrgs combines the subgroups into the organization field of the reference +// This is to work around a bug in the go-git-providers library where it doesn't handle subgroups correctly. +// https://github.com/fluxcd/go-git-providers/issues/183 +func WithCombinedSubOrgs(ref gitprovider.OrgRepositoryRef) *gitprovider.OrgRepositoryRef { + orgsWithSubGroups := append([]string{ref.Organization}, ref.SubOrganizations...) + ref.Organization = path.Join(orgsWithSubGroups...) + ref.SubOrganizations = nil + return &ref +} + +func getGitProviderClient(gpi GitProvider) (gitprovider.Client, error) { + var client gitprovider.Client + var err error + + // quirk of ggp + hostname := addSchemeToDomain(gpi.Hostname) + + switch gpi.Type { + case "github": + if gpi.Hostname != "github.com" { + client, err = github.NewClient( + gitprovider.WithOAuth2Token(gpi.Token), + gitprovider.WithDomain(hostname), + ) + } else { + client, err = github.NewClient( + gitprovider.WithOAuth2Token(gpi.Token), + ) + } + if err != nil { + return nil, err + } + case "gitlab": + if gpi.Hostname != "gitlab.com" { + client, err = gitlab.NewClient(gpi.Token, gpi.TokenType, gitprovider.WithDomain(hostname), gitprovider.WithConditionalRequests(true)) + } else { + client, err = gitlab.NewClient(gpi.Token, gpi.TokenType, gitprovider.WithConditionalRequests(true)) + } + if err != nil { + return nil, err + } + case "bitbucket-server": + client, err = stash.NewStashClient("git", gpi.Token, gitprovider.WithDomain(hostname), gitprovider.WithConditionalRequests(true)) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("the Git provider %q is not supported", gpi.Type) + } + return client, err +} + +func addSchemeToDomain(domain string) string { + // Fixing https:// again (ggp quirk) + if domain != "github.com" && domain != "gitlab.com" && !strings.HasPrefix(domain, "http://") && !strings.HasPrefix(domain, "https://") { + return "https://" + domain + } + return domain +} + +type writeFilesToBranchRequest struct { + Repository gitprovider.OrgRepository + HeadBranch string + BaseBranch string + Commits []Commit +} + +func writeFilesToBranch(ctx context.Context, log logr.Logger, req writeFilesToBranchRequest) error { + + var commits []gitprovider.Commit + err := retry.OnError(DefaultBackoff, + func(err error) bool { + // Ideally this should return true only for 404 (gitprovider.ErrNotFound) and 409 errors + return true + }, func() error { + var err error + commits, err = req.Repository.Commits().ListPage(ctx, req.BaseBranch, 1, 1) + if err != nil { + log.Info("Retrying getting the repository") + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("unable to get most recent commit for branch %q: %w", req.BaseBranch, err) + } + if len(commits) == 0 { + return fmt.Errorf("no commits were found for branch %q, is the repository empty?", req.BaseBranch) + } + + err = req.Repository.Branches().Create(ctx, req.HeadBranch, commits[0].Get().Sha) + if err != nil { + return fmt.Errorf("unable to create new branch %q from commit %q in branch %q: %w", req.HeadBranch, commits[0].Get().Sha, req.BaseBranch, err) + } + + // Loop through all the commits and write the files. + for _, c := range req.Commits { + // Adapt for go-git-providers + adapted := make([]gitprovider.CommitFile, 0) + for _, f := range c.Files { + adapted = append(adapted, gitprovider.CommitFile{ + Path: &f.Path, + Content: f.Content, + }) + } + + commit, err := req.Repository.Commits().Create(ctx, req.HeadBranch, c.CommitMessage, adapted) + if err != nil { + return fmt.Errorf("unable to commit changes to %q: %w", req.HeadBranch, err) + } + log.WithValues("sha", commit.Get().Sha, "branch", req.HeadBranch).Info("Files committed") + } + + return nil +} + +type createPullRequestRequest struct { + Repository gitprovider.OrgRepository + HeadBranch string + BaseBranch string + Title string + Description string +} + +type createPullRequestResponse struct { + WebURL string +} + +func createPullRequest(ctx context.Context, log logr.Logger, req createPullRequestRequest) (*createPullRequestResponse, error) { + pr, err := req.Repository.PullRequests().Create(ctx, req.Title, req.HeadBranch, req.BaseBranch, req.Description) + if err != nil { + return nil, fmt.Errorf("unable to create new pull request for branch %q: %w", req.HeadBranch, err) + } + log.WithValues("pullRequestURL", pr.Get().WebURL).Info("Created pull request") + + return &createPullRequestResponse{ + WebURL: pr.Get().WebURL, + }, nil +}