Merge pull request #73 from Clever/refactor-provider

Create a Provider (Github/Gitlab) during init and pass it on to other steps

.github/pull_request_template.md

@@ -1,9 +1,8 @@
 ## JIRA
-[Link to JIRA](insert url)
+_If there was a JIRA ticket associated with this changes, please link it here._
 
 ## Overview
-- [ ] Bumped up the version in VERSION file
+_Give a high level description of your changes._
 
 ## Testing
-
-## Rollout
+_Describe testing you did to ensure your changes work and that existing functionality was not broken by the changes._

DEVELOPMENT.md

@@ -0,0 +1,58 @@
+## Development
+
+### Build Locally
+
+Microplane is a Golang project.
+
+First, clone the repo.
+
+To install dependencies run
+```
+make install_deps
+```
+
+To build, run
+```
+make build
+```
+
+You should now have a working build of Microplane in `./bin/mp`.
+
+### Design
+
+Microplane parallelizes various git commands and API calls.
+
+At each step in the Microplane workflow, a repo only moves forward if the previous step for that repo was successful.
+
+We persist the progress of a Microplane run in the following local file structure.
+
+```
+mp/
+  init.json
+  repo1/
+    clone/
+      clone.json
+      <git-repo>
+    plan/
+      plan.json
+      <git-repo-with-commit>
+    push/
+      push.json
+    merge/
+      merge.json
+  repo2/
+    ...
+```
+
+### Releasing
+
+Before releasing:
+
+- Ensure you've tested microplane flow end to end. (future: improve integration tests)
+- PR your changes and get them reviewed.
+
+To publish a release:
+
+- Merge your approved pull request to `master`.
+- Push another commit to `master`, updating both `VERSION` with the new version and `CHANGELOG.md` with a description of the changes.
+- CircleCI will publish a release to GitHub.

Makefile

@@ -7,10 +7,6 @@ VERSION := $(shell head -n 1 VERSION)
 EXECUTABLE := mp
 $(eval $(call golang-version-check,1.15))
 
-$(GOPATH)/bin/dep:
-	@go get github.com/golang/dep
-	@go install github.com/golang/dep/...
-
 all: test build release
 
 test: $(PKGS)
@@ -29,4 +25,6 @@ release:
 		-o="$@/$(EXECUTABLE)-$(VERSION)-darwin-amd64"
 
 install_deps:
+	go get github.com/GeertJohan/fgt@262f7b11eec07dc7b147c44641236f3212fee89d
+	go get golang.org/x/lint/golint@738671d3881b9731cc63024d5d88cf28db875626
 	go mod vendor

README.md

@@ -1,7 +1,8 @@
 # microplane
 
-A CLI tool to make git changes across many repos, especially useful with Microservices.
-You can learn more about microplane in this [introductory blogpost](https://medium.com/always-a-student/mo-repos-mo-problems-how-we-make-changes-across-many-git-repositories-293ad7d418f0).
+A CLI tool to make git changes across many repos.
+
+Learn more about microplane in this [introductory blogpost](https://medium.com/always-a-student/mo-repos-mo-problems-how-we-make-changes-across-many-git-repositories-293ad7d418f0).
 
 ![microplane](https://cdn.pixabay.com/photo/2013/07/12/14/16/lemon-148119_640.png)
 
@@ -23,16 +24,19 @@ The `GITHUB_API_TOKEN` environment variable must be set for Github. This should
 
 ### GitLab setup
 
+
 The `GITLAB_API_TOKEN` environment variable must be set for Gitlab. This should be a [GitLab access token](https://gitlab.com/profile/personal_access_tokens)
 
-Optionally: The `GITLAB_URL` environment variable can be set to use a Gitlab on-premise setup, otherwise it will use https://gitlab.com.
+To use Gitlab, you must specifically pass `--provider=gitlab` when running `mp init`.
+
+Optional: If you use a self-hosted Gitlab, you can specify its URL by passing `--provider_url=<your URL>` when running `mp init`.
 
 ### Using Microplane
 
 Microplane has an opinionated workflow for how you should manage git changes across many repos.
 To make a change, use the following series of commands.
 
-1. [Init](docs/mp_init.md) - target the repos you want to change using a [Github "Advanced Search" Query](https://github.com/search/advanced)
+1. [Init](docs/mp_init.md) - target the repos you want to change
 2. [Clone](docs/mp_clone.md) - clone the repos you just targeted
 3. [Plan](docs/mp_plan.md) - run a script against each of the repos and preview the diff
 4. [Push](docs/mp_push.md) - commit, push, and open a Pull Request
@@ -42,54 +46,4 @@ For an in-depth example, check out the [introductory blogpost](https://medium.co
 
 ## Development
 
-### Build Locally
-
-Microplane is a Golang project.
-
-First, clone the repo.
-
-To install dependencies run
-```
-make install_deps
-```
-
-To build, run
-```
-make build
-```
-
-You should now have a working build of Microplane in `./bin/mp`.
-
-### Design
-
-Microplane parallelizes various git commands and API calls.
-
-At each step in the Microplane workflow, a repo only moves forward if the previous step for that repo was successful.
-
-We persist the progress of a Microplane run in the following local file structure.
-
-```
-mp/
-  init.json
-  repo1/
-    clone/
-      clone.json
-      <git-repo>
-    plan/
-      plan.json
-      <git-repo-with-commit>
-    push/
-      push.json
-    merge/
-      merge.json
-  repo2/
-    ...
-```
-
-### Releasing
-
-To publish a release:
-
-- PR then merge changes to master.
-- Push another commit, updating `VERSION` with the new version and `CHANGELOG.md` with a description of the changes.
-- CircleCI will publish a release to GitHub.
+See [`Development.md`](./DEVELOPMENT.md).

cmd/clone.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/Clever/microplane/clone"
-	"github.com/Clever/microplane/initialize"
+	"github.com/Clever/microplane/lib"
 	"github.com/spf13/cobra"
 )
 
@@ -28,7 +28,7 @@ var cloneCmd = &cobra.Command{
 	},
 }
 
-func cloneOneRepo(r initialize.Repo, ctx context.Context) error {
+func cloneOneRepo(r lib.Repo, ctx context.Context) error {
 	log.Printf("cloning: %s/%s", r.Owner, r.Name)
 
 	// Prepare workdir for current step's output

cmd/helpers.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 
 	"github.com/Clever/microplane/initialize"
+	"github.com/Clever/microplane/lib"
 	"github.com/facebookgo/errgroup"
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/semaphore"
@@ -28,18 +29,18 @@ func writeJSON(obj interface{}, path string) error {
 	return ioutil.WriteFile(path, b, 0644)
 }
 
-func parallelize(repos []initialize.Repo, f func(initialize.Repo, context.Context) error) error {
+func parallelize(repos []lib.Repo, f func(lib.Repo, context.Context) error) error {
 	return parallelizeLimited(repos, f, 10)
 }
 
 // parallelize take a list of repos and applies a function (clone, plan, ...) to them
-func parallelizeLimited(repos []initialize.Repo, f func(initialize.Repo, context.Context) error, parallelismLimit int64) error {
+func parallelizeLimited(repos []lib.Repo, f func(lib.Repo, context.Context) error, parallelismLimit int64) error {
 	ctx := context.Background()
 	var eg errgroup.Group
 	parallelLimit := semaphore.NewWeighted(parallelismLimit)
 	for _, r := range repos {
 		eg.Add(1)
-		go func(repo initialize.Repo) {
+		go func(repo lib.Repo) {
 			parallelLimit.Acquire(ctx, 1)
 			defer parallelLimit.Release(1)
 			defer eg.Done()
@@ -57,15 +58,15 @@ func parallelizeLimited(repos []initialize.Repo, f func(initialize.Repo, context
 
 // whichRepos determines which repos are relevant to the current command.
 // It also handles the `singleRepo` flag, allowing a user to target just one repo.
-func whichRepos(cmd *cobra.Command) ([]initialize.Repo, error) {
+func whichRepos(cmd *cobra.Command) ([]lib.Repo, error) {
 	var initOutput initialize.Output
 	if err := loadJSON(outputPath("", "init"), &initOutput); err != nil {
-		return []initialize.Repo{}, err
+		return []lib.Repo{}, err
 	}
 
 	singleRepo, err := cmd.Flags().GetString("repo")
 	if err != nil {
-		return []initialize.Repo{}, err
+		return []lib.Repo{}, err
 	}
 
 	// All repos
@@ -76,9 +77,9 @@ func whichRepos(cmd *cobra.Command) ([]initialize.Repo, error) {
 	// Single repo
 	for _, r := range initOutput.Repos {
 		if r.Name == singleRepo {
-			return []initialize.Repo{r}, nil
+			return []lib.Repo{r}, nil
 		}
 	}
 	// TODO: showing valid repo names would be helpful
-	return []initialize.Repo{}, fmt.Errorf("%s not a targeted repo name", singleRepo)
+	return []lib.Repo{}, fmt.Errorf("%s not a targeted repo name", singleRepo)
 }

cmd/helpers_test.go

@@ -6,14 +6,14 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/Clever/microplane/initialize"
+	"github.com/Clever/microplane/lib"
 	"github.com/stretchr/testify/assert"
 )
 
 var total = 0
 var mutex = sync.RWMutex{}
 
-func doNothingOneRepo(r initialize.Repo, ctx context.Context) error {
+func doNothingOneRepo(r lib.Repo, ctx context.Context) error {
 	// TODO: Write to a channel
 	fmt.Println("do nothing: ", r.Name)
 	// Side effect ... write a temp file
@@ -25,14 +25,14 @@ func doNothingOneRepo(r initialize.Repo, ctx context.Context) error {
 }
 
 func TestParallelize(t *testing.T) {
-	repos := []initialize.Repo{
-		initialize.Repo{
+	repos := []lib.Repo{
+		lib.Repo{
 			Name: "repo1",
 		},
-		initialize.Repo{
+		lib.Repo{
 			Name: "repo2",
 		},
-		initialize.Repo{
+		lib.Repo{
 			Name: "repo3",
 		},
 	}

cmd/init.go

@@ -9,11 +9,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var repoProviderFlag string
-var initFlagReposFile string
-var repoSearch bool
-var allRepos bool
-
 var initCmd = &cobra.Command{
 	Use:   "init [query]",
 	Short: "Initialize a microplane workflow",
@@ -95,13 +90,14 @@ See https://docs.gitlab.com/ee/user/search/advanced_search_syntax.html for more
 		}
 
 		output, err := initialize.Initialize(initialize.Input{
-			AllRepos:      allRepos,
+			AllRepos:      initAllrepos,
 			Query:         query,
 			WorkDir:       workDir,
 			Version:       cliVersion,
-			RepoProvider:  repoProviderFlag,
+			Provider:      initProvider,
+			ProviderURL:   initProviderURL,
 			ReposFromFile: initFlagReposFile,
-			RepoSearch:    repoSearch,
+			RepoSearch:    initRepoSearch,
 		})
 		if err != nil {
 			log.Fatal(err)
@@ -117,3 +113,17 @@ See https://docs.gitlab.com/ee/user/search/advanced_search_syntax.html for more
 		}
 	},
 }
+
+var initFlagReposFile string
+var initRepoSearch bool
+var initAllrepos bool
+var initProvider string
+var initProviderURL string
+
+func init() {
+	initCmd.Flags().StringVarP(&initFlagReposFile, "file", "f", "", "get repos from a file instead of searching")
+	initCmd.Flags().BoolVar(&initRepoSearch, "repo-search", false, "get repos from a github repo search")
+	initCmd.Flags().BoolVar(&initAllrepos, "all-repos", false, "get all repos for a given org")
+	initCmd.Flags().StringVar(&initProvider, "provider", "github", "'github' or 'gitlab'")
+	initCmd.Flags().StringVar(&initProviderURL, "provider-url", "", "custom URL for enterprise setups")
+}

cmd/merge.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Clever/microplane/initialize"
+	"github.com/Clever/microplane/lib"
 	"github.com/Clever/microplane/merge"
 	"github.com/Clever/microplane/push"
 	"github.com/spf13/cobra"
@@ -53,7 +53,7 @@ var mergeCmd = &cobra.Command{
 	},
 }
 
-func mergeOneRepo(r initialize.Repo, ctx context.Context) error {
+func mergeOneRepo(r lib.Repo, ctx context.Context) error {
 	log.Printf("%s/%s - merging...", r.Owner, r.Name)
 
 	// Exit early if already merged
@@ -87,17 +87,16 @@ func mergeOneRepo(r initialize.Repo, ctx context.Context) error {
 
 	// Execute
 	input := merge.Input{
-		Org:                   r.Owner,
-		Repo:                  r.Name,
+		Repo:                  r,
 		PRNumber:              prNumber,
 		CommitSHA:             pushOutput.CommitSHA,
 		RequireReviewApproval: !mergeFlagIgnoreReviewApproval,
 		RequireBuildSuccess:   !mergeFlagIgnoreBuildStatus,
 	}
 	var output merge.Output
-	if r.Provider == "gitlab" {
+	if r.IsGitlab() {
 		output, err = merge.GitlabMerge(ctx, input, repoLimiter, mergeThrottle)
-	} else if r.Provider == "github" {
+	} else if r.IsGithub() {
 		output, err = merge.GitHubMerge(ctx, input, repoLimiter, mergeThrottle)
 	} else {
 		log.Fatal("Provider must be github or gitlab")

cmd/plan.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/Clever/microplane/clone"
-	"github.com/Clever/microplane/initialize"
+	"github.com/Clever/microplane/lib"
 	"github.com/Clever/microplane/merge"
 	"github.com/Clever/microplane/plan"
 	"github.com/spf13/cobra"
@@ -75,7 +75,7 @@ mp plan -b microplaning -m 'microplane fun' -r app-service -- python /absolute/p
 	},
 }
 
-func planOneRepo(r initialize.Repo, ctx context.Context) error {
+func planOneRepo(r lib.Repo, ctx context.Context) error {
 	log.Printf("planning: %s/%s", r.Owner, r.Name)
 
 	// Get previous step's output