Switch v2 Scaffolding to Use Go Modules

This switches the v2 scaffolding to use Go modules.  Currently,
a custom fork of controller-runtime is being used.  We'll need to merge
that to controller-runtime master before merging this.

Author: DirectXMan12

.travis.yml

@@ -5,7 +5,7 @@ os:
   - osx
 
 go:
-- "1.11"
+- "1.12"
 
 git:
   depth: 3
@@ -28,7 +28,7 @@ install:
   -
 
 script:
-- TRACE=1 ./test.sh
+- GO111MODULE=on TRACE=1 ./test.sh
 - ./scripts/install_test.sh
 
 # TBD. Suppressing for now.

cmd/init_project.go

@@ -17,7 +17,6 @@ limitations under the License.
 package main
 
 import (
-	"bufio"
 	"fmt"
 	"log"
 	"os"
@@ -35,9 +34,7 @@ import (
 )
 
 func newInitProjectCmd() *cobra.Command {
-	o := projectOptions{
-		projectScaffolder: &scaffold.Project{},
-	}
+	o := projectOptions{}
 
 	initCmd := &cobra.Command{
 		Use:   "init",
@@ -70,50 +67,54 @@ kubebuilder init --domain example.org --license apache2 --owner "The Kubernetes
 }
 
 type projectOptions struct {
-	projectScaffolder *scaffold.Project
 
+	// flags
+	fetchDeps          bool
+	skipGoVersionCheck bool
+
+	boilerplate project.Boilerplate
+	project project.Project
+
+	// deprecated flags
 	dep                bool
 	depFlag            *flag.Flag
 	depArgs            []string
-	skipGoVersionCheck bool
+
+	// final result
+	scaffolder scaffold.ProjectScaffolder
 }
 
 func (o *projectOptions) bindCmdlineFlags(cmd *cobra.Command) {
-	cmd.Flags().BoolVar(
-		&o.skipGoVersionCheck, "skip-go-version-check", false, "if specified, skip checking the Go version")
-	cmd.Flags().BoolVar(
-		&o.dep, "dep", true, "if specified, determines whether dep will be used.")
+	cmd.Flags().BoolVar(&o.skipGoVersionCheck, "skip-go-version-check", false, "if specified, skip checking the Go version")
+
+	// dependency args
+	cmd.Flags().BoolVar(&o.fetchDeps, "fetch-deps", true, "ensure dependencies are downloaded")
+
+	// deprecated dependency args
+	cmd.Flags().BoolVar(&o.dep, "dep", true, "if specified, determines whether dep will be used.")
 	o.depFlag = cmd.Flag("dep")
 	cmd.Flags().StringArrayVar(&o.depArgs, "depArgs", nil, "Additional arguments for dep")
+	cmd.Flags().MarkDeprecated("dep", "use the fetch-deps flag instead")
+	cmd.Flags().MarkDeprecated("depArgs", "will be removed with version 1 scaffolding")
 
-	o.bindBoilerplateFlags(cmd)
-	o.bindProjectFlags(cmd)
-}
+	// boilerplate args
+	cmd.Flags().StringVar(&o.boilerplate.Path, "path", "", "path for boilerplate")
+	cmd.Flags().StringVar(&o.boilerplate.License, "license", "apache2", "license to use to boilerplate.  May be one of apache2,none")
+	cmd.Flags().StringVar(&o.boilerplate.Owner, "owner", "", "Owner to add to the copyright")
 
-// projectForFlags registers flags for Project fields and returns the Project
-func (o *projectOptions) bindProjectFlags(cmd *cobra.Command) {
-	p := &o.projectScaffolder.Info
-	cmd.Flags().StringVar(&p.Repo, "repo", "", "name of the github repo.  "+
+	// project args
+	cmd.Flags().StringVar(&o.project.Repo, "repo", util.Repo, "name of the github repo.  "+
 		"defaults to the go package of the current working directory.")
-	cmd.Flags().StringVar(&p.Domain, "domain", "k8s.io", "domain for groups")
-	cmd.Flags().StringVar(&p.Version, "project-version", project.Version1, "project version")
-}
-
-// bindBoilerplateFlags registers flags for Boilerplate fields and returns the Boilerplate
-func (o *projectOptions) bindBoilerplateFlags(cmd *cobra.Command) {
-	bp := &o.projectScaffolder.Boilerplate
-	cmd.Flags().StringVar(&bp.Path, "path", "", "path for boilerplate")
-	cmd.Flags().StringVar(&bp.License, "license", "apache2", "license to use to boilerplate.  Maybe one of apache2,none")
-	cmd.Flags().StringVar(&bp.Owner, "owner", "", "Owner to add to the copyright")
+	cmd.Flags().StringVar(&o.project.Domain, "domain", "k8s.io", "domain for groups")
+	cmd.Flags().StringVar(&o.project.Version, "project-version", project.Version1, "project version")
 }
 
 func (o *projectOptions) initializeProject() {
-
 	if err := o.validate(); err != nil {
 		log.Fatal(err)
 	}
 
-	if err := o.projectScaffolder.Scaffold(); err != nil {
+	if err := o.scaffolder.Scaffold(); err != nil {
 		log.Fatalf("error scaffolding project: %v", err)
 	}
 
@@ -132,8 +133,30 @@ func (o *projectOptions) validate() error {
 		}
 	}
 
-	if !depExists() {
-		return fmt.Errorf("Dep is not installed. Follow steps at: https://golang.github.io/dep/docs/installation.html")
+	switch o.project.Version {
+	case project.Version1:
+		var defEnsure *bool
+		if o.depFlag.Changed {
+			defEnsure = &o.dep
+		}
+		o.scaffolder = &scaffold.V1Project{
+			Project: o.project,
+			Boilerplate: o.boilerplate,
+
+			DepArgs: o.depArgs,
+			DefinitelyEnsure: defEnsure,
+		}
+	case project.Version2:
+		o.scaffolder = &scaffold.V2Project{
+			Project: o.project,
+			Boilerplate: o.boilerplate,
+		}
+	default:
+		return fmt.Errorf("unknown project version %v", o.project.Version)
+	}
+
+	if err := o.scaffolder.Validate(); err != nil {
+		return err
 	}
 
 	if util.ProjectExist() {
@@ -193,39 +216,27 @@ func checkGoVersion(verStr string) error {
 	return nil
 }
 
-func depExists() bool {
-	_, err := exec.LookPath("dep")
-	return err == nil
-}
-
 func (o *projectOptions) postScaffold() error {
-	if !o.depFlag.Changed {
-		reader := bufio.NewReader(os.Stdin)
-		fmt.Println("Run `dep ensure` to fetch dependencies (Recommended) [y/n]?")
-		o.dep = util.Yesno(reader)
-	}
-	if o.dep {
-		c := exec.Command("dep", "ensure") // #nosec
-		if len(o.depArgs) > 0 {
-			c.Args = append(c.Args, o.depArgs...)
-		}
-		c.Stderr = os.Stderr
-		c.Stdout = os.Stdout
-		fmt.Println(strings.Join(c.Args, " "))
-		if err := c.Run(); err != nil {
-			return err
-		}
+	// preserve old "ask if not explicitly set" behavior for the `--dep` flag
+	// (asking is handled by the v1 scaffolder)
+	if (o.depFlag.Changed && !o.dep) || !o.fetchDeps {
+		fmt.Println("Skipping fetching dependencies.")
+		return nil
+	}
 
-		fmt.Println("Running make...")
-		c = exec.Command("make") // #nosec
-		c.Stderr = os.Stderr
-		c.Stdout = os.Stdout
-		fmt.Println(strings.Join(c.Args, " "))
-		if err := c.Run(); err != nil {
-			return err
-		}
-	} else {
-		fmt.Println("Skipping `dep ensure`.  Dependencies will not be fetched.")
+	ensured, err := o.scaffolder.EnsureDependencies()
+	if err != nil {
+		return err
 	}
-	return nil
+
+	if !ensured {
+		return nil
+	}
+
+	fmt.Println("Running make...")
+	c := exec.Command("make") // #nosec
+	c.Stderr = os.Stderr
+	c.Stdout = os.Stdout
+	fmt.Println(strings.Join(c.Args, " "))
+	return c.Run()
 }

cmd/main.go

@@ -17,37 +17,95 @@ limitations under the License.
 package main
 
 import (
-	gobuild "go/build"
-	"log"
+	"fmt"
 	"os"
-	"regexp"
+	"os/exec"
+	"log"
+	"encoding/json"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/tools/go/packages"
 
-	toolsutil "sigs.k8s.io/controller-tools/pkg/crd/util"
 	"sigs.k8s.io/kubebuilder/cmd/util"
 	"sigs.k8s.io/kubebuilder/cmd/version"
+	"sigs.k8s.io/kubebuilder/pkg/scaffold"
 )
 
-func main() {
-	gopath := gobuild.Default.GOPATH
+// module and goMod arg just enough of the output of `go mod edit -json` for our purposes
+type goMod struct {
+	Module module
+}
+type module struct {
+	Path string
+}
 
-	wd, err := os.Getwd()
+// findGoModulePath finds the path of the current module, if present.
+func findGoModulePath(forceModules bool) (string, error) {
+	cmd := exec.Command("go", "mod", "edit", "-json")
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	if forceModules {
+		cmd.Env = append(cmd.Env, "GO111MODULE=on" /* turn on modules just for these commands */)
+	}
+	out, err := cmd.Output()
 	if err != nil {
-		log.Fatal(err)
+		if exitErr, isExitErr := err.(*exec.ExitError); isExitErr {
+			err = fmt.Errorf("%s", string(exitErr.Stderr))
+		}
+		return "", err
+	}
+	mod := goMod{}
+	if err := json.Unmarshal(out, &mod); err != nil {
+		return "", err
+	}
+	return mod.Module.Path, nil
+}
+
+// findCurrentRepo attempts to determine the current repository
+// though a combination of go/packages and `go mod` commands/tricks.
+func findCurrentRepo() (string, error) {
+	// easiest case: project file already exists
+	projFile, err := scaffold.LoadProjectFile("PROJECT")
+	if err == nil {
+		return projFile.Repo, nil
+	}
+
+	// next easy case: existing go module
+	path, err := findGoModulePath(false)
+	if err == nil {
+		return path, nil
+	}
+
+	// next, check if we've got a package in the current directory
+	pkgCfg := &packages.Config{
+		Mode: packages.NeedName,  // name gives us path as well
+	}
+	pkgs, err := packages.Load(pkgCfg, ".")
+	if err == nil && len(pkgs) > 0 {
+		return pkgs[0].PkgPath, nil
 	}
 
-	if !toolsutil.IsUnderGoSrcPath(wd) {
-		log.Fatalf("kubebuilder must be run from the project root under $GOPATH/src/<package>. "+
-			"\nCurrent GOPATH=%s.  \nCurrent directory=%s", gopath, wd)
+	// otherwise, try to get `go mod init` to guess for us -- it's pretty good
+	cmd := exec.Command("go", "mod", "init")
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "GO111MODULE=on" /* turn on modules just for these commands */)
+	if _, err := cmd.Output(); err != nil {
+		if exitErr, isExitErr := err.(*exec.ExitError); isExitErr {
+			err = fmt.Errorf("%s", string(exitErr.Stderr))
+		}
+		// give up, let the user figure it out
+		return "", fmt.Errorf("could not determine repository path from module data, package data, or by initializing a module: %v", err)
 	}
-	util.Repo, err = toolsutil.DirToGoPkg(wd)
+	defer os.Remove("go.mod") // clean up after ourselves
+	return findGoModulePath(true)
+}
+
+func main() {
+	repoPath, err := findCurrentRepo()
 	if err != nil {
-		log.Fatal(err)
+		log.Fatal(fmt.Errorf("error finding current repository: %v", err))
 	}
 
-	re := regexp.MustCompile(`(^.*\/src)(\/.*$)`)
-	util.GoSrc = re.ReplaceAllString(wd, "$1")
+	util.Repo = repoPath
 
 	rootCmd := defaultCommand()
 

cmd/util/util.go

@@ -34,7 +34,6 @@ import (
 
 var Domain string
 var Repo string
-var GoSrc string
 
 // writeIfNotFound returns true if the file was created and false if it already exists
 func WriteIfNotFound(path, templateName, templateValue string, data interface{}) bool {

common.sh

@@ -126,6 +126,9 @@ function fetch_tools {
     curl -sL ${kb_tools_download_url} -o "$kb_tools_archive_path"
   fi
   tar -zvxf "$kb_tools_archive_path" -C "$tmp_root/"
+
+  header_text "fetching controller-gen from source (till it's packaged)"
+  GO111MODULE=on go get sigs.k8s.io/controller-tools/cmd/controller-gen@crdgeneratortmp
 }
 
 function build_kb {

generate_vendor.sh

@@ -34,7 +34,7 @@ fi
 
 build_kb() {
 	rm -f /tmp/kb && \
-	go build -o /tmp/kb sigs.k8s.io/kubebuilder/cmd
+	GO111MODULE=on go build -o /tmp/kb sigs.k8s.io/kubebuilder/cmd
 }
 
 

generated_golden.sh

@@ -16,6 +16,7 @@
 
 set -e
 
+source common.sh
 
 build_kb() {
 	go build -o ./bin/kubebuilder sigs.k8s.io/kubebuilder/cmd
@@ -34,14 +35,16 @@ scaffold_test_project() {
 	rm -rf ./testdata/$project/*
 	pushd . 
 	cd testdata/$project
-	# untar Gopkg.lock and vendor directory for appropriate project version
-	tar -zxf ../vendor.v$version.tgz
 
 	kb=$testdata_dir/../bin/kubebuilder
 
     oldgopath=$GOPATH
 	if [ $version == "1" ]; then
+        export GO111MODULE=auto
         export GOPATH=$(pwd)/../.. # go ignores vendor under testdata, so fake out a gopath
+        # untar Gopkg.lock and vendor directory for appropriate project version
+        tar -zxf $testdata_dir/vendor.v$version.tgz
+
         $kb init --project-version $version --domain testproject.org --license apache2 --owner "The Kubernetes authors" --dep=false
 		$kb create api --group crew --version v1 --kind FirstMate --controller=true --resource=true --make=false
 		$kb alpha webhook --group crew --version v1 --kind FirstMate --type=mutating --operations=create,update --make=false
@@ -54,6 +57,11 @@ scaffold_test_project() {
 		$kb alpha webhook --group core --version v1 --kind Namespace --type=mutating --operations=update --make=false
 		$kb create api --group policy --version v1beta1 --kind HealthCheckPolicy --example=false --controller=true --resource=true --namespaced=false --make=false
 	elif [ $version == "2" ]; then
+        export GO111MODULE=on
+        go get sigs.k8s.io/controller-tools/cmd/controller-gen@crdgeneratortmp
+        export PATH=$PATH:$(go env GOPATH)/bin
+        go mod init sigs.k8s.io/kubebuilder/testdata/project_v2  # our repo autodetection will traverse up to the kb module if we don't do this
+
         $kb init --project-version $version --domain testproject.org --license apache2 --owner "The Kubernetes authors"
 		$kb create api --group crew --version v1 --kind Captain --controller=true --resource=true --make=false
 		$kb create api --group crew --version v1 --kind FirstMate --controller=true --resource=true --make=false
@@ -66,7 +74,7 @@ scaffold_test_project() {
 		$kb alpha webhook --group core --version v1 --kind Namespace --type=mutating --operations=update --make=false
 		# $kb create api --group policy --version v1beta1 --kind HealthCheckPolicy --example=false --controller=true --resource=true --namespaced=false --make=false
 	fi
-	make
+	make all test # v2 doesn't test by default
 	rm -f Gopkg.lock
 	rm -rf ./vendor
 	rm -rf ./bin