merge changes

bundlegen/proxybundle/proxybundle.go

@@ -16,20 +16,31 @@ package proxybundle
 
 import (
 	"archive/zip"
+	"context"
+	"errors"
+	"fmt"
 	"io"
+	"net/http"
 	"os"
+	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
+	"github.com/google/go-github/github"
 	genapi "github.com/srinandan/apigeecli/bundlegen"
 	apiproxy "github.com/srinandan/apigeecli/bundlegen/apiproxydef"
 	policies "github.com/srinandan/apigeecli/bundlegen/policies"
 	proxies "github.com/srinandan/apigeecli/bundlegen/proxies"
 	target "github.com/srinandan/apigeecli/bundlegen/targetendpoint"
+	"github.com/srinandan/apigeecli/clilog"
+	"golang.org/x/oauth2"
 )
 
+const rootDir = "apiproxy"
+
 func GenerateAPIProxyBundle(name string, content string, fileName string, resourceType string, skipPolicy bool, addCORS bool) (err error) {
-	const rootDir = "apiproxy"
+
 	var apiProxyData, proxyEndpointData, targetEndpointData string
 
 	if err = os.Mkdir(rootDir, os.ModePerm); err != nil {
@@ -143,21 +154,27 @@ func GenerateArchiveBundle(pathToZip, destinationPath string) error {
 	return archiveBundle(pathToZip, destinationPath)
 }
 
-func archiveBundle(pathToZip, destinationPath string) error {
-	destinationFile, err := os.Create(destinationPath)
+func archiveBundle(pathToZip, destinationPath string) (err error) {
+
+	var destinationFile *os.File
+
+	destinationFile, err = os.Create(destinationPath)
 	if err != nil {
 		return err
 	}
+
 	myZip := zip.NewWriter(destinationFile)
 	err = filepath.Walk(pathToZip, func(filePath string, info os.FileInfo, err error) error {
 		if info.IsDir() {
-			return nil
+			relPath := strings.TrimPrefix(filePath, filepath.Dir(pathToZip))
+			_, err = myZip.Create(strings.TrimPrefix(relPath, "/") + "/")
+			return err
 		}
 		if err != nil {
 			return err
 		}
 		relPath := strings.TrimPrefix(filePath, filepath.Dir(pathToZip))
-		zipFile, err := myZip.Create(relPath)
+		zipFile, err := myZip.Create(strings.TrimPrefix(relPath, "/"))
 		if err != nil {
 			return err
 		}
@@ -180,3 +197,138 @@ func archiveBundle(pathToZip, destinationPath string) error {
 	}
 	return nil
 }
+
+func GitHubImportBundle(owner string, repo string, repopath string) (err error) {
+
+	//clean up any files or folders
+	CleanUp()
+	os.RemoveAll(rootDir)
+
+	//
+	token := os.Getenv("GITHUB_TOKEN")
+	ctx := context.Background()
+	ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
+	tc := oauth2.NewClient(ctx, ts)
+	client := github.NewClient(tc)
+
+	//1. download the proxy
+	if err := downloadProxyFromRepo(client, ctx, owner, repo, repopath); err != nil {
+		return err
+	}
+
+	if client != nil {
+		fmt.Println("")
+	}
+
+	//2. compress the proxy folder
+	curDir, _ := os.Getwd()
+	if err := archiveBundle(path.Join(curDir, rootDir), path.Join(curDir, rootDir+".zip")); err != nil {
+		return err
+	}
+
+	defer os.RemoveAll(rootDir) // clean up
+	return err
+}
+
+func CleanUp() {
+	if _, err := os.Stat(rootDir + ".zip"); err == nil {
+		_ = os.Remove(rootDir + ".zip")
+	}
+}
+
+func downloadProxyFromRepo(client *github.Client, ctx context.Context, owner string, repo string, repopath string) (err error) {
+
+	var fileContent *github.RepositoryContent
+	var directoryContents []*github.RepositoryContent
+
+	if fileContent, directoryContents, _, err = client.Repositories.GetContents(ctx, owner, repo, repopath, nil); err != nil {
+		return err
+	}
+
+	if fileContent != nil {
+		if err = downloadResource(*fileContent.Path, *fileContent.DownloadURL); err != nil {
+			return err
+		}
+	}
+
+	if len(directoryContents) > 0 {
+		for _, directoryContent := range directoryContents {
+			if *directoryContent.Type == "dir" {
+				if err = downloadProxyFromRepo(client, ctx, owner, repo, path.Join(repopath, *directoryContent.Name)); err != nil {
+					return err
+				}
+			} else if *directoryContent.Type == "file" {
+				if err = downloadResource(*directoryContent.Path, *directoryContent.DownloadURL); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func getApiProxyFolder(repoPath string) (apiProxyFolder string, apiProxyFile string) {
+	re := regexp.MustCompile(`(\S*)?(\/?)apiproxy`)
+
+	apiProxyFileBytes := re.ReplaceAll([]byte(repoPath), []byte(rootDir))
+	apiProxyFile = string(apiProxyFileBytes)
+
+	apiProxyFolder = filepath.Dir(apiProxyFile)
+	return apiProxyFolder, apiProxyFile
+}
+
+//downloadResource method is used to download resources, proxy bundles, sharedflows
+func downloadResource(repoPath string, url string) (err error) {
+
+	var apiproxyFolder, apiproxyFile string
+
+	if apiproxyFolder, apiproxyFile = getApiProxyFolder(repoPath); err != nil {
+		return err
+	}
+
+	_ = os.MkdirAll(apiproxyFolder, 0755)
+
+	out, err := os.Create(apiproxyFile)
+	if err != nil {
+		clilog.Info.Println("error creating file: ", err)
+		return err
+	}
+	defer out.Close()
+
+	client := &http.Client{}
+
+	clilog.Info.Println("Connecting to : ", url)
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		clilog.Info.Println("error in client: ", err)
+		return err
+	}
+
+	resp, err := client.Do(req)
+
+	if err != nil {
+		clilog.Info.Println("error connecting: ", err)
+		return err
+	} else if resp.StatusCode > 299 {
+		clilog.Info.Println("error in response: ", resp.Body)
+		return errors.New("error in response")
+	}
+
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+
+	if resp == nil {
+		clilog.Info.Println("error in response: Response was null")
+		return fmt.Errorf("error in response: Response was null")
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		clilog.Info.Println("error writing response to file: ", err)
+		return err
+	}
+
+	clilog.Info.Println("Resource " + apiproxyFolder + " completed")
+	return nil
+}

cmd/apis/crtapi.go

@@ -16,6 +16,8 @@ package apis
 
 import (
 	"fmt"
+	"os"
+	"regexp"
 
 	"github.com/spf13/cobra"
 	"github.com/srinandan/apigeecli/apiclient"
@@ -30,12 +32,17 @@ var CreateCmd = &cobra.Command{
 	Short: "Creates an API proxy in an Apigee Org",
 	Long:  "Creates an API proxy in an Apigee Org",
 	Args: func(cmd *cobra.Command, args []string) (err error) {
-		if proxy != "" && (oasFile != "" || oasURI != "") {
-			return fmt.Errorf("Importing a bundle (--proxy) cannot be combined with importing via an OAS file")
+
+		if useGitHub, err = gitHubValidations(); err != nil {
+			return err
+		}
+
+		if proxy != "" && (oasFile != "" || oasURI != "" || useGitHub) {
+			return fmt.Errorf("Importing a bundle (--proxy) cannot be combined with importing via an OAS file or GitHub import")
 		}
 
-		if proxy != "" && (gqlFile != "" || gqlURI != "") {
-			return fmt.Errorf("Importing a bundle (--proxy) cannot be combined with importing via an GraphQL file")
+		if proxy != "" && (gqlFile != "" || gqlURI != "" || useGitHub) {
+			return fmt.Errorf("Importing a bundle (--proxy) cannot be combined with importing via an GraphQL file or GitHub import")
 		}
 
 		if oasFile != "" && oasURI != "" {
@@ -46,10 +53,27 @@ var CreateCmd = &cobra.Command{
 			return fmt.Errorf("Cannot combine importing a GraphQL schema through a file and URI")
 		}
 
+		if useGitHub && (oasFile != "" || oasURI != "") {
+			return fmt.Errorf("Cannot combine importing via OAS document and Github")
+		}
+
+		if useGitHub && (gqlFile != "" || gqlURI != "") {
+			return fmt.Errorf("Cannot combine importing via GraphQL schema and Github")
+		}
+
 		return apiclient.SetApigeeOrg(org)
 	},
 	RunE: func(cmd *cobra.Command, args []string) (err error) {
-		if proxy != "" {
+
+		if useGitHub {
+			if err = proxybundle.GitHubImportBundle(ghOwner, ghRepo, ghPath); err != nil {
+				proxybundle.CleanUp()
+				return err
+			}
+			_, err = apis.CreateProxy(name, bundleName)
+			proxybundle.CleanUp()
+			return err
+		} else if proxy != "" {
 			_, err = apis.CreateProxy(name, proxy)
 		} else if oasFile != "" || oasURI != "" {
 			var content []byte
@@ -85,8 +109,11 @@ var CreateCmd = &cobra.Command{
 	},
 }
 
+const bundleName = "apiproxy.zip"
+
 var proxy, oasFile, oasURI, gqlFile, gqlURI string
-var importProxy, validateSpec, skipPolicy, addCORS, formatValidation bool
+var ghOwner, ghRepo, ghPath string
+var importProxy, validateSpec, skipPolicy, addCORS, useGitHub bool
 
 func init() {
 
@@ -108,6 +135,47 @@ func init() {
 		false, "Add a CORS policy")
 	CreateCmd.Flags().BoolVarP(&formatValidation, "formatValidation", "",
 		true, "disables validation of schema type formats")
+	CreateCmd.Flags().StringVarP(&ghOwner, "gh-owner", "",
+		"", "The github organization or username. ex: In https://github.com/srinandan, srinandan is the user")
+	CreateCmd.Flags().StringVarP(&ghRepo, "gh-repo", "",
+		"", "The github repo name. ex: https://github.com/srinandan/sample-apps, sample-apps is the repo")
+	CreateCmd.Flags().StringVarP(&ghPath, "gh-proxy-path", "",
+		"", "The path in the repo to the apiproxy folder. ex: my-repo/apiproxy")
 
 	_ = CreateCmd.MarkFlagRequired("name")
 }
+
+func gitHubValidations() (bool, error) {
+
+	if ghOwner == "" && ghRepo == "" && ghPath == "" {
+		return false, nil
+	}
+
+	if ghOwner == "" && (ghRepo != "" || ghPath != "") {
+		return false, fmt.Errorf("GitHub Owner must be set along with GitHub Repo and GitHub path")
+	}
+
+	if ghRepo == "" && (ghOwner != "" || ghPath != "") {
+		return false, fmt.Errorf("GitHub repo must be set along with GitHub owner and GitHub path")
+	}
+
+	if ghPath == "" && (ghRepo != "" || ghOwner != "") {
+		return false, fmt.Errorf("GitHub path must be set along with GitHub Repo and GitHub owner")
+	}
+
+	if os.Getenv("GITHUB_TOKEN") == "" {
+		return false, fmt.Errorf("Github access token must be set with this feature")
+	}
+
+	//(\w+)?\/apiproxy$
+	re := regexp.MustCompile(`(\w+)?\/apiproxy$`)
+	if ok := re.Match([]byte(ghPath)); !ok {
+		return false, fmt.Errorf("Github path must end with /apiproxy")
+	}
+
+	if ghOwner != "" {
+		return true, nil
+	}
+
+	return false, nil
+}

go.mod

@@ -5,10 +5,13 @@ go 1.16
 require (
 	github.com/getkin/kin-openapi v0.83.0
 	github.com/ghodss/yaml v1.0.0
+	github.com/google/go-github v17.0.0+incompatible
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/lestrrat-go/jwx v1.0.1
 	github.com/lestrrat/go-jwx v0.0.0-20210302221443-a9d01c1b7121
 	github.com/lestrrat/go-pdebug v0.0.0-20180220043741-569c97477ae8 // indirect
 	github.com/spf13/cobra v1.2.1
 	github.com/thedevsaddam/gojsonq v2.2.2+incompatible
+	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -110,6 +110,7 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -123,7 +124,12 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
+github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
+github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
+github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -343,6 +349,7 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -355,6 +362,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
+golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc=
 golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -474,6 +482,7 @@ golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
@@ -503,6 +512,7 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
@@ -576,6 +586,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=