diff --git a/README.md b/README.md index 9533c5c0..17efc7de 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,42 @@ The Devfile Parser library is a Golang module that: 2. writes to the devfile.yaml with the updated data. 3. generates Kubernetes objects for the various devfile resources. 4. defines util functions for the devfile. +5. downloads resources from a parent devfile if specified in the devfile.yaml + +## Private repository support + +Tokens are required to be set in the following cases: +1. parsing a devfile from a private repository +2. parsing a devfile containing a parent devfile from a private repository [1] +3. parsing a devfile from a private repository containing a parent devfile from a public repository [2] + +Set the token for the repository: +```go +parser.ParserArgs{ + ... + // URL must point to a devfile.yaml + URL: /devfile.yaml + Token: + ... +} +``` +Note: The url must also be set with a supported git provider repo url. + +Minimum token scope required: +1. GitHub: Read access to code +2. GitLab: Read repository +3. Bitbucket: Read repository + +Note: To select token scopes for GitHub, a fine-grained token is required. + +For more information about personal access tokens: +1. [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) +2. [GitLab docs](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#create-a-personal-access-token) +3. [Bitbucket docs](https://support.atlassian.com/bitbucket-cloud/docs/repository-access-tokens/) + +[1] Currently, this works under the assumption that the token can authenticate the devfile and the parent devfile; both devfiles are in the same repository. + +[2] In this scenario, the token will be used to authenticate the main devfile. ## Usage @@ -35,7 +71,6 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g devfile, variableWarning, err := devfilePkg.ParseDevfileAndValidate(parserArgs) ``` - 2. To override the HTTP request and response timeouts for a devfile with a parent reference from a registry URL, specify the HTTPTimeout value in the parser arguments ```go // specify the timeout in seconds @@ -45,7 +80,6 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g } ``` - 3. To get specific content from devfile ```go // To get all the components from the devfile @@ -77,7 +111,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g }, }) ``` - + 4. To get the Kubernetes objects from the devfile, visit [generators.go source file](pkg/devfile/generator/generators.go) ```go // To get a slice of Kubernetes containers of type corev1.Container from the devfile component containers @@ -94,7 +128,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g } deployment := generator.GetDeployment(deployParams) ``` - + 5. To update devfile content ```go // To update an existing component in devfile object @@ -131,20 +165,19 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g ```go // If the devfile object has been created with devfile path already set, can simply call WriteYamlDevfile to writes the devfile err := devfile.WriteYamlDevfile() - - + // To write to a devfile from scratch // create a new DevfileData with a specific devfile version devfileData, err := data.NewDevfileData(devfileVersion) // set schema version devfileData.SetSchemaVersion(devfileVersion) - + // add devfile content use library APIs devfileData.AddComponents([]v1.Component{...}) devfileData.AddCommands([]v1.Commands{...}) ...... - + // create a new DevfileCtx ctx := devfileCtx.NewDevfileCtx(devfilePath) err = ctx.SetAbsPath() @@ -154,10 +187,11 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g Ctx: ctx, Data: devfileData, } - + // write to the devfile on disk err = devfile.WriteYamlDevfile() ``` + 7. To parse the outerloop Kubernetes/OpenShift component's uri or inline content, call the read and parse functions ```go // Read the YAML content @@ -166,6 +200,7 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g // Get the Kubernetes resources resources, err := ParseKubernetesYaml(values) ``` + 8. By default, the parser will set all unset boolean properties to their spec defined default values. Clients can override this behaviour by specifiying the parser argument `SetBooleanDefaults` to false ```go setDefaults := false @@ -174,6 +209,15 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g } ``` +9. When parsing a devfile that contains a parent reference, if the parent uri is a supported git provider repo url with the correct personal access token, all resources from the parent git repo excluding the parent devfile.yaml will be downloaded to the location of the devfile being parsed. **Note: The URL must point to a devfile.yaml** + ```yaml + schemaVersion: 2.2.0 + ... + parent: + uri: /devfile.yaml + ... + ``` + ## Projects using devfile/library The following projects are consuming this library as a Golang dependency diff --git a/pkg/devfile/parser/context/content.go b/pkg/devfile/parser/context/content.go index 3bab1fa0..52b3a45c 100644 --- a/pkg/devfile/parser/context/content.go +++ b/pkg/devfile/parser/context/content.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -69,6 +69,9 @@ func (d *DevfileCtx) SetDevfileContent() error { if d.url != "" { // set the client identifier for telemetry params := util.HTTPRequestParams{URL: d.url, TelemetryClientName: util.TelemetryClientName} + if d.token != "" { + params.Token = d.token + } data, err = util.DownloadInMemory(params) if err != nil { return errors.Wrap(err, "error getting devfile info from url") diff --git a/pkg/devfile/parser/context/context.go b/pkg/devfile/parser/context/context.go index bd2e6d6d..97ab49d3 100644 --- a/pkg/devfile/parser/context/context.go +++ b/pkg/devfile/parser/context/context.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -45,13 +45,16 @@ type DevfileCtx struct { // devfile json schema jsonSchema string - //url path of the devfile + // url path of the devfile url string + // token is a personal access token used with a private git repo URL + token string + // filesystem for devfile fs filesystem.Filesystem - // devfile kubernetes components has been coverted from uri to inlined in memory + // devfile kubernetes components has been converted from uri to inlined in memory convertUriToInlined bool } @@ -150,6 +153,16 @@ func (d *DevfileCtx) GetURL() string { return d.url } +// GetToken func returns current devfile token +func (d *DevfileCtx) GetToken() string { + return d.token +} + +// SetToken sets the token for the devfile +func (d *DevfileCtx) SetToken(token string) { + d.token = token +} + // SetAbsPath sets absolute file path for devfile func (d *DevfileCtx) SetAbsPath() (err error) { // Set devfile absolute path diff --git a/pkg/devfile/parser/context/context_test.go b/pkg/devfile/parser/context/context_test.go index 9fe14cd9..3f332e88 100644 --- a/pkg/devfile/parser/context/context_test.go +++ b/pkg/devfile/parser/context/context_test.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -83,6 +83,20 @@ func TestPopulateFromInvalidURL(t *testing.T) { }) } +func TestNewURLDevfileCtx(t *testing.T) { + var ( + token = "fake-token" + url = "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml" + ) + { + d := NewURLDevfileCtx(url) + assert.Equal(t, "https://github.com/devfile/registry/blob/main/stacks/go/2.0.0/devfile.yaml", d.GetURL()) + assert.Equal(t, "", d.GetToken()) + d.SetToken(token) + assert.Equal(t, "fake-token", d.GetToken()) + } +} + func invalidJsonRawContent200() []byte { return []byte(InvalidDevfileContent) } diff --git a/pkg/devfile/parser/parse.go b/pkg/devfile/parser/parse.go index ca17e253..e22fa0eb 100644 --- a/pkg/devfile/parser/parse.go +++ b/pkg/devfile/parser/parse.go @@ -19,6 +19,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/devfile/library/v2/pkg/git" + "github.com/hashicorp/go-multierror" "io/ioutil" "net/url" "os" @@ -46,6 +48,57 @@ import ( "github.com/pkg/errors" ) +// downloadGitRepoResources is exposed as a global variable for the purpose of running mock tests +var downloadGitRepoResources = func(url string, destDir string, httpTimeout *int, token string) error { + var returnedErr error + + gitUrl, err := git.NewGitUrlWithURL(url) + if err != nil { + return err + } + + if gitUrl.IsGitProviderRepo() { + if !gitUrl.IsFile || gitUrl.Revision == "" || !strings.Contains(gitUrl.Path, OutputDevfileYamlPath) { + return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url) + } + + stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources")) + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + } + }(stackDir) + + if !gitUrl.IsPublic(httpTimeout) { + err = gitUrl.SetToken(token, httpTimeout) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + return returnedErr + } + } + + err = gitUrl.CloneGitRepo(stackDir) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + return returnedErr + } + + dir := path.Dir(path.Join(stackDir, gitUrl.Path)) + err = git.CopyAllDirFiles(dir, destDir) + if err != nil { + returnedErr = multierror.Append(returnedErr, err) + return returnedErr + } + } + + return nil +} + // ParseDevfile func validates the devfile integrity. // Creates devfile context and runtime objects func parseDevfile(d DevfileObj, resolveCtx *resolutionContextTree, tool resolverTools, flattenedDevfile bool) (DevfileObj, error) { @@ -97,6 +150,8 @@ type ParserArgs struct { // RegistryURLs is a list of registry hosts which parser should pull parent devfile from. // If registryUrl is defined in devfile, this list will be ignored. RegistryURLs []string + // Token is a GitHub, GitLab, or Bitbucket personal access token used with a private git repo URL + Token string // DefaultNamespace is the default namespace to use // If namespace is defined under devfile's parent kubernetes object, this namespace will be ignored. DefaultNamespace string @@ -129,6 +184,10 @@ func ParseDevfile(args ParserArgs) (d DevfileObj, err error) { return d, errors.Wrap(err, "the devfile source is not provided") } + if args.Token != "" { + d.Ctx.SetToken(args.Token) + } + tool := resolverTools{ defaultNamespace: args.DefaultNamespace, registryURLs: args.RegistryURLs, @@ -431,17 +490,16 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return DevfileObj{}, fmt.Errorf("failed to resolve parent uri, devfile context is missing absolute url and path to devfile. %s", resolveImportReference(importReference)) } + token := curDevfileCtx.GetToken() d.Ctx = devfileCtx.NewURLDevfileCtx(newUri) - if strings.Contains(newUri, "raw.githubusercontent.com") { - urlComponents, err := util.GetGitUrlComponentsFromRaw(newUri) - if err != nil { - return DevfileObj{}, err - } - destDir := path.Dir(curDevfileCtx.GetAbsPath()) - err = getResourcesFromGit(urlComponents, destDir) - if err != nil { - return DevfileObj{}, err - } + if token != "" { + d.Ctx.SetToken(token) + } + + destDir := path.Dir(curDevfileCtx.GetAbsPath()) + err = downloadGitRepoResources(newUri, destDir, tool.httpTimeout, token) + if err != nil { + return DevfileObj{}, err } } importReference.Uri = newUri @@ -450,27 +508,6 @@ func parseFromURI(importReference v1.ImportReference, curDevfileCtx devfileCtx.D return populateAndParseDevfile(d, newResolveCtx, tool, true) } -func getResourcesFromGit(gitUrlComponents map[string]string, destDir string) error { - stackDir, err := ioutil.TempDir(os.TempDir(), fmt.Sprintf("git-resources")) - if err != nil { - return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) - } - defer os.RemoveAll(stackDir) - - err = util.CloneGitRepo(gitUrlComponents, stackDir) - if err != nil { - return err - } - - dir := path.Dir(path.Join(stackDir, gitUrlComponents["file"])) - err = util.CopyAllDirFiles(dir, destDir) - if err != nil { - return err - } - - return nil -} - func parseFromRegistry(importReference v1.ImportReference, resolveCtx *resolutionContextTree, tool resolverTools) (d DevfileObj, err error) { id := importReference.Id registryURL := importReference.RegistryUrl @@ -839,6 +876,9 @@ func getKubernetesDefinitionFromUri(uri string, d devfileCtx.DevfileCtx) ([]byte newUri = uri } params := util.HTTPRequestParams{URL: newUri} + if d.GetToken() != "" { + params.Token = d.GetToken() + } data, err = util.DownloadInMemory(params) if err != nil { return nil, errors.Wrapf(err, "error getting kubernetes resources definition information") diff --git a/pkg/devfile/parser/parse_test.go b/pkg/devfile/parser/parse_test.go index bdc5edae..0971f8d9 100644 --- a/pkg/devfile/parser/parse_test.go +++ b/pkg/devfile/parser/parse_test.go @@ -16,15 +16,18 @@ package parser import ( + "bytes" "context" "fmt" v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2" + "github.com/devfile/library/v2/pkg/git" "io/ioutil" "net" "net/http" "net/http/httptest" "os" "path" + "path/filepath" "reflect" "strings" "testing" @@ -2858,6 +2861,7 @@ func Test_parseParentAndPluginFromURI(t *testing.T) { tt.args.devFileObj.Data.AddComponents(plugincomp) } + downloadGitRepoResources = mockDownloadGitRepoResources(&git.GitUrl{}, "") err := parseParentAndPlugin(tt.args.devFileObj, &resolutionContextTree{}, resolverTools{}) // Unexpected error @@ -3068,11 +3072,16 @@ func Test_parseParentAndPlugin_RecursivelyReference(t *testing.T) { testK8sClient := &testingutil.FakeK8sClient{ DevWorkspaceResources: devWorkspaceResources, } + + httpTimeout := 0 + tool := resolverTools{ - k8sClient: testK8sClient, - context: context.Background(), + k8sClient: testK8sClient, + context: context.Background(), + httpTimeout: &httpTimeout, } + downloadGitRepoResources = mockDownloadGitRepoResources(&git.GitUrl{}, "") err := parseParentAndPlugin(devFileObj, &resolutionContextTree{}, tool) // devfile has a cycle in references: main devfile -> uri: http://127.0.0.1:8080 -> name: testcrd, namespace: defaultnamespace -> uri: http://127.0.0.1:8090 -> uri: http://127.0.0.1:8080 expectedErr := fmt.Sprintf("devfile has an cycle in references: main devfile -> uri: %s%s -> name: %s, namespace: %s -> uri: %s%s -> uri: %s%s", httpPrefix, uri1, name, namespace, @@ -4143,6 +4152,7 @@ func Test_parseFromURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + downloadGitRepoResources = mockDownloadGitRepoResources(&git.GitUrl{}, "") got, err := parseFromURI(tt.importReference, tt.curDevfileCtx, &resolutionContextTree{}, resolverTools{}) if (err != nil) != (tt.wantErr != nil) { t.Errorf("Test_parseFromURI() unexpected error: %v, wantErr %v", err, tt.wantErr) @@ -4155,6 +4165,291 @@ func Test_parseFromURI(t *testing.T) { } } +func Test_parseFromURI_GitProviders(t *testing.T) { + const ( + invalidToken = "invalid-token" + validToken = "valid-token" + invalidRevision = "invalid-revision" + ) + + minimalDevfileContent := fmt.Sprintf("schemaVersion: 2.2.0\nmetadata:\n name: devfile") + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + _, err := rw.Write([]byte(minimalDevfileContent)) + if err != nil { + t.Error(err) + } + })) + defer server.Close() + + minimalDevfile := DevfileObj{ + Ctx: devfileCtx.NewURLDevfileCtx(OutputDevfileYamlPath), + Data: &v2.DevfileV2{ + Devfile: v1.Devfile{ + DevfileHeader: devfilepkg.DevfileHeader{ + SchemaVersion: schemaVersion, + Metadata: devfilepkg.DevfileMetadata{ + Name: "devfile", + }, + }, + }, + }, + } + + validGitUrl := &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + Path: "devfile.yaml", + IsFile: true, + } + + invalidTokenError := "failed to clone repo with token, ensure that the url and token is correct" + invalidGitSwitchError := "failed to switch repo to revision*" + invalidDevfilePathError := "error getting devfile from url: failed to retrieve*" + + tests := []struct { + name string + gitUrl *git.GitUrl + token string + destDir string + importReference v1.ImportReference + wantDevFile DevfileObj + wantError *string + wantResources []string + wantResourceContent []byte + }{ + { + name: "private parent devfile", + gitUrl: validGitUrl, + token: validToken, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantDevFile: minimalDevfile, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("private repo\ngit switched"), + }, + { + name: "public parent devfile", + gitUrl: validGitUrl, + token: "", + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantDevFile: minimalDevfile, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("public repo\ngit switched"), + }, + { + // a valid parent url must contain a revision + name: "private parent devfile without a revision", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: "", + Path: "devfile.yaml", + IsFile: true, + }, + token: validToken, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: &invalidDevfilePathError, + wantResources: []string{}, + }, + { + name: "public parent devfile with no devfile path", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + IsFile: false, + }, + token: "", + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: &invalidDevfilePathError, + wantResources: []string{}, + }, + { + name: "public parent devfile with invalid devfile path", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + Path: "text.txt", + IsFile: true, + }, + token: "", + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: &invalidDevfilePathError, + wantResources: []string{}, + }, + { + name: "private parent devfile with invalid token", + gitUrl: validGitUrl, + token: invalidToken, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: &invalidTokenError, + wantResources: []string{}, + }, + { + name: "private parent devfile with invalid revision", + gitUrl: &git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: invalidRevision, + Path: "devfile.yaml", + IsFile: true, + }, + token: validToken, + importReference: v1.ImportReference{ + ImportReferenceUnion: v1.ImportReferenceUnion{ + Uri: server.URL, + }, + }, + wantError: &invalidGitSwitchError, + wantResources: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + destDir := t.TempDir() + curDevfileContext := devfileCtx.NewDevfileCtx(path.Join(destDir, OutputDevfileYamlPath)) + err := curDevfileContext.SetAbsPath() + if err != nil { + t.Errorf("Unexpected err: %+v", err) + } + + // tt.gitUrl is the parent devfile URL + downloadGitRepoResources = mockDownloadGitRepoResources(tt.gitUrl, tt.token) + got, err := parseFromURI(tt.importReference, curDevfileContext, &resolutionContextTree{}, resolverTools{}) + + // validate even if we want an error; check that no files are copied to destDir + validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir) + + if (err != nil) != (tt.wantError != nil) { + t.Errorf("Unexpected error: %v, wantErr %v", err, tt.wantError) + } else if err == nil && !reflect.DeepEqual(got.Data, tt.wantDevFile.Data) { + t.Errorf("Wanted: %v, got: %v, difference at %v", tt.wantDevFile, got, pretty.Compare(tt.wantDevFile, got)) + } else if err != nil { + assert.Regexp(t, *tt.wantError, err.Error(), "Error message should match") + } + }) + } +} + +// copied from: https://github.com/devfile/registry-support/blob/main/registry-library/library/library_test.go#L1118 +func validateGitResourceFunctions(t *testing.T, wantFiles []string, wantResourceContent []byte, path string) { + wantNumFiles := len(wantFiles) + files, err := os.ReadDir(path) + if err != nil { + if wantNumFiles != 0 { + t.Errorf("error reading directory %s", path) + } + } else { + // verify only the expected number of files are downloaded + gotNumFiles := len(files) + if gotNumFiles != wantNumFiles { + t.Errorf("The number of downloaded files do not match, want %d got %d", wantNumFiles, gotNumFiles) + } + // verify the expected resources are copied to the dest directory + for _, wantFile := range wantFiles { + if _, err = os.Stat(path + "/" + wantFile); err != nil && os.IsNotExist(err) { + t.Errorf("file %s should exist ", wantFile) + } + } + + // verify contents of resource file; don't need to check if wantResourceContent is nil + if wantResourceContent != nil { + resourceContent, err := os.ReadFile(filepath.Clean(path) + "/resource.file") + if err != nil { + t.Errorf("failed to open test resource: %v", err) + } + if !bytes.Equal(resourceContent, wantResourceContent) { + t.Errorf("Wanted resource content:\n%v\ngot:\n%v\ndifference at\n%v", wantResourceContent, resourceContent, pretty.Compare(string(wantResourceContent), string(resourceContent))) + } + } + } +} + +func mockDownloadGitRepoResources(gURL *git.GitUrl, mockToken string) func(url string, destDir string, httpTimeout *int, token string) error { + return func(url string, destDir string, httpTimeout *int, token string) error { + // this converts the real git URL to a mock URL + mockGitUrl := git.MockGitUrl{ + Protocol: gURL.Protocol, + Host: gURL.Host, + Owner: gURL.Owner, + Repo: gURL.Repo, + Revision: gURL.Revision, + Path: gURL.Path, + IsFile: gURL.IsFile, + } + + if mockGitUrl.IsGitProviderRepo() { + if !mockGitUrl.IsFile || mockGitUrl.Revision == "" || !strings.Contains(mockGitUrl.Path, OutputDevfileYamlPath) { + return fmt.Errorf("error getting devfile from url: failed to retrieve %s", url+"/"+mockGitUrl.Path) + } + + stackDir, err := os.MkdirTemp("", fmt.Sprintf("git-resources")) + if err != nil { + return fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + + defer func(path string) { + err := os.RemoveAll(path) + if err != nil { + err = fmt.Errorf("failed to create dir: %s, error: %v", stackDir, err) + } + }(stackDir) + + err = mockGitUrl.SetToken(mockToken) + if err != nil { + return err + } + + err = mockGitUrl.CloneGitRepo(stackDir) + if err != nil { + return err + } + + err = git.CopyAllDirFiles(stackDir, destDir) + if err != nil { + return err + } + } + + return nil + } +} + func Test_parseFromRegistry(t *testing.T) { const ( registry = "127.0.0.1:8080" @@ -4489,51 +4784,70 @@ func Test_parseFromKubeCRD(t *testing.T) { } } -func Test_getResourcesFromGit(t *testing.T) { - destDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create dest dir: %s, error: %v", destDir, err) +func Test_DownloadGitRepoResources(t *testing.T) { + httpTimeout := 0 + + validGitUrl := git.GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "registry", + Revision: "main", + Path: "stacks/python/3.0.0/devfile.yaml", + IsFile: true, } - defer os.RemoveAll(destDir) - invalidGitUrl := map[string]string{ - "username": "devfile", - "project": "nonexistent", - "branch": "nonexistent", - } - validGitUrl := map[string]string{ - "host": "raw.githubusercontent.com", - "username": "devfile", - "project": "registry", - "branch": "main", - "file": "stacks/nodejs/devfile.yaml", - } + invalidTokenErr := "failed to clone repo with token, ensure that the url and token is correct" tests := []struct { - name string - gitUrlComponents map[string]string - destDir string - wantErr bool + name string + url string + gitUrl git.GitUrl + destDir string + token string + wantErr bool + wantResources []string + wantResourceContent []byte }{ { - name: "should fail with invalid git url", - gitUrlComponents: invalidGitUrl, - destDir: path.Join(os.TempDir(), "nonexistent"), - wantErr: true, + name: "should be able to get resources with valid token", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "valid-token", + wantErr: false, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("private repo\ngit switched"), }, { - name: "should be able to get resources from valid git url", - gitUrlComponents: validGitUrl, - destDir: destDir, - wantErr: false, + name: "should be able to get resources from public repo (empty token)", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "", + wantErr: false, + wantResources: []string{"resource.file"}, + wantResourceContent: []byte("public repo\ngit switched"), + }, + { + name: "should fail to get resources with invalid token", + url: "https://raw.githubusercontent.com/devfile/registry/main/stacks/python/3.0.0/devfile.yaml", + gitUrl: validGitUrl, + token: "invalid-token", + wantErr: true, + wantResources: []string{}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := getResourcesFromGit(tt.gitUrlComponents, tt.destDir) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) + destDir := t.TempDir() + downloadGitRepoResources = mockDownloadGitRepoResources(&tt.gitUrl, tt.token) + err := downloadGitRepoResources(tt.url, destDir, &httpTimeout, tt.token) + if (err != nil) && (tt.wantErr != true) { + t.Errorf("Unexpected error = %v", err) + } else if tt.wantErr == true { + assert.Containsf(t, err.Error(), invalidTokenErr, "expected error containing %q, got %s", invalidTokenErr, err) + } else { + validateGitResourceFunctions(t, tt.wantResources, tt.wantResourceContent, destDir) } }) } diff --git a/pkg/git/git.go b/pkg/git/git.go new file mode 100644 index 00000000..94c7680e --- /dev/null +++ b/pkg/git/git.go @@ -0,0 +1,372 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + GitHubHost string = "github.com" + RawGitHubHost string = "raw.githubusercontent.com" + GitLabHost string = "gitlab.com" + BitbucketHost string = "bitbucket.org" +) + +type GitUrl struct { + Protocol string // URL scheme + Host string // URL domain name + Owner string // name of the repo owner + Repo string // name of the repo + Revision string // branch name, tag name, or commit id + Path string // path to a directory or file in the repo + token string // authenticates private repo actions for parent devfiles + IsFile bool // defines if the URL points to a file in the repo +} + +// NewGitUrlWithURL NewGitUrl creates a GitUrl from a string url +func NewGitUrlWithURL(url string) (GitUrl, error) { + gitUrl, err := ParseGitUrl(url) + if err != nil { + return gitUrl, err + } + return gitUrl, nil +} + +// ParseGitUrl extracts information from a support git url +// Only supports git repositories hosted on GitHub, GitLab, and Bitbucket +func ParseGitUrl(fullUrl string) (GitUrl, error) { + var g GitUrl + err := ValidateURL(fullUrl) + if err != nil { + return g, err + } + + parsedUrl, err := url.Parse(fullUrl) + if err != nil { + return g, err + } + + if len(parsedUrl.Path) == 0 { + return g, fmt.Errorf("url path should not be empty") + } + + if parsedUrl.Host == RawGitHubHost || parsedUrl.Host == GitHubHost { + err = g.parseGitHubUrl(parsedUrl) + } else if parsedUrl.Host == GitLabHost { + err = g.parseGitLabUrl(parsedUrl) + } else if parsedUrl.Host == BitbucketHost { + err = g.parseBitbucketUrl(parsedUrl) + } else { + err = fmt.Errorf("url host should be a valid GitHub, GitLab, or Bitbucket host; received: %s", parsedUrl.Host) + } + + return g, err +} + +func (g *GitUrl) GetToken() string { + return g.token +} + +type CommandType string + +const ( + GitCommand CommandType = "git" + unsupportedCmdMsg = "Unsupported command \"%s\" " +) + +// Execute is exposed as a global variable for the purpose of running mock tests +// only "git" is supported +/* #nosec G204 -- used internally to execute various git actions and eventual cleanup of artifacts. Calling methods validate user input to ensure commands are used appropriately */ +var execute = func(baseDir string, cmd CommandType, args ...string) ([]byte, error) { + if cmd == GitCommand { + c := exec.Command(string(cmd), args...) + c.Dir = baseDir + output, err := c.CombinedOutput() + return output, err + } + + return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd)) +} + +func (g *GitUrl) CloneGitRepo(destDir string) error { + exist := CheckPathExists(destDir) + if !exist { + return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) + } + + host := g.Host + if host == RawGitHubHost { + host = GitHubHost + } + + var repoUrl string + if g.GetToken() == "" { + repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, host, g.Owner, g.Repo) + } else { + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", g.Protocol, g.GetToken(), host, g.Owner, g.Repo) + if g.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", g.Protocol, g.GetToken(), host, g.Owner, g.Repo) + } + } + + _, err := execute(destDir, "git", "clone", repoUrl, destDir) + + if err != nil { + if g.GetToken() == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private. error: %v", err) + } else { + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct. error: %v", err) + } + } + + if g.Revision != "" { + _, err := execute(destDir, "git", "switch", "--detach", "origin/"+g.Revision) + if err != nil { + err = os.RemoveAll(destDir) + if err != nil { + return err + } + return fmt.Errorf("failed to switch repo to revision. repo dir: %v, revision: %v", destDir, g.Revision) + } + } + + return nil +} + +func (g *GitUrl) parseGitHubUrl(url *url.URL) error { + var splitUrl []string + var err error + + g.Protocol = url.Scheme + g.Host = url.Host + + if g.Host == RawGitHubHost { + g.IsFile = true + // raw GitHub urls don't contain "blob" or "tree" + // https://raw.githubusercontent.com/devfile/library/main/devfile.yaml -> [devfile library main devfile.yaml] + splitUrl = strings.SplitN(url.Path[1:], "/", 4) + if len(splitUrl) == 4 { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + g.Revision = splitUrl[2] + g.Path = splitUrl[3] + } else { + // raw GitHub urls have to be a file + err = fmt.Errorf("raw url path should contain ///, received: %s", url.Path[1:]) + } + return err + } + + if g.Host == GitHubHost { + // https://github.com/devfile/library/blob/main/devfile.yaml -> [devfile library blob main devfile.yaml] + splitUrl = strings.SplitN(url.Path[1:], "/", 5) + if len(splitUrl) < 2 { + err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + } else { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + + // url doesn't contain a path to a directory or file + if len(splitUrl) == 2 { + return nil + } + + switch splitUrl[2] { + case "tree": + g.IsFile = false + case "blob": + g.IsFile = true + default: + return fmt.Errorf("url path to directory or file should contain 'tree' or 'blob'") + } + + // url has a path to a file or directory + if len(splitUrl) == 5 { + g.Revision = splitUrl[3] + g.Path = splitUrl[4] + } else if !g.IsFile && len(splitUrl) == 4 { + g.Revision = splitUrl[3] + } else { + err = fmt.Errorf("url path should contain ////, received: %s", url.Path[1:]) + } + } + } + + return err +} + +func (g *GitUrl) parseGitLabUrl(url *url.URL) error { + var splitFile, splitOrg []string + var err error + + g.Protocol = url.Scheme + g.Host = url.Host + g.IsFile = false + + // GitLab urls contain a '-' separating the root of the repo + // and the path to a file or directory + split := strings.Split(url.Path[1:], "/-/") + + splitOrg = strings.SplitN(split[0], "/", 2) + if len(splitOrg) < 2 { + return fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + } else { + g.Owner = splitOrg[0] + g.Repo = splitOrg[1] + } + + // url doesn't contain a path to a directory or file + if len(split) == 1 { + return nil + } + + // url may contain a path to a directory or file + if len(split) == 2 { + splitFile = strings.SplitN(split[1], "/", 3) + } + + if len(splitFile) == 3 { + if splitFile[0] == "blob" || splitFile[0] == "tree" || splitFile[0] == "raw" { + g.Revision = splitFile[1] + g.Path = splitFile[2] + ext := filepath.Ext(g.Path) + if ext != "" { + g.IsFile = true + } + } else { + err = fmt.Errorf("url path should contain 'blob' or 'tree' or 'raw', received: %s", url.Path[1:]) + } + } else { + return fmt.Errorf("url path to directory or file should contain //, received: %s", url.Path[1:]) + } + + return err +} + +func (g *GitUrl) parseBitbucketUrl(url *url.URL) error { + var splitUrl []string + var err error + + g.Protocol = url.Scheme + g.Host = url.Host + g.IsFile = false + + splitUrl = strings.SplitN(url.Path[1:], "/", 5) + if len(splitUrl) < 2 { + err = fmt.Errorf("url path should contain /, received: %s", url.Path[1:]) + } else if len(splitUrl) == 2 { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + } else { + g.Owner = splitUrl[0] + g.Repo = splitUrl[1] + if len(splitUrl) == 5 { + if splitUrl[2] == "raw" || splitUrl[2] == "src" { + g.Revision = splitUrl[3] + g.Path = splitUrl[4] + ext := filepath.Ext(g.Path) + if ext != "" { + g.IsFile = true + } + } else { + err = fmt.Errorf("url path should contain 'raw' or 'src', received: %s", url.Path[1:]) + } + } else { + err = fmt.Errorf("url path should contain path to directory or file, received: %s", url.Path[1:]) + } + } + + return err +} + +// SetToken validates the token with a get request to the repo before setting the token +// Defaults token to empty on failure. +func (g *GitUrl) SetToken(token string, httpTimeout *int) error { + err := g.validateToken(HTTPRequestParams{Token: token, Timeout: httpTimeout}) + if err != nil { + g.token = "" + return fmt.Errorf("failed to set token. error: %v", err) + } + g.token = token + return nil +} + +// IsPublic checks if the GitUrl is public with a get request to the repo using an empty token +// Returns true if the request succeeds +func (g *GitUrl) IsPublic(httpTimeout *int) bool { + err := g.validateToken(HTTPRequestParams{Token: "", Timeout: httpTimeout}) + if err != nil { + return false + } + return true +} + +// validateToken makes a http get request to the repo with the GitUrl token +// Returns an error if the get request fails +func (g *GitUrl) validateToken(params HTTPRequestParams) error { + var apiUrl string + + switch g.Host { + case GitHubHost, RawGitHubHost: + apiUrl = fmt.Sprintf("https://api.github.com/repos/%s/%s", g.Owner, g.Repo) + case GitLabHost: + apiUrl = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s", g.Owner, g.Repo) + case BitbucketHost: + apiUrl = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s", g.Owner, g.Repo) + default: + apiUrl = fmt.Sprintf("%s://%s/%s/%s.git", g.Protocol, g.Host, g.Owner, g.Repo) + } + + params.URL = apiUrl + res, err := HTTPGetRequest(params, 0) + if len(res) == 0 || err != nil { + return err + } + + return nil +} + +// GitRawFileAPI returns the endpoint for the git providers raw file +func (g *GitUrl) GitRawFileAPI() string { + var apiRawFile string + + switch g.Host { + case GitHubHost, RawGitHubHost: + apiRawFile = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", g.Owner, g.Repo, g.Revision, g.Path) + case GitLabHost: + apiRawFile = fmt.Sprintf("https://gitlab.com/api/v4/projects/%s%%2F%s/repository/files/%s/raw?ref=%s", g.Owner, g.Repo, g.Path, g.Revision) + case BitbucketHost: + apiRawFile = fmt.Sprintf("https://api.bitbucket.org/2.0/repositories/%s/%s/src/%s/%s", g.Owner, g.Repo, g.Revision, g.Path) + } + + return apiRawFile +} + +// IsGitProviderRepo checks if the url matches a repo from a supported git provider +func (g *GitUrl) IsGitProviderRepo() bool { + switch g.Host { + case GitHubHost, RawGitHubHost, GitLabHost, BitbucketHost: + return true + default: + return false + } +} diff --git a/pkg/git/git_test.go b/pkg/git/git_test.go new file mode 100644 index 00000000..09116fef --- /dev/null +++ b/pkg/git/git_test.go @@ -0,0 +1,462 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" + "os" + "path/filepath" + "reflect" + "testing" +) + +func Test_ParseGitUrl(t *testing.T) { + invalidUrlError := "URL is invalid" + invalidUrlPathError := "url path to directory or file should contain*" + missingUserAndRepoError := "url path should contain /*" + + invalidGitHostError := "url host should be a valid GitHub, GitLab, or Bitbucket host*" + invalidGitHubPathError := "url path should contain ////*" + invalidGitHubRawPathError := "raw url path should contain ///*" + + invalidGitLabPathError := "url path to directory or file should contain //*" + missingGitLabKeywordError := "url path should contain 'blob' or 'tree' or 'raw'*" + + invalidBitbucketPathError := "url path should contain path to directory or file*" + missingBitbucketKeywordError := "url path should contain 'raw' or 'src'*" + + tests := []struct { + name string + url string + wantUrl GitUrl + wantErr string + }{ + { + name: "should fail with empty url", + url: "", + wantErr: invalidUrlError, + }, + { + name: "should fail with invalid git host", + url: "https://google.ca/", + wantErr: invalidGitHostError, + }, + // GitHub + { + name: "should parse GitHub repo with root path", + url: "https://github.com/devfile/library", + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Revision: "", + Path: "", + IsFile: false, + }, + }, + { + name: "should parse GitHub repo with root path and tag", + url: "https://github.com/devfile/library/tree/v2.2.0", + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Revision: "v2.2.0", + Path: "", + IsFile: false, + }, + }, + { + name: "should parse GitHub repo with root path and revision", + url: "https://github.com/devfile/library/tree/0ce592a416fb185564516353891a45016ac7f671", + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Revision: "0ce592a416fb185564516353891a45016ac7f671", + Path: "", + IsFile: false, + }, + }, + { + name: "should fail with only GitHub host", + url: "https://github.com/", + wantErr: missingUserAndRepoError, + }, + { + name: "should parse GitHub repo with file path", + url: "https://github.com/devfile/library/blob/main/devfile.yaml", + wantUrl: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + Path: "devfile.yaml", + IsFile: true, + }, + }, + { + name: "should parse GitHub repo with raw file path", + url: "https://raw.githubusercontent.com/devfile/library/main/devfile.yaml", + wantUrl: GitUrl{ + Protocol: "https", + Host: "raw.githubusercontent.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + Path: "devfile.yaml", + IsFile: true, + }, + }, + { + name: "should fail with missing GitHub repo", + url: "https://github.com/devfile", + wantErr: missingUserAndRepoError, + }, + { + name: "should fail with missing GitHub blob", + url: "https://github.com/devfile/library/main/devfile.yaml", + wantErr: invalidUrlPathError, + }, + { + name: "should fail with missing GitHub tree", + url: "https://github.com/devfile/library/main/tests/yamls", + wantErr: invalidUrlPathError, + }, + { + name: "should fail with just GitHub tree", + url: "https://github.com/devfile/library/tree", + wantErr: invalidGitHubPathError, + }, + { + name: "should fail with just GitHub blob", + url: "https://github.com/devfile/library/blob", + wantErr: invalidGitHubPathError, + }, + { + name: "should fail with invalid GitHub raw file path", + url: "https://raw.githubusercontent.com/devfile/library/devfile.yaml", + wantErr: invalidGitHubRawPathError, + }, + // Gitlab + { + name: "should parse GitLab repo with root path", + url: "https://gitlab.com/gitlab-org/gitlab-foss", + wantUrl: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "gitlab-org", + Repo: "gitlab-foss", + Revision: "", + Path: "", + IsFile: false, + }, + }, + { + name: "should fail with only GitLab host", + url: "https://gitlab.com/", + wantErr: missingUserAndRepoError, + }, + { + name: "should parse GitLab repo with file path", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/blob/master/README.md", + wantUrl: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "gitlab-org", + Repo: "gitlab-foss", + Revision: "master", + Path: "README.md", + IsFile: true, + }, + }, + { + name: "should fail with missing GitLab repo", + url: "https://gitlab.com/gitlab-org", + wantErr: missingUserAndRepoError, + }, + { + name: "should fail with missing GitLab keywords", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/master/directory/README.md", + wantErr: missingGitLabKeywordError, + }, + { + name: "should fail with missing GitLab file or directory path", + url: "https://gitlab.com/gitlab-org/gitlab-foss/-/tree/master", + wantErr: invalidGitLabPathError, + }, + // Bitbucket + { + name: "should parse Bitbucket repo with root path", + url: "https://bitbucket.org/fake-owner/fake-public-repo", + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Revision: "", + Path: "", + IsFile: false, + }, + }, + { + name: "should fail with only Bitbucket host", + url: "https://bitbucket.org/", + wantErr: missingUserAndRepoError, + }, + { + name: "should parse Bitbucket repo with file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/README.md", + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Revision: "main", + Path: "README.md", + IsFile: true, + }, + }, + { + name: "should parse Bitbucket file path with nested path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/src/main/directory/test.txt", + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Revision: "main", + Path: "directory/test.txt", + IsFile: true, + }, + }, + { + name: "should parse Bitbucket repo with raw file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/raw/main/README.md", + wantUrl: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "fake-owner", + Repo: "fake-public-repo", + Revision: "main", + Path: "README.md", + IsFile: true, + }, + }, + { + name: "should fail with missing Bitbucket repo", + url: "https://bitbucket.org/fake-owner", + wantErr: missingUserAndRepoError, + }, + { + name: "should fail with invalid Bitbucket directory or file path", + url: "https://bitbucket.org/fake-owner/fake-public-repo/main/README.md", + wantErr: invalidBitbucketPathError, + }, + { + name: "should fail with missing Bitbucket keywords", + url: "https://bitbucket.org/fake-owner/fake-public-repo/main/test/README.md", + wantErr: missingBitbucketKeywordError, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseGitUrl(tt.url) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Unxpected error: %t, want: %v", err, tt.wantUrl) + } else if err == nil && !reflect.DeepEqual(got, tt.wantUrl) { + t.Errorf("Expected: %v, received: %v, difference at %v", tt.wantUrl, got, pretty.Compare(tt.wantUrl, got)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } +} + +func Test_GetGitRawFileAPI(t *testing.T) { + tests := []struct { + name string + g GitUrl + want string + }{ + { + name: "Github url", + g: GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + Path: "tests/README.md", + }, + want: "https://raw.githubusercontent.com/devfile/library/main/tests/README.md", + }, + { + name: "GitLab url", + g: GitUrl{ + Protocol: "https", + Host: "gitlab.com", + Owner: "gitlab-org", + Repo: "gitlab", + Revision: "v15.11.0-ee", + Path: "README.md", + }, + want: "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/repository/files/README.md/raw?ref=v15.11.0-ee", + }, + { + name: "Bitbucket url", + g: GitUrl{ + Protocol: "https", + Host: "bitbucket.org", + Owner: "owner", + Repo: "repo-name", + Revision: "main", + Path: "path/to/file.md", + }, + want: "https://api.bitbucket.org/2.0/repositories/owner/repo-name/src/main/path/to/file.md", + }, + { + name: "Empty GitUrl", + g: GitUrl{}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.g.GitRawFileAPI() + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("Got: %v, want: %v", result, tt.want) + } + }) + } +} + +func Test_IsPublic(t *testing.T) { + publicGitUrl := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "devfile", + Repo: "library", + Revision: "main", + token: "fake-token", + } + + privateGitUrl := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "not", + Repo: "a-valid", + Revision: "none", + token: "fake-token", + } + + httpTimeout := 0 + + tests := []struct { + name string + g GitUrl + want bool + }{ + { + name: "should be public", + g: publicGitUrl, + want: true, + }, + { + name: "should be private", + g: privateGitUrl, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.g.IsPublic(&httpTimeout) + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("Got: %t, want: %t", result, tt.want) + } + }) + } +} + +func Test_CloneGitRepo(t *testing.T) { + tempInvalidDir := t.TempDir() + + invalidGitUrl := GitUrl{ + Protocol: "", + Host: "", + Owner: "nonexistent", + Repo: "nonexistent", + Revision: "nonexistent", + } + + invalidPrivateGitHubRepo := GitUrl{ + Protocol: "https", + Host: "github.com", + Owner: "fake-owner", + Repo: "fake-private-repo", + Revision: "master", + token: "fake-github-token", + } + + privateRepoBadTokenErr := "failed to clone repo with token*" + publicRepoInvalidUrlErr := "failed to clone repo without a token" + missingDestDirErr := "failed to clone repo, destination directory*" + + tests := []struct { + name string + gitUrl GitUrl + destDir string + wantErr string + }{ + { + name: "should fail with invalid destination directory", + gitUrl: invalidGitUrl, + destDir: filepath.Join(os.TempDir(), "nonexistent"), + wantErr: missingDestDirErr, + }, + { + name: "should fail with invalid git url", + gitUrl: invalidGitUrl, + destDir: tempInvalidDir, + wantErr: publicRepoInvalidUrlErr, + }, + { + name: "should fail to clone invalid private git url with a bad token", + gitUrl: invalidPrivateGitHubRepo, + destDir: tempInvalidDir, + wantErr: privateRepoBadTokenErr, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.gitUrl.CloneGitRepo(tt.destDir) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Unxpected error: %t, want: %v", err, tt.wantErr) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } +} diff --git a/pkg/git/mock.go b/pkg/git/mock.go new file mode 100644 index 00000000..458ab369 --- /dev/null +++ b/pkg/git/mock.go @@ -0,0 +1,146 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "strings" +) + +type MockGitUrl struct { + Protocol string // URL scheme + Host string // URL domain name + Owner string // name of the repo owner + Repo string // name of the repo + Revision string // branch name, tag name, or commit id + Path string // path to a directory or file in the repo + token string // used for authenticating a private repo + IsFile bool // defines if the URL points to a file in the repo +} + +func (m *MockGitUrl) GetToken() string { + return m.token +} + +var mockExecute = func(baseDir string, cmd CommandType, args ...string) ([]byte, error) { + if cmd == GitCommand { + if len(args) > 0 && args[0] == "clone" { + u, _ := url.Parse(args[1]) + password, hasPassword := u.User.Password() + + resourceFile, err := os.Create(filepath.Clean(baseDir) + "/resource.file") + if err != nil { + return nil, fmt.Errorf("failed to create test resource: %v", err) + } + + // private repository + if hasPassword { + switch password { + case "valid-token": + _, err := resourceFile.WriteString("private repo\n") + if err != nil { + return nil, fmt.Errorf("failed to write to test resource: %v", err) + } + return []byte(""), nil + default: + return []byte(""), fmt.Errorf("not a valid token") + } + } + + _, err = resourceFile.WriteString("public repo\n") + if err != nil { + return nil, fmt.Errorf("failed to write to test resource: %v", err) + } + return []byte(""), nil + } + + if len(args) > 0 && args[0] == "switch" { + revision := strings.TrimPrefix(args[2], "origin/") + if revision != "invalid-revision" { + resourceFile, err := os.OpenFile(filepath.Clean(baseDir)+"/resource.file", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + return nil, fmt.Errorf("failed to open test resource: %v", err) + } + _, err = resourceFile.WriteString("git switched") + if err != nil { + return nil, fmt.Errorf("failed to write to test resource: %v", err) + } + return []byte("git switched to revision"), nil + } + return []byte(""), fmt.Errorf("failed to switch revision") + } + } + + return []byte(""), fmt.Errorf(unsupportedCmdMsg, string(cmd)) +} + +func (m *MockGitUrl) CloneGitRepo(destDir string) error { + exist := CheckPathExists(destDir) + if !exist { + return fmt.Errorf("failed to clone repo, destination directory: '%s' does not exists", destDir) + } + + host := m.Host + if host == RawGitHubHost { + host = GitHubHost + } + + var repoUrl string + if m.GetToken() == "" { + repoUrl = fmt.Sprintf("%s://%s/%s/%s.git", m.Protocol, host, m.Owner, m.Repo) + } else { + repoUrl = fmt.Sprintf("%s://token:%s@%s/%s/%s.git", m.Protocol, m.GetToken(), host, m.Owner, m.Repo) + if m.Host == BitbucketHost { + repoUrl = fmt.Sprintf("%s://x-token-auth:%s@%s/%s/%s.git", m.Protocol, m.GetToken(), host, m.Owner, m.Repo) + } + } + + _, err := mockExecute(destDir, "git", "clone", repoUrl, destDir) + + if err != nil { + if m.GetToken() == "" { + return fmt.Errorf("failed to clone repo without a token, ensure that a token is set if the repo is private") + } else { + return fmt.Errorf("failed to clone repo with token, ensure that the url and token is correct") + } + } + + if m.Revision != "" { + _, err := mockExecute(destDir, "git", "switch", "--detach", "origin/"+m.Revision) + if err != nil { + return fmt.Errorf("failed to switch repo to revision. repo dir: %v, revision: %v", destDir, m.Revision) + } + } + + return nil +} + +func (m *MockGitUrl) SetToken(token string) error { + m.token = token + return nil +} + +func (m *MockGitUrl) IsGitProviderRepo() bool { + switch m.Host { + case GitHubHost, RawGitHubHost, GitLabHost, BitbucketHost: + return true + default: + return false + } +} diff --git a/pkg/git/util.go b/pkg/git/util.go new file mode 100644 index 00000000..79ed650a --- /dev/null +++ b/pkg/git/util.go @@ -0,0 +1,260 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "fmt" + "github.com/devfile/library/v2/pkg/testingutil/filesystem" + "github.com/gregjones/httpcache" + "github.com/gregjones/httpcache/diskcache" + "github.com/pkg/errors" + "io" + "io/ioutil" + "k8s.io/klog" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "time" +) + +const ( + HTTPRequestResponseTimeout = 30 * time.Second // HTTPRequestTimeout configures timeout of all HTTP requests +) + +// httpCacheDir determines directory where odo will cache HTTP responses +var httpCacheDir = filepath.Join(os.TempDir(), "odohttpcache") + +// HTTPRequestParams holds parameters of forming http request +type HTTPRequestParams struct { + URL string + Token string + Timeout *int + TelemetryClientName string //optional client name for telemetry +} + +// HTTPGetRequest gets resource contents given URL and token (if applicable) +// cacheFor determines how long the response should be cached (in minutes), 0 for no caching +func HTTPGetRequest(request HTTPRequestParams, cacheFor int) ([]byte, error) { + // Build http request + req, err := http.NewRequest("GET", request.URL, nil) + if err != nil { + return nil, err + } + if request.Token != "" { + bearer := "Bearer " + request.Token + req.Header.Add("Authorization", bearer) + } + + //add the telemetry client name + req.Header.Add("Client", request.TelemetryClientName) + + overriddenTimeout := HTTPRequestResponseTimeout + timeout := request.Timeout + if timeout != nil { + //if value is invalid, the default will be used + if *timeout > 0 { + //convert timeout to seconds + overriddenTimeout = time.Duration(*timeout) * time.Second + klog.V(4).Infof("HTTP request and response timeout overridden value is %v ", overriddenTimeout) + } else { + klog.V(4).Infof("Invalid httpTimeout is passed in, using default value") + } + + } + + httpClient := &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + ResponseHeaderTimeout: overriddenTimeout, + }, + Timeout: overriddenTimeout, + } + + klog.V(4).Infof("HTTPGetRequest: %s", req.URL.String()) + + if cacheFor > 0 { + // if there is an error during cache setup we show warning and continue without using cache + cacheError := false + httpCacheTime := time.Duration(cacheFor) * time.Minute + + // make sure that cache directory exists + err = os.MkdirAll(httpCacheDir, 0750) + if err != nil { + cacheError = true + klog.WarningDepth(4, "Unable to setup cache: ", err) + } + err = cleanHttpCache(httpCacheDir, httpCacheTime) + if err != nil { + cacheError = true + klog.WarningDepth(4, "Unable to clean up cache directory: ", err) + } + + if !cacheError { + httpClient.Transport = httpcache.NewTransport(diskcache.New(httpCacheDir)) + klog.V(4).Infof("Response will be cached in %s for %s", httpCacheDir, httpCacheTime) + } else { + klog.V(4).Info("Response won't be cached.") + } + } + + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.Header.Get(httpcache.XFromCache) != "" { + klog.V(4).Infof("Cached response used.") + } + + // We have a non 1xx / 2xx status, return an error + if (resp.StatusCode - 300) > 0 { + return nil, errors.Errorf("failed to retrieve %s, %v: %s", request.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + // Process http response + bytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return bytes, err +} + +// ValidateURL validates the URL +func ValidateURL(sourceURL string) error { + u, err := url.Parse(sourceURL) + if err != nil { + return err + } + + if len(u.Host) == 0 || len(u.Scheme) == 0 { + return errors.New("URL is invalid") + } + + return nil +} + +// cleanHttpCache checks cacheDir and deletes all files that were modified more than cacheTime back +func cleanHttpCache(cacheDir string, cacheTime time.Duration) error { + cacheFiles, err := ioutil.ReadDir(cacheDir) + if err != nil { + return err + } + + for _, f := range cacheFiles { + if f.ModTime().Add(cacheTime).Before(time.Now()) { + klog.V(4).Infof("Removing cache file %s, because it is older than %s", f.Name(), cacheTime.String()) + err := os.Remove(filepath.Join(cacheDir, f.Name())) + if err != nil { + return err + } + } + } + return nil +} + +// CheckPathExists checks if a path exists or not +func CheckPathExists(path string) bool { + return checkPathExistsOnFS(path, filesystem.DefaultFs{}) +} + +func checkPathExistsOnFS(path string, fs filesystem.Filesystem) bool { + if _, err := fs.Stat(path); !os.IsNotExist(err) { + // path to file does exist + return true + } + klog.V(4).Infof("path %s doesn't exist, skipping it", path) + return false +} + +// CopyAllDirFiles recursively copies a source directory to a destination directory +func CopyAllDirFiles(srcDir, destDir string) error { + return copyAllDirFilesOnFS(srcDir, destDir, filesystem.DefaultFs{}) +} + +func copyAllDirFilesOnFS(srcDir, destDir string, fs filesystem.Filesystem) error { + var info os.FileInfo + + files, err := fs.ReadDir(srcDir) + if err != nil { + return errors.Wrapf(err, "failed reading dir %v", srcDir) + } + + for _, file := range files { + srcPath := path.Join(srcDir, file.Name()) + destPath := path.Join(destDir, file.Name()) + + if file.IsDir() { + if info, err = fs.Stat(srcPath); err != nil { + return err + } + if err = fs.MkdirAll(destPath, info.Mode()); err != nil { + return err + } + if err = copyAllDirFilesOnFS(srcPath, destPath, fs); err != nil { + return err + } + } else { + if file.Name() == "devfile.yaml" { + continue + } + // Only copy files that do not exist in the destination directory + if !checkPathExistsOnFS(destPath, fs) { + if err := copyFileOnFs(srcPath, destPath, fs); err != nil { + return errors.Wrapf(err, "failed to copy %s to %s", srcPath, destPath) + } + } + } + } + return nil +} + +// copied from: https://github.com/devfile/registry-support/blob/main/index/generator/library/util.go +func copyFileOnFs(src, dst string, fs filesystem.Filesystem) error { + var err error + var srcinfo os.FileInfo + + srcfd, err := fs.Open(src) + if err != nil { + return err + } + defer func() { + if e := srcfd.Close(); e != nil { + fmt.Printf("err occurred while closing file: %v", e) + } + }() + + dstfd, err := fs.Create(dst) + if err != nil { + return err + } + defer func() { + if e := dstfd.Close(); e != nil { + fmt.Printf("err occurred while closing file: %v", e) + } + }() + + if _, err = io.Copy(dstfd, srcfd); err != nil { + return err + } + if srcinfo, err = fs.Stat(src); err != nil { + return err + } + return fs.Chmod(dst, srcinfo.Mode()) +} diff --git a/pkg/git/util_test.go b/pkg/git/util_test.go new file mode 100644 index 00000000..3c0f54c3 --- /dev/null +++ b/pkg/git/util_test.go @@ -0,0 +1,122 @@ +// +// Copyright 2023 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package git + +import ( + "github.com/devfile/library/v2/pkg/testingutil/filesystem" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestHTTPGetRequest(t *testing.T) { + invalidHTTPTimeout := -1 + validHTTPTimeout := 20 + + // Start a local HTTP server + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Send response to be tested + _, err := rw.Write([]byte("OK")) + if err != nil { + t.Error(err) + } + })) + // Close the server when test finishes + defer server.Close() + + tests := []struct { + name string + url string + want []byte + timeout *int + }{ + { + name: "Case 1: Input url is valid", + url: server.URL, + // Want(Expected) result is "OK" + // According to Unicode table: O == 79, K == 75 + want: []byte{79, 75}, + }, + { + name: "Case 2: Input url is invalid", + url: "invalid", + want: nil, + }, + { + name: "Case 3: Test invalid httpTimeout, default timeout will be used", + url: server.URL, + timeout: &invalidHTTPTimeout, + want: []byte{79, 75}, + }, + { + name: "Case 4: Test valid httpTimeout", + url: server.URL, + timeout: &validHTTPTimeout, + want: []byte{79, 75}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + request := HTTPRequestParams{ + URL: tt.url, + Timeout: tt.timeout, + } + got, err := HTTPGetRequest(request, 0) + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Got: %v, want: %v", got, tt.want) + t.Logf("Error message is: %v", err) + } + }) + } +} + +func TestCheckPathExists(t *testing.T) { + fs := filesystem.NewFakeFs() + fs.MkdirAll("/path/to/devfile", 0755) + fs.WriteFile("/path/to/devfile/devfile.yaml", []byte(""), 0755) + + file := "/path/to/devfile/devfile.yaml" + missingFile := "/path/to/not/devfile" + + tests := []struct { + name string + filePath string + want bool + }{ + { + name: "should be able to get file that exists", + filePath: file, + want: true, + }, + { + name: "should fail if file does not exist", + filePath: missingFile, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := checkPathExistsOnFS(tt.filePath, fs) + if !reflect.DeepEqual(result, tt.want) { + t.Errorf("Got error: %t, want error: %t", result, tt.want) + } + }) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index b24a3d5c..b4a4fb3c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -1,5 +1,5 @@ // -// Copyright 2022 Red Hat, Inc. +// Copyright 2022-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,6 +21,9 @@ import ( "bytes" "crypto/rand" "fmt" + "github.com/devfile/library/v2/pkg/git" + gitpkg "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" "io" "io/ioutil" "math/big" @@ -41,11 +44,8 @@ import ( "syscall" "time" - "github.com/go-git/go-git/v5/plumbing" - "github.com/devfile/library/v2/pkg/testingutil/filesystem" "github.com/fatih/color" - gitpkg "github.com/go-git/go-git/v5" "github.com/gobwas/glob" "github.com/gregjones/httpcache" "github.com/gregjones/httpcache/diskcache" @@ -57,6 +57,10 @@ import ( "k8s.io/klog" ) +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + const ( HTTPRequestResponseTimeout = 30 * time.Second // HTTPRequestTimeout configures timeout of all HTTP requests ModeReadWriteFile = 0600 // default Permission for a file @@ -881,6 +885,15 @@ func ConvertGitSSHRemoteToHTTPS(remote string) string { return remote } +// IsGitProviderRepo checks if the url matches a repo from a supported git provider +func IsGitProviderRepo(url string) bool { + if strings.Contains(url, git.RawGitHubHost) || strings.Contains(url, git.GitHubHost) || + strings.Contains(url, git.GitLabHost) || strings.Contains(url, git.BitbucketHost) { + return true + } + return false +} + // GetAndExtractZip downloads a zip file from a URL with a http prefix or // takes an absolute path prefixed with file:// and extracts it to a destination. // pathToUnzip specifies the path within the zip folder to extract @@ -1083,17 +1096,47 @@ func DownloadFileInMemory(url string) ([]byte, error) { // DownloadInMemory uses HTTPRequestParams to download the file and return bytes func DownloadInMemory(params HTTPRequestParams) ([]byte, error) { - var httpClient = &http.Client{Transport: &http.Transport{ ResponseHeaderTimeout: HTTPRequestResponseTimeout, }, Timeout: HTTPRequestResponseTimeout} - url := params.URL + var g git.GitUrl + var err error + + if IsGitProviderRepo(params.URL) { + g, err = git.NewGitUrlWithURL(params.URL) + if err != nil { + return nil, errors.Errorf("failed to parse git repo. error: %v", err) + } + } + + return downloadInMemoryWithClient(params, httpClient, g) +} + +func downloadInMemoryWithClient(params HTTPRequestParams, httpClient HTTPClient, g git.GitUrl) ([]byte, error) { + var url string + url = params.URL req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } + if IsGitProviderRepo(url) { + url = g.GitRawFileAPI() + req, err = http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + if !g.IsPublic(params.Timeout) { + // check that the token is valid before adding to the header + err = g.SetToken(params.Token, params.Timeout) + if err != nil { + return nil, err + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", params.Token)) + } + } + //add the telemetry client name in the header req.Header.Add("Client", params.TelemetryClientName) resp, err := httpClient.Do(req) @@ -1187,6 +1230,7 @@ func ValidateFile(filePath string) error { } // GetGitUrlComponentsFromRaw converts a raw GitHub file link to a map of the url components +// Deprecated: in favor of the method git.ParseGitUrl() with the devfile/library/v2/pkg/git package func GetGitUrlComponentsFromRaw(rawGitURL string) (map[string]string, error) { var urlComponents map[string]string @@ -1219,6 +1263,7 @@ func GetGitUrlComponentsFromRaw(rawGitURL string) (map[string]string, error) { } // CloneGitRepo clones a GitHub repo to a destination directory +// Deprecated: in favor of the method git.CloneGitRepo() with the devfile/library/v2/pkg/git package func CloneGitRepo(gitUrlComponents map[string]string, destDir string) error { gitUrl := fmt.Sprintf("https://github.com/%s/%s.git", gitUrlComponents["username"], gitUrlComponents["project"]) branch := fmt.Sprintf("refs/heads/%s", gitUrlComponents["branch"]) diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 4c0feb32..55a8bdae 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1,5 +1,5 @@ // -// Copyright 2021-2022 Red Hat, Inc. +// Copyright 2021-2023 Red Hat, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ package util import ( "fmt" "github.com/devfile/library/v2/pkg/testingutil/filesystem" + "github.com/kylelemons/godebug/pretty" + "github.com/stretchr/testify/assert" "io/ioutil" corev1 "k8s.io/api/core/v1" "net" @@ -936,6 +938,64 @@ func TestDownloadFile(t *testing.T) { } } +func TestDownloadInMemory(t *testing.T) { + // Start a local HTTP server + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + // Send response to be tested + _, err := rw.Write([]byte("OK")) + if err != nil { + t.Error(err) + } + })) + + // Close the server when test finishes + defer server.Close() + + tests := []struct { + name string + url string + token string + want []byte + wantErr string + }{ + { + name: "Case 1: Input url is valid", + url: server.URL, + want: []byte{79, 75}, + }, + { + name: "Case 2: Input url is invalid", + url: "invalid", + wantErr: "unsupported protocol scheme", + }, + { + name: "Case 3: Git provider with invalid url", + url: "github.com/mike-hoang/invalid-repo", + token: "", + want: []byte(nil), + wantErr: "failed to parse git repo. error:*", + }, + { + name: "Case 4: Public Github repo with missing blob", + url: "https://github.com/devfile/library/main/README.md", + wantErr: "failed to parse git repo. error: url path to directory or file should contain 'tree' or 'blob'*", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := DownloadInMemory(HTTPRequestParams{URL: tt.url, Token: tt.token}) + if (err != nil) != (tt.wantErr != "") { + t.Errorf("Failed to download file with error: %s", err) + } else if err == nil && !reflect.DeepEqual(data, tt.want) { + t.Errorf("Expected: %v, received: %v, difference at %v", tt.want, string(data[:]), pretty.Compare(tt.want, data)) + } else if err != nil { + assert.Regexp(t, tt.wantErr, err.Error(), "Error message should match") + } + }) + } +} + func TestValidateK8sResourceName(t *testing.T) { tests := []struct { name string @@ -1017,85 +1077,6 @@ func TestValidateFile(t *testing.T) { } } -func TestGetGitUrlComponentsFromRaw(t *testing.T) { - validRawGitUrl := "https://raw.githubusercontent.com/username/project/branch/file/path" - invalidUrl := "github.com/not/valid/url" - - tests := []struct { - name string - url string - wantErr bool - }{ - { - name: "should be able to get git url components", - url: validRawGitUrl, - wantErr: false, - }, - { - name: "should fail with invalid raw git url", - url: invalidUrl, - wantErr: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - _, err := GetGitUrlComponentsFromRaw(tt.url) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) - } - }) - } -} - -func TestCloneGitRepo(t *testing.T) { - tempDir, err := ioutil.TempDir("", "") - if err != nil { - t.Errorf("Failed to create temp dir: %s, error: %v", tempDir, err) - } - defer os.RemoveAll(tempDir) - - invalidGitUrl := map[string]string{ - "username": "devfile", - "project": "nonexistent", - "branch": "nonexistent", - } - validGitUrl := map[string]string{ - "username": "devfile", - "project": "library", - "branch": "main", - } - - tests := []struct { - name string - gitUrlComponents map[string]string - destDir string - wantErr bool - }{ - { - name: "should fail with invalid git url", - gitUrlComponents: invalidGitUrl, - destDir: filepath.Join(os.TempDir(), "nonexistent"), - wantErr: true, - }, - { - name: "should be able to clone valid git url", - gitUrlComponents: validGitUrl, - destDir: tempDir, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - err := CloneGitRepo(tt.gitUrlComponents, tt.destDir) - if (err != nil) != tt.wantErr { - t.Errorf("Expected error: %t, got error: %t", tt.wantErr, err) - } - }) - } -} - func TestCopyFile(t *testing.T) { // Create temp dir tempDir, err := ioutil.TempDir("", "")