Allow for parsing spec file from URL (#149)

* Use Parse method to parse api spec regardless of whether its in the form of a local file or remote file via a URL

* Turn api spec into string reader and parse it using ParseFromReader

* Create 'entrypoint' function Parse which accepts a path that should be parseable into a resource locator and then parse via URL or file path as required

* Create new parser that accepts Loader interface which allows us to test it easier

cmd/generators.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/providers/posflag"
 	"github.com/knadh/koanf/providers/structs"
@@ -48,7 +49,7 @@ func init() {
 				}
 
 				// parse OpenAPI spec
-				apiSpec, err := spec.ParseFromFile(apiSpecPath)
+				apiSpec, err := spec.NewParser(openapi3.NewLoader()).Parse(apiSpecPath)
 				if err != nil {
 					log.Fatal(err)
 				}

cmd/wizard.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"os"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/mattn/go-isatty"
 	"github.com/spf13/cobra"
 
@@ -24,7 +25,7 @@ func init() {
 			}
 
 			// parse OpenAPI spec
-			apiSpec, err := spec.ParseFromFile(apiSpecPath)
+			apiSpec, err := spec.NewParser(openapi3.NewLoader()).Parse(apiSpecPath)
 			if err != nil {
 				log.Fatal(err)
 			}

generators/ambassador/ambassador_test.go

@@ -1,8 +1,10 @@
 package ambassador
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/stretchr/testify/require"
 
 	"github.com/kubeshop/kusk/options"
@@ -23,7 +25,7 @@ func TestAmbassador(t *testing.T) {
 		t.Run(testCase.name, func(t *testing.T) {
 			r := require.New(t)
 
-			spec, err := spec.Parse([]byte(testCase.spec))
+			spec, err := spec.NewParser(openapi3.NewLoader()).ParseFromReader(strings.NewReader(testCase.spec))
 			r.NoError(err, "failed to parse spec")
 
 			mappings, err := gen.Generate(&testCase.options, spec)

generators/linkerd/linkerd_test.go

@@ -1,8 +1,10 @@
 package linkerd
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/stretchr/testify/require"
 
 	"github.com/kubeshop/kusk/options"
@@ -23,7 +25,7 @@ func TestLinkerd(t *testing.T) {
 		t.Run(testCase.name, func(t *testing.T) {
 			r := require.New(t)
 
-			spec, err := spec.Parse([]byte(testCase.spec))
+			spec, err := spec.NewParser(openapi3.NewLoader()).ParseFromReader(strings.NewReader(testCase.spec))
 			r.NoError(err, "failed to parse spec")
 
 			profile, err := gen.Generate(&testCase.options, spec)

generators/nginx_ingress/nginx_ingress_test.go

@@ -1,8 +1,10 @@
 package nginx_ingress
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/stretchr/testify/require"
 
 	"github.com/kubeshop/kusk/options"
@@ -522,7 +524,8 @@ status:
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
 			r := require.New(t)
-			spec, err := spec.Parse([]byte(testCase.spec))
+
+			spec, err := spec.NewParser(openapi3.NewLoader()).ParseFromReader(strings.NewReader(testCase.spec))
 			r.NoError(err)
 			profile, err := gen.Generate(&testCase.options, spec)
 			r.NoError(err)

spec/spec.go

@@ -1,39 +1,86 @@
 package spec
 
 import (
+	"bytes"
 	"fmt"
-	"os"
+	"io"
+	"io/ioutil"
+	"net/url"
 
 	"github.com/getkin/kin-openapi/openapi2"
 	"github.com/getkin/kin-openapi/openapi2conv"
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/ghodss/yaml"
 )
 
-type header struct {
-	Swagger string `json:"swagger"`
-	OpenAPI string `json:"openapi"` // we might need that later to distinguish 3.1.x vs 3.0.x
-}
-
 // isSwagger tries to decode the spec header
 func isSwagger(spec []byte) bool {
-	var header header
+	// internal helper struct to help us differentiate
+	// between openapi spec 2.0 (swagger) and openapi 3+
+	var header struct {
+		Swagger string `json:"swagger"`
+		OpenAPI string `json:"openapi"` // we might need that later to distinguish 3.1.x vs 3.0.x
+	}
 
 	_ = yaml.Unmarshal(spec, &header)
 
 	return header.Swagger != ""
 }
 
-func ParseFromFile(path string) (*openapi3.T, error) {
-	contents, err := os.ReadFile(path)
+type Loader interface {
+	LoadFromURI(location *url.URL) (*openapi3.T, error)
+	LoadFromFile(location string) (*openapi3.T, error)
+}
+
+type Parser struct {
+	loader Loader
+}
+
+func NewParser(loader Loader) Parser {
+	return Parser{
+		loader: loader,
+	}
+}
+
+// Parse is the entrypoint for the spec package
+// Accepts a path that should be parseable into a resource locater
+// i.e. a URL or relative file path
+func (p Parser) Parse(path string) (*openapi3.T, error) {
+	u, err := url.Parse(path)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read spec file: %w", err)
+		return nil, fmt.Errorf("invalid resource path %s: %w", path, err)
 	}
 
-	return Parse(contents)
+	var spec *openapi3.T
+	if isURLRelative := u.Host == ""; isURLRelative {
+		spec, err = p.loader.LoadFromFile(path)
+	} else {
+		spec, err = p.loader.LoadFromURI(u)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("unable to load spec: %w", err)
+	}
+
+	// we need to marshal the struct back to yaml while we support
+	// both openapi spec 2.0 and 3.0, so we can differentiate between the two
+	// and convert 2.0 to 3.0 if needed
+	bSpec, err := yaml.Marshal(&spec)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal spec to yaml: %w", err)
+	}
+
+	return p.ParseFromReader(bytes.NewReader(bSpec))
 }
 
-func Parse(spec []byte) (*openapi3.T, error) {
+// ParseFromReader allows for providing your own Reader implementation
+// to parse the API spec from
+func (p Parser) ParseFromReader(contents io.Reader) (*openapi3.T, error) {
+	spec, err := ioutil.ReadAll(contents)
+	if err != nil {
+		return nil, fmt.Errorf("could not read contents of api spec: %w", err)
+	}
+
 	if isSwagger(spec) {
 		return parseSwagger(spec)
 	}

spec/spec_test.go

@@ -0,0 +1,146 @@
+package spec
+
+import (
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	loadedFromURI  = "loaded from URI"
+	loadedFromFile = "loaded from file"
+)
+
+type mockLoader struct{}
+
+func (m mockLoader) LoadFromURI(_ *url.URL) (*openapi3.T, error) {
+	return &openapi3.T{
+		OpenAPI: "3.0.3",
+		Info: &openapi3.Info{
+			Title:       "Sample API",
+			Description: loadedFromURI,
+			Version:     "1.0.0",
+		},
+	}, nil
+}
+
+func (m mockLoader) LoadFromFile(_ string) (*openapi3.T, error) {
+	return &openapi3.T{
+		OpenAPI: "3.0.3",
+		Info: &openapi3.Info{
+			Title:       "Sample API",
+			Description: loadedFromFile,
+			Version:     "1.0.0",
+		},
+	}, nil
+}
+
+func TestParse(t *testing.T) {
+	testCases := []struct {
+		name     string
+		url      string
+		result   string
+	}{
+		{
+			name:     "load spec from url",
+			url:      "https://someurl.io/swagger.yaml",
+			result:   loadedFromURI,
+		},
+		{
+			name:     "load spec from local file",
+			url:      "some-folder/swagger.yaml",
+			result:   loadedFromFile,
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			r := require.New(t)
+
+			parser := Parser{loader: mockLoader{}}
+			u, err := url.Parse(testCase.url)
+			r.NoError(err, "please provide a valid url")
+
+			actual, err := parser.Parse(u.String())
+			r.NoError(err, "expected no error when running parse from mocked loader")
+			r.True(actual.Info.Description == testCase.result)
+		})
+	}
+}
+
+func TestParseFromReader(t *testing.T) {
+	testCases := []struct {
+		name   string
+		spec   string
+		result *openapi3.T
+	}{
+		{
+			name: "swagger",
+			spec: `swagger: "2.0"
+info:
+  title: Sample API
+  description: API description in Markdown.
+  version: 1.0.0
+paths:
+  /users:
+    get: {}
+`,
+			result: &openapi3.T{
+				OpenAPI: "3.0.3",
+				Info: &openapi3.Info{
+					Title:       "Sample API",
+					Description: "API description in Markdown.",
+					Version:     "1.0.0",
+				},
+				Paths: openapi3.Paths{
+					"/users": &openapi3.PathItem{
+						Get: &openapi3.Operation{},
+					},
+				},
+			},
+		},
+		{
+			name: "openapi",
+			spec: `openapi: "3.0.3"
+info:
+  title: Sample API
+  description: API description in Markdown.
+  version: 1.0.0
+paths:
+  /users:
+    get: {}
+`,
+			result: &openapi3.T{
+				OpenAPI: "3.0.3",
+				Info: &openapi3.Info{
+					Title:       "Sample API",
+					Description: "API description in Markdown.",
+					Version:     "1.0.0",
+				},
+				Paths: openapi3.Paths{
+					"/users": &openapi3.PathItem{
+						Get: &openapi3.Operation{},
+					},
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			r := require.New(t)
+
+			actual, err := Parser{loader: openapi3.NewLoader()}.ParseFromReader(strings.NewReader(testCase.spec))
+			r.NoError(err, "failed to parse spec from reader")
+			r.Equal(testCase.result.OpenAPI, actual.OpenAPI)
+			r.Equal(testCase.result.Info.Title, actual.Info.Title)
+			r.Equal(testCase.result.Info.Description, actual.Info.Description)
+			r.Equal(testCase.result.Info.Version, actual.Info.Version)
+			r.NotNil(testCase.result.Paths.Find("/users"))
+		})
+
+	}
+}