feat: impl kcl openapi schema generation with kcl decorators

Signed-off-by: peefy <xpf6677@163.com>

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/chai2010/jsonv v1.1.3
 	github.com/chai2010/protorpc v1.1.4
+	github.com/getkin/kin-openapi v0.122.0
 	github.com/goccy/go-yaml v1.11.0
 	github.com/gofrs/flock v0.8.1
 	github.com/golang/protobuf v1.5.3
@@ -52,13 +53,17 @@ require (
 	github.com/go-git/go-git/v5 v5.6.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/invopop/yaml v0.2.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
@@ -67,10 +72,12 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/moby/locker v1.0.1 // indirect
 	github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc4 // indirect
 	github.com/otiai10/copy v1.9.0 // indirect
+	github.com/perimeterx/marshmallow v1.1.5 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

go.sum

@@ -123,6 +123,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
+github.com/getkin/kin-openapi v0.122.0 h1:WB9Jbl0Hp/T79/JF9xlSW5Kl9uYdk/AWD0yAd9HOM10=
+github.com/getkin/kin-openapi v0.122.0/go.mod h1:PCWw/lfBrJY4HcdqE3jj+QFkaFK8ABoqo7PvqVhXXqw=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
@@ -150,10 +152,16 @@ github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
 github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=
+github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
+github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
 github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54=
 github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng=
 github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
@@ -237,9 +245,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY=
+github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -293,6 +304,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -309,6 +322,8 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.4.0 h1:umwcf7gbpEwf7WFzqmWwSv0CzbeMsae2u9ZvpP8j2q4=
 github.com/otiai10/mint v1.4.0/go.mod h1:gifjb2MYOoULtKLqUAEILUG/9KONW6f7YsJ6vQLTlFI=
+github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
+github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI=
 github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4=
 github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI=
@@ -366,14 +381,20 @@ github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/thoas/go-funk v0.9.3 h1:7+nAEx3kn5ZJcnDm2Bh23N2yOtweO14bi//dvRtgLpw=
 github.com/thoas/go-funk v0.9.3/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q=
+github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0=
 github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
 github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=

pkg/tools/gen/gendoc.go

@@ -11,6 +11,8 @@ import (
 	"sort"
 	"strings"
 	"text/template"
+
+	"github.com/yuin/goldmark"
 )
 
 //go:embed templates/doc/schemaDoc.gotmpl
@@ -71,6 +73,7 @@ type Format string
 const (
 	Html     Format = "html"
 	Markdown Format = "md"
+	OpenAPI  Format = "openapi"
 )
 
 // KclPackage contains package information of package metadata(such as name, version, description, ...) and exported models(such as schemas)
@@ -90,12 +93,8 @@ func (g *GenContext) render(spec *SwaggerV2Spec) error {
 	if err != nil {
 		return fmt.Errorf("failed to create docs/ directory under the target directory: %s", err)
 	}
-	// extract kcl package from swaggerV2 spec
-	rootPkg := spec.toKclPackage()
-	// sort schemas and subpackages by their names
-	rootPkg.sortSchemasAndPkgs()
 	// render the package
-	err = g.renderPackage(rootPkg, g.Target)
+	err = g.renderPackage(spec, g.Target)
 	if err != nil {
 		return err
 	}
@@ -231,28 +230,72 @@ func (pkg *KclPackage) getIndexContent(level int, indentation string) string {
 	return content
 }
 
-func (g *GenContext) renderPackage(pkg *KclPackage, parentDir string) error {
+func (g *GenContext) renderPackage(spec *SwaggerV2Spec, parentDir string) error {
+	// extract kcl package from swaggerV2 spec
+	pkg := spec.toKclPackage()
+	// sort schemas and subpackages by their names
+	pkg.sortSchemasAndPkgs()
 	pkgName := pkg.Name
 	if pkg.Name == "" {
 		pkgName = "main"
 	}
-	fmt.Println(fmt.Sprintf("generating doc for package %s", pkgName))
-	docFileName := fmt.Sprintf("%s.%s", pkgName, g.Format)
-	var contentBuf bytes.Buffer
-	err := g.Template.ExecuteTemplate(&contentBuf, "packageDoc", struct {
-		EscapeHtml bool
-		Data       *KclPackage
-	}{
-		EscapeHtml: g.EscapeHtml,
-		Data:       pkg,
-	})
-	if err != nil {
-		return fmt.Errorf("failed to render package %s with template, err: %s", pkg.Name, err)
-	}
-	// write content to file
-	err = os.WriteFile(filepath.Join(parentDir, docFileName), contentBuf.Bytes(), 0644)
-	if err != nil {
-		return fmt.Errorf("failed to write file %s in %s: %v", docFileName, parentDir, err)
+	fmt.Printf("generating doc for package %s\n", pkgName)
+	// --- format ---
+	switch strings.ToLower(string(g.Format)) {
+	case string(Markdown):
+		docFileName := fmt.Sprintf("%s.%s", pkgName, g.Format)
+		var buf bytes.Buffer
+		err := g.Template.ExecuteTemplate(&buf, "packageDoc", struct {
+			EscapeHtml bool
+			Data       *KclPackage
+		}{
+			EscapeHtml: g.EscapeHtml,
+			Data:       pkg,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to render package %s with template, err: %s", pkg.Name, err)
+		}
+		// write content to file
+		err = os.WriteFile(filepath.Join(parentDir, docFileName), buf.Bytes(), 0644)
+		if err != nil {
+			return fmt.Errorf("failed to write file %s in %s: %v", docFileName, parentDir, err)
+		}
+	case string(Html):
+		var mdBuf bytes.Buffer
+		err := g.Template.ExecuteTemplate(&mdBuf, "packageDoc", struct {
+			EscapeHtml bool
+			Data       *KclPackage
+		}{
+			EscapeHtml: g.EscapeHtml,
+			Data:       pkg,
+		})
+		if err != nil {
+			return fmt.Errorf("failed to render package %s with template, err: %s", pkg.Name, err)
+		}
+		var htmlBuf bytes.Buffer
+		if err := goldmark.Convert(mdBuf.Bytes(), &htmlBuf); err != nil {
+			panic(err)
+		}
+		docFileName := fmt.Sprintf("%s.%s", pkgName, g.Format)
+		// write content to file
+		err = os.WriteFile(filepath.Join(parentDir, docFileName), htmlBuf.Bytes(), 0644)
+		if err != nil {
+			return fmt.Errorf("failed to write file %s in %s: %v", docFileName, parentDir, err)
+		}
+	case string(OpenAPI):
+		docFileName := fmt.Sprintf("%s.%s", pkgName, "json")
+		spec := SwaggerV2TotOpenAPIV3Spec(spec)
+		json, err := spec.MarshalJSON()
+		if err != nil {
+			return err
+		}
+		// write content to file
+		err = os.WriteFile(filepath.Join(parentDir, docFileName), json, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to write file %s in %s: %v", docFileName, parentDir, err)
+		}
+	default:
+		return fmt.Errorf("invalid generate format. Allow values: %s", []Format{Markdown, Html, OpenAPI})
 	}
 	return nil
 }
@@ -263,12 +306,12 @@ func (opts *GenOpts) ValidateComplete() (*GenContext, error) {
 	switch strings.ToLower(opts.Format) {
 	case string(Markdown):
 		g.Format = Markdown
-		break
 	case string(Html):
 		g.Format = Html
-		break
+	case string(OpenAPI):
+		g.Format = OpenAPI
 	default:
-		return nil, fmt.Errorf("invalid generate format. Allow values: %s", []Format{Markdown, Html})
+		return nil, fmt.Errorf("invalid generate format. Allow values: %s", []Format{Markdown, Html, OpenAPI})
 	}
 
 	// --- package path ---
@@ -374,7 +417,7 @@ func (opts *GenOpts) ValidateComplete() (*GenContext, error) {
 	g.Target = path.Join(g.Target, "docs")
 	if _, err := os.Stat(g.Target); err == nil {
 		// check and warn if the docs directory already exists
-		fmt.Println(fmt.Sprintf("[Warn] path %s exists, all the content will be overwritten", g.Target))
+		fmt.Printf("[Warn] path %s exists, all the content will be overwritten\n", g.Target)
 		if err := os.RemoveAll(g.Target); err != nil {
 			return nil, fmt.Errorf("failed to remove existing content in %s:%s", g.Target, err)
 		}
@@ -385,7 +428,7 @@ func (opts *GenOpts) ValidateComplete() (*GenContext, error) {
 
 // GenDoc generate document files from KCL source files
 func (g *GenContext) GenDoc() error {
-	spec, err := KclPackageToSwaggerV2Spec(g.PackagePath)
+	spec, err := ExportSwaggerV2Spec(g.PackagePath)
 	if err != nil {
 		return err
 	}