Add support for Konnect documents

cmd/konnect_diff.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -21,6 +22,9 @@ the entities present in files locally. This allows you to see the entities
 that will be created or updated or deleted.` + konnectAlphaState,
 	Args: validateNoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if konnectDumpCmdKongStateFile == "-" {
+			return errors.New("writing to stdout is not supported in Konnect mode")
+		}
 		return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, true,
 			konnectDiffCmdParallelism)
 	},
@@ -30,8 +34,7 @@ func init() {
 	konnectCmd.AddCommand(konnectDiffCmd)
 	konnectDiffCmd.Flags().StringSliceVarP(&konnectDiffCmdKongStateFile,
 		"state", "s", []string{"konnect.yaml"}, "file(s) containing Konnect's configuration.\n"+
-			"This flag can be specified multiple times for multiple files.\n"+
-			"Use '-' to read from stdin.")
+			"This flag can be specified multiple times for multiple files.")
 	konnectDiffCmd.Flags().BoolVar(&konnectDumpIncludeConsumers, "include-consumers",
 		false, "export consumers, associated credentials and any plugins associated "+
 			"with consumers")

cmd/konnect_dump.go

@@ -31,6 +31,10 @@ configure Konnect.` + konnectAlphaState,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		httpClient := http.DefaultClient
 
+		if konnectDumpCmdKongStateFile == "-" {
+			return errors.New("writing to stdout is not supported in Konnect mode")
+		}
+
 		if yes, err := confirmFileOverwrite(konnectDumpCmdKongStateFile, dumpCmdStateFormat, assumeYes); err != nil {
 			return err
 		} else if !yes {
@@ -87,8 +91,7 @@ configure Konnect.` + konnectAlphaState,
 func init() {
 	konnectCmd.AddCommand(konnectDumpCmd)
 	konnectDumpCmd.Flags().StringVarP(&konnectDumpCmdKongStateFile, "output-file", "o",
-		"konnect", "file to which to write Kong's configuration."+
-			"Use '-' to write to stdout.")
+		"konnect", "file to which to write Kong's configuration.")
 	konnectDumpCmd.Flags().StringVar(&konnectDumpCmdStateFormat, "format",
 		"yaml", "output file format: json or yaml")
 	konnectDumpCmd.Flags().BoolVar(&konnectDumpWithID, "with-id",

cmd/konnect_sync.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 )
 
@@ -13,6 +14,9 @@ var konnectSyncCmd = &cobra.Command{
 to get Konnect's state in sync with the input state.` + konnectAlphaState,
 	Args: validateNoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if konnectDumpCmdKongStateFile == "-" {
+			return errors.New("writing to stdout is not supported in Konnect mode")
+		}
 		return syncKonnect(cmd.Context(), konnectDiffCmdKongStateFile, false,
 			konnectDiffCmdParallelism)
 	},
@@ -22,8 +26,7 @@ func init() {
 	konnectCmd.AddCommand(konnectSyncCmd)
 	konnectSyncCmd.Flags().StringSliceVarP(&konnectDiffCmdKongStateFile,
 		"state", "s", []string{"konnect.yaml"}, "file(s) containing Konnect's configuration.\n"+
-			"This flag can be specified multiple times for multiple files.\n"+
-			"Use '-' to read from stdin.")
+			"This flag can be specified multiple times for multiple files.")
 	konnectSyncCmd.Flags().BoolVar(&konnectDumpIncludeConsumers, "include-consumers",
 		false, "export consumers, associated credentials and any plugins associated "+
 			"with consumers")

dump/dump_konnect.go

@@ -50,7 +50,7 @@ func GetFromKonnect(ctx context.Context, konnectClient *konnect.Client,
 		for i := 0; i < len(servicePackages); i++ {
 			// control the number of outstanding go routines, also controlling
 			// the number of parallel requests
-			err := semaphore.Acquire(ctx, 1)
+			err := semaphore.Acquire(ctx, 2)
 			if err != nil {
 				return fmt.Errorf("acquire semaphore: %v", err)
 			}
@@ -63,6 +63,33 @@ func GetFromKonnect(ctx context.Context, konnectClient *konnect.Client,
 				}
 				servicePackages[i].Versions = versions
 			}(i)
+			go func(i int) {
+				defer semaphore.Release(1)
+				documents, err := konnectClient.Documents.ListAllForParent(ctx, servicePackages[i])
+				if err != nil {
+					errChan <- err
+					return
+				}
+				res.Documents = append(res.Documents, documents...)
+			}(i)
+		}
+		for i := 0; i < len(servicePackages); i++ {
+			for j := 0; j < len(servicePackages[i].Versions); j++ {
+				err := semaphore.Acquire(ctx, 1)
+				if err != nil {
+					return fmt.Errorf("acquire semaphore: %v", err)
+				}
+				go func(i int, j int) {
+					defer semaphore.Release(1)
+					documents, err := konnectClient.Documents.ListAllForParent(ctx,
+						&servicePackages[i].Versions[j])
+					if err != nil {
+						errChan <- err
+						return
+					}
+					res.Documents = append(res.Documents, documents...)
+				}(i, j)
+			}
 		}
 		err = semaphore.Acquire(ctx, 10)
 		if err != nil {

file/types.go

@@ -523,6 +523,7 @@ type FServiceVersion struct {
 	ID             *string         `json:"id,omitempty" yaml:"id,omitempty"`
 	Version        *string         `json:"version,omitempty" yaml:"version,omitempty"`
 	Implementation *Implementation `json:"implementation,omitempty" yaml:"implementation,omitempty"`
+	Documents      []FDocument     `json:"documents,omitempty" yaml:"documents,omitempty"`
 }
 
 // +k8s:deepcopy-gen=true
@@ -531,6 +532,15 @@ type FServicePackage struct {
 	Name        *string           `json:"name,omitempty" yaml:"name,omitempty"`
 	Description *string           `json:"description,omitempty" yaml:"description,omitempty"`
 	Versions    []FServiceVersion `json:"versions,omitempty" yaml:"versions,omitempty"`
+	Documents   []FDocument       `json:"documents,omitempty" yaml:"documents,omitempty"`
+}
+
+// +k8s:deepcopy-gen=true
+type FDocument struct {
+	ID        *string `json:"id,omitempty" yaml:"id,omitempty"`
+	Path      *string `json:"path,omitempty" yaml:"path,omitempty"`
+	Published *bool   `json:"published,omitempty" yaml:"published,omitempty"`
+	Content   *string `json:"-" yaml:"-"`
 }
 
 // id is used for sorting.

file/writer.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -129,16 +131,31 @@ func populateServicePackages(kongState *state.KongState, file *Content,
 		return err
 	}
 
-	for _, p := range packages {
+	for _, sp := range packages {
 		p := FServicePackage{
-			ID:          p.ID,
-			Name:        p.Name,
-			Description: p.Description,
+			ID:          sp.ID,
+			Name:        sp.Name,
+			Description: sp.Description,
 		}
 		versions, err := kongState.ServiceVersions.GetAllByServicePackageID(*p.ID)
 		if err != nil {
 			return err
 		}
+		documents, err := kongState.Documents.GetAllByParent(sp)
+		if err != nil {
+			return err
+		}
+
+		for _, d := range documents {
+			fDocument := FDocument{
+				ID:        d.ID,
+				Path:      d.Path,
+				Published: d.Published,
+				Content:   d.Content,
+			}
+			utils.ZeroOutID(&fDocument, fDocument.Path, config.WithID)
+			p.Documents = append(p.Documents, fDocument)
+		}
 
 		for _, v := range versions {
 			fVersion := FServiceVersion{
@@ -160,6 +177,21 @@ func populateServicePackages(kongState *state.KongState, file *Content,
 					},
 				}
 			}
+			documents, err := kongState.Documents.GetAllByParent(v)
+			if err != nil {
+				return err
+			}
+
+			for _, d := range documents {
+				fDocument := FDocument{
+					ID:        d.ID,
+					Path:      d.Path,
+					Published: d.Published,
+					Content:   d.Content,
+				}
+				utils.ZeroOutID(&fDocument, fDocument.Path, config.WithID)
+				fVersion.Documents = append(fVersion.Documents, fDocument)
+			}
 			utils.ZeroOutID(&fVersion, fVersion.Version, config.WithID)
 			p.Versions = append(p.Versions, fVersion)
 		}
@@ -581,13 +613,45 @@ func writeFile(content *Content, filename string, format Format) error {
 	}
 
 	if filename == "-" {
-		_, err = fmt.Print(string(c))
+		if _, err := fmt.Print(string(c)); err != nil {
+			return errors.Wrap(err, "writing file")
+		}
 	} else {
 		filename = utils.AddExtToFilename(filename, strings.ToLower(string(format)))
-		err = ioutil.WriteFile(filename, c, 0600)
-	}
-	if err != nil {
-		return errors.Wrap(err, "writing file")
+		prefix, _ := filepath.Split(filename)
+		if err := ioutil.WriteFile(filename, c, 0600); err != nil {
+			return errors.Wrap(err, "writing file")
+		}
+		for _, sp := range content.ServicePackages {
+			safePackageName := utils.NameToFilename(*sp.Name)
+			if len(sp.Documents) > 0 {
+				if err := os.MkdirAll(filepath.Join(prefix, safePackageName), 0700); err != nil {
+					return errors.Wrap(err, "creating document directory")
+				}
+				for _, d := range sp.Documents {
+					safeDocPath := utils.NameToFilename(*d.Path)
+					if err := os.WriteFile(filepath.Join(prefix, safePackageName, safeDocPath),
+						[]byte(*d.Content), 0600); err != nil {
+						return errors.Wrap(err, "writing document file")
+					}
+				}
+			}
+			for _, v := range sp.Versions {
+				if len(v.Documents) > 0 {
+					safeVersionName := utils.NameToFilename(*v.Version)
+					if err := os.MkdirAll(filepath.Join(prefix, safePackageName, safeVersionName), 0700); err != nil {
+						return errors.Wrap(err, "creating document directory")
+					}
+					for _, d := range v.Documents {
+						safeDocPath := utils.NameToFilename(*d.Path)
+						if err := os.WriteFile(filepath.Join(prefix, safePackageName, safeVersionName, safeDocPath),
+							[]byte(*d.Content), 0600); err != nil {
+							return errors.Wrap(err, "writing document file")
+						}
+					}
+				}
+			}
+		}
 	}
 	return nil
 }

konnect/client.go

@@ -28,6 +28,7 @@ type Client struct {
 	Auth                  *AuthService
 	ServicePackages       *ServicePackageService
 	ServiceVersions       *ServiceVersionService
+	Documents             *DocumentService
 	ControlPlanes         *ControlPlaneService
 	ControlPlaneRelations *ControlPlaneRelationsService
 	logger                io.Writer
@@ -51,6 +52,7 @@ func NewClient(httpClient *http.Client) (*Client, error) {
 	client.Auth = (*AuthService)(&client.common)
 	client.ServicePackages = (*ServicePackageService)(&client.common)
 	client.ServiceVersions = (*ServiceVersionService)(&client.common)
+	client.Documents = (*DocumentService)(&client.common)
 	client.ControlPlanes = (*ControlPlaneService)(&client.common)
 	client.ControlPlaneRelations = (*ControlPlaneRelationsService)(&client.common)
 	client.logger = os.Stderr