Allow configuration of S3 archiving on a repository (#160)

* Allow configuration of S3 archiving on a repository

* Add --enable-s3-archiving and --edisable-s3-archiving flags to the command validation

api/internal/humiographql/s3-configuration.go

@@ -0,0 +1,55 @@
+package humiographql
+
+import (
+	"fmt"
+	"strings"
+)
+
+type S3Configuration struct {
+	Bucket   string            `graphql:"bucket"`
+	Region   string            `graphql:"region"`
+	Disabled bool              `graphql:"disabled"`
+	Format   S3ArchivingFormat `graphql:"format"`
+}
+
+// IsEnabled - determine if S3Configuration is enabled based on values and the Disabled field
+// to avoid a bool defaulting to false
+func (s *S3Configuration) IsEnabled() bool {
+	if s.IsConfigured() == false {
+		return false
+	}
+	return !s.Disabled
+}
+
+func (s *S3Configuration) IsConfigured() bool {
+	if s.Bucket != "" && s.Region != "" && s.Format != "" {
+		return true
+	}
+	return false
+}
+
+// S3ArchivingFormat - the format in which to store the archived data on S3, either RAW or NDJSON
+type S3ArchivingFormat string
+
+const DefaultS3ArchivingFormat = S3ArchivingFormat("NDSON")
+
+var ValidS3ArchivingFormats = []S3ArchivingFormat{"NDJSON", "RAW"}
+
+// NewS3ArchivingFormat - creates a S3ArchivingFormat and ensures the value is uppercase
+func NewS3ArchivingFormat(format string) (S3ArchivingFormat, error) {
+	f := S3ArchivingFormat(strings.ToUpper(string(format)))
+	err := f.Validate()
+	if err != nil {
+		return "", err
+	}
+	return f, nil
+}
+
+func (f *S3ArchivingFormat) Validate() error {
+	for _, v := range ValidS3ArchivingFormats {
+		if v == *f {
+			return nil
+		}
+	}
+	return fmt.Errorf("invalid S3 archiving format. Valid formats: %s", ValidS3ArchivingFormats)
+}

api/repositories.go

@@ -2,22 +2,25 @@ package api
 
 import (
 	"fmt"
-	graphql "github.com/cli/shurcooL-graphql"
 	"strings"
+
+	graphql "github.com/cli/shurcooL-graphql"
+	"github.com/humio/cli/api/internal/humiographql"
 )
 
 type Repositories struct {
 	client *Client
 }
 
 type Repository struct {
-	ID                     string
-	Name                   string
-	Description            string
-	RetentionDays          float64 `graphql:"timeBasedRetention"`
-	IngestRetentionSizeGB  float64 `graphql:"ingestSizeBasedRetention"`
-	StorageRetentionSizeGB float64 `graphql:"storageSizeBasedRetention"`
-	SpaceUsed              int64   `graphql:"compressedByteSize"`
+	ID                       string
+	Name                     string
+	Description              string
+	RetentionDays            float64                      `graphql:"timeBasedRetention"`
+	IngestRetentionSizeGB    float64                      `graphql:"ingestSizeBasedRetention"`
+	StorageRetentionSizeGB   float64                      `graphql:"storageSizeBasedRetention"`
+	SpaceUsed                int64                        `graphql:"compressedByteSize"`
+	S3ArchivingConfiguration humiographql.S3Configuration `graphql:"s3ArchivingConfiguration"`
 }
 
 func (c *Client) Repositories() *Repositories { return &Repositories{client: c} }
@@ -242,3 +245,87 @@ func (r *Repositories) UpdateDescription(name, description string) error {
 
 	return r.client.Mutate(&mutation, variables)
 }
+
+func (r *Repositories) EnableS3Archiving(name string) error {
+	existingRepo, err := r.Get(name)
+	if err != nil {
+		return err
+	}
+
+	if existingRepo.S3ArchivingConfiguration.IsConfigured() == false {
+		return fmt.Errorf("repository has no configuration for S3 archiving")
+	}
+
+	var mutation struct {
+		S3EnableArchiving struct {
+			// We have to make a selection, so just take __typename
+			Typename graphql.String `graphql:"__typename"`
+		} `graphql:"s3EnableArchiving(repositoryName: $name)"`
+	}
+
+	variables := map[string]interface{}{
+		"name": graphql.String(name),
+	}
+
+	return r.client.Mutate(&mutation, variables)
+}
+
+func (r *Repositories) DisableS3Archiving(name string) error {
+	existingRepo, err := r.Get(name)
+	if err != nil {
+		return err
+	}
+
+	if existingRepo.S3ArchivingConfiguration.IsConfigured() == false {
+		return fmt.Errorf("repository has no configuration for S3 archiving")
+	}
+
+	var mutation struct {
+		S3DisableArchiving struct {
+			// We have to make a selection, so just take __typename
+			Typename graphql.String `graphql:"__typename"`
+		} `graphql:"s3DisableArchiving(repositoryName: $name)"`
+	}
+
+	variables := map[string]interface{}{
+		"name": graphql.String(name),
+	}
+
+	return r.client.Mutate(&mutation, variables)
+}
+
+func (r *Repositories) UpdateS3ArchivingConfiguration(name string, bucket string, region string, format string) error {
+	_, err := r.Get(name)
+	if err != nil {
+		return err
+	}
+
+	if bucket == "" {
+		return fmt.Errorf("bucket name cannot have an empty value")
+	}
+
+	if region == "" {
+		return fmt.Errorf("region cannot have an empty value")
+	}
+
+	archivingFormat, ferr := humiographql.NewS3ArchivingFormat(format)
+	if ferr != nil {
+		return ferr
+	}
+
+	var mutation struct {
+		S3ConfigureArchiving struct {
+			// We have to make a selection, so just take __typename
+			Typename graphql.String `graphql:"__typename"`
+		} `graphql:"s3ConfigureArchiving(repositoryName: $name, bucket: $bucket, region: $region, format: $format)"`
+	}
+
+	variables := map[string]interface{}{
+		"name":   graphql.String(name),
+		"bucket": graphql.String(bucket),
+		"region": graphql.String(region),
+		"format": archivingFormat,
+	}
+
+	return r.client.Mutate(&mutation, variables)
+}

cmd/humioctl/repos.go

@@ -45,6 +45,10 @@ func printRepoDetailsTable(cmd *cobra.Command, repo api.Repository) {
 		{format.String("Ingest Retention (Size)"), ByteCountDecimal(repo.IngestRetentionSizeGB * 1e9)},
 		{format.String("Storage Retention (Size)"), ByteCountDecimal(repo.StorageRetentionSizeGB * 1e9)},
 		{format.String("Retention (Days)"), format.Int(repo.RetentionDays)},
+		{format.String("S3 Archiving Enabled"), format.Bool(repo.S3ArchivingConfiguration.IsEnabled())},
+		{format.String("S3 Archiving Bucket"), format.String(repo.S3ArchivingConfiguration.Bucket)},
+		{format.String("S3 Archiving Region"), format.String(repo.S3ArchivingConfiguration.Region)},
+		{format.String("S3 Archiving Format"), format.String(repo.S3ArchivingConfiguration.Format)},
 	}
 
 	printDetailsTable(cmd, details)

cmd/humioctl/repos_update.go

@@ -21,8 +21,8 @@ import (
 )
 
 func newReposUpdateCmd() *cobra.Command {
-	var allowDataDeletionFlag bool
-	var descriptionFlag stringPtrFlag
+	var allowDataDeletionFlag, enableS3ArchivingFlag, disableS3ArchivingFlag bool
+	var descriptionFlag, s3ArchivingBucketFlag, s3ArchivingRegionFlag, s3ArchivingFormatFlag stringPtrFlag
 	var retentionTimeFlag, ingestSizeBasedRetentionFlag, storageSizeBasedRetentionFlag float64PtrFlag
 
 	cmd := cobra.Command{
@@ -33,10 +33,10 @@ func newReposUpdateCmd() *cobra.Command {
 			repoName := args[0]
 			client := NewApiClient(cmd)
 
-			if descriptionFlag.value == nil && retentionTimeFlag.value == nil && ingestSizeBasedRetentionFlag.value == nil && storageSizeBasedRetentionFlag.value == nil {
+			if descriptionFlag.value == nil && retentionTimeFlag.value == nil && ingestSizeBasedRetentionFlag.value == nil && storageSizeBasedRetentionFlag.value == nil &&
+				enableS3ArchivingFlag == false && disableS3ArchivingFlag == false && s3ArchivingBucketFlag.value == nil && s3ArchivingRegionFlag.value == nil && s3ArchivingFormatFlag.value == nil {
 				exitOnError(cmd, fmt.Errorf("you must specify at least one flag to update"), "Nothing specified to update")
 			}
-
 			if descriptionFlag.value != nil {
 				err := client.Repositories().UpdateDescription(repoName, *descriptionFlag.value)
 				exitOnError(cmd, err, "Error updating repository description")
@@ -54,6 +54,21 @@ func newReposUpdateCmd() *cobra.Command {
 				exitOnError(cmd, err, "Error updating repository storage size based retention")
 			}
 
+			if s3ArchivingBucketFlag.value != nil && s3ArchivingRegionFlag.value != nil && s3ArchivingFormatFlag.value != nil {
+				err := client.Repositories().UpdateS3ArchivingConfiguration(repoName, *s3ArchivingBucketFlag.value, *s3ArchivingRegionFlag.value, *s3ArchivingFormatFlag.value)
+				exitOnError(cmd, err, "Error updating S3 archiving configuration")
+			}
+
+			if disableS3ArchivingFlag == true {
+				err := client.Repositories().DisableS3Archiving(repoName)
+				exitOnError(cmd, err, "Error disabling S3 archiving")
+			}
+
+			if enableS3ArchivingFlag == true {
+				err := client.Repositories().EnableS3Archiving(repoName)
+				exitOnError(cmd, err, "Error enabling S3 archiving")
+			}
+
 			fmt.Fprintf(cmd.OutOrStdout(), "Successfully updated repository %q\n", repoName)
 		},
 	}
@@ -63,6 +78,13 @@ func newReposUpdateCmd() *cobra.Command {
 	cmd.Flags().Var(&retentionTimeFlag, "retention-time", "The retention time in days for the repository.")
 	cmd.Flags().Var(&ingestSizeBasedRetentionFlag, "ingest-size-based-retention", "The ingest size based retention for the repository.")
 	cmd.Flags().Var(&storageSizeBasedRetentionFlag, "storage-size-based-retention", "The storage size based retention for the repository.")
+	cmd.Flags().BoolVar(&enableS3ArchivingFlag, "enable-s3-archiving", false, "Enable S3 Archiving")
+	cmd.Flags().BoolVar(&disableS3ArchivingFlag, "disable-s3-archiving", false, "Disable S3 Archiving")
+	cmd.Flags().Var(&s3ArchivingBucketFlag, "s3-archiving-bucket", "The name of the bucket to be used for S3 Archiving")
+	cmd.Flags().Var(&s3ArchivingRegionFlag, "s3-archiving-region", "The S3 region to be used for S3 Archiving")
+	cmd.Flags().Var(&s3ArchivingFormatFlag, "s3-archiving-format", "The S3 archiving format to be used for S3 Archiving. Formats: RAW, NDJSON")
+	cmd.MarkFlagsRequiredTogether("s3-archiving-bucket", "s3-archiving-region", "s3-archiving-format")
+	cmd.MarkFlagsMutuallyExclusive("enable-s3-archiving", "disable-s3-archiving")
 
 	return &cmd
 }