Introduce Snowflake backend for Theia

.gitignore

@@ -5,6 +5,12 @@ bin
 .vscode/
 .idea/
 
+# VIM
+.*.swp
+
+# Emacs
+*~
+
 # Common backup files
 *.bak
 
@@ -16,3 +22,5 @@ plugins/grafana-custom-plugins/grafana-sankey-plugin/coverage/
 plugins/grafana-custom-plugins/grafana-chord-plugin/node_modules/
 plugins/grafana-custom-plugins/grafana-chord-plugin/dist/
 plugins/grafana-custom-plugins/grafana-chord-plugin/coverage/
+
+.golangci-bin

snowflake/.gitignore

@@ -0,0 +1 @@
+creds

snowflake/Makefile

@@ -0,0 +1,16 @@
+GO                 ?= go
+BINDIR := $(CURDIR)/bin
+
+all: bin
+
+.PHONY: bin
+bin:
+	$(GO) build -o $(BINDIR)/theia-sf antrea.io/theia/snowflake
+
+.PHONY: test
+test:
+	$(GO) test -v ./...
+
+.PHONY: clean
+clean:
+	rm -rf bin

snowflake/README.md

@@ -0,0 +1,119 @@
+# Theia with Snowflake
+
+We are introducing the ability to use Snowflake as the storage and computing
+platform for Theia. When using Snowflake, it is no longer necessary to run
+ClickHouse DB (flow records are stored in a Snowflake database) or Spark (flow
+processing is done by Snowflake virtual warehouses). Using Snowflake for Theia
+means that you have to bring your own AWS and Snowflake accounts (you will be
+charged for resource usage) and some features available with "standard" Theia
+are not available yet with Snowflake.
+
+## Getting started
+
+### Prerequisites
+
+Theia with Snowflake requires Antrea >= v1.9.0 and Theia >= v0.3.0.
+
+### Install the theia-sf CLI
+
+Because it is not yet distributed as a release asset, you will need to build the
+CLI yourself, which requires Git, Golang and Make.
+
+```bash
+git clone git@github.com:antrea-io/theia.git
+cd theia/snowflake
+make
+```
+
+### Configure AWS credentials
+
+Follow the steps in the [AWS
+documentation](https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials)
+to specify credentials that can be used by the AWS Go SDK. Either configure the
+~/.aws/credentials file or set the required environment variables.
+
+You can also export the `AWS_REGION` environment variable, set to your preferred
+region.
+
+### Configure Snowflake credentials
+
+Export the following environment variables: `SNOWFLAKE_ACCOUNT`,
+`SNOWFLAKE_USER`, `SNOWFLAKE_PASSWORD`.
+
+### Create an S3 bucket to store infrastructure state
+
+You may skip this step if you already have a bucket that you want to use.
+
+```bash
+./bin/theia-sf create-bucket
+```
+
+Retrieve the bucket name output by the command.
+
+### Create a KMS key to encrypt infrastructure state
+
+You may skip this step if you already have a KMS key that you want to use. If
+you choose *not* to use a KMS key, Snowflake credentials will be stored in your
+S3 bucket in clear text.
+
+```bash
+./bin/theia-sf create-kms-key
+```
+
+Retrieve the key ID output by the command.
+
+### Provision all cloud resources
+
+```bash
+./bin/theia-sf onboard --bucket-name <BUCKET NAME> --key-id <KEY ID>
+```
+
+The command output will include a table like this one, with important information:
+
+```text
++----------------------------+------------------------------------------------------------------+
+| Region                     | us-west-2                                                        |
+| Bucket Name                | antrea-flows-93e9ojn80fgn5vwt                                    |
+| Bucket Flows Folder        | flows                                                            |
+| Snowflake Database Name    | ANTREA_93E9OJN80FGN5VWT                                          |
+| Snowflake Schema Name      | THEIA                                                            |
+| Snowflake Flows Table Name | FLOWS                                                            |
+| SNS Topic ARN              | arn:aws:sns:us-west-2:867393676014:antrea-flows-93e9ojn80fgn5vwt |
+| SQS Queue ARN              | arn:aws:sqs:us-west-2:867393676014:antrea-flows-93e9ojn80fgn5vwt |
++----------------------------+------------------------------------------------------------------+
+```
+
+### Configure the Flow Aggregator in your cluster(s)
+
+```bash
+helm repo add antrea https://charts.antrea.io
+helm repo update
+helm install antrea antrea/antrea -n kube-system --set featureGates.FlowExporter=true
+helm install flow-aggregator antrea/flow-aggregator \
+     --set s3Uploader.enable=true \
+     --set s3Uploader.bucketName=<BUCKET NAME> \
+     --set s3Uploader.bucketPrefix=flows \
+     --set s3Uploader.awsCredentials.aws_access_key_id=<AWS ACCESS KEY ID> \
+     --set s3Uploader.awsCredentials.aws_secret_access_key=<AWS SECRET ACCESS KEY> \
+     -n flow-aggregator --create-namespace
+```
+
+## Clean up
+
+Follow these steps if you want to delete all resources created by
+`theia-sf`. Just like for [onboarding](#getting-started), AWS credentials and
+Snowflake credentials are required.
+
+```bash
+# always call offboard with the same arguments as onboard!
+./bin/theia-sf offboard --bucket-name <BUCKET NAME> --key-id <KEY ID>
+# if you created a KMS key and you want to schedule it for deletion
+./bin/theia-sf delete-kms-key --key-id <KEY ID>
+# if you created an S3 bucket to store infra state and you want to delete it
+./bin/theia-sf delete-bucket --name <BUCKET NAME>
+```
+
+## Running applications
+
+We are in the process of adding support for applications to Snowflake-powered
+Theia, starting with NetworkPolicy recommendation.

snowflake/cmd/createBucket.go

@@ -0,0 +1,108 @@
+// Copyright 2022 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/dustinkirkland/golang-petname"
+	"github.com/spf13/cobra"
+
+	s3client "antrea.io/theia/snowflake/pkg/aws/client/s3"
+)
+
+// createBucketCmd represents the create-bucket command
+var createBucketCmd = &cobra.Command{
+	Use:   "create-bucket",
+	Short: "Create an AWS S3 bucket",
+	Long: `This command creates a new S3 bucket in your AWS account. This is
+useful if you don't already have a bucket available to store infrastructure
+state (such a bucket is required for the "onboard" command).
+
+Before calling this command, ensure that AWS credentials are available:
+https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials
+
+To create a bucket with a specific name:
+"theia-sf create-bucket --name this-is-the-name-i-want"
+
+To create a bucket with a random name in a specific non-default region:
+"theia-sf create-bucket --region us-east-2"`,
+	Args: cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		region, _ := cmd.Flags().GetString("region")
+		bucketName, _ := cmd.Flags().GetString("name")
+		bucketPrefix, _ := cmd.Flags().GetString("prefix")
+		if bucketName == "" {
+			suffix := petname.Generate(4, "-")
+			bucketName = fmt.Sprintf("%s-%s", bucketPrefix, suffix)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
+		if err != nil {
+			return fmt.Errorf("unable to load AWS SDK config: %w", err)
+
+		}
+		s3Client := s3client.GetClient(awsCfg)
+		if err := createBucket(ctx, s3Client, bucketName, region); err != nil {
+			return err
+		}
+		fmt.Printf("Bucket name: %s\n", bucketName)
+		return nil
+	},
+}
+
+func createBucket(ctx context.Context, s3Client s3client.Interface, name string, region string) error {
+	logger := logger.WithValues("bucket", name, "region", region)
+	logger.Info("Checking if S3 bucket exists")
+	_, err := s3Client.HeadBucket(ctx, &s3.HeadBucketInput{
+		Bucket: &name,
+	})
+	var notFoundError *s3types.NotFound
+	if err != nil && !errors.As(err, &notFoundError) {
+		return fmt.Errorf("error when looking for bucket '%s': %w", name, err)
+	}
+	if notFoundError == nil {
+		logger.Info("S3 bucket already exists")
+		return nil
+	}
+	logger.Info("Creating S3 bucket")
+	if _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
+		Bucket: &name,
+		ACL:    s3types.BucketCannedACLPrivate,
+		CreateBucketConfiguration: &s3types.CreateBucketConfiguration{
+			LocationConstraint: s3types.BucketLocationConstraint(region),
+		},
+	}); err != nil {
+		return fmt.Errorf("error when creating bucket '%s': %w", name, err)
+	}
+	logger.Info("Created S3 bucket")
+	return nil
+}
+
+func init() {
+	rootCmd.AddCommand(createBucketCmd)
+
+	createBucketCmd.Flags().String("region", GetEnv("AWS_REGION", defaultRegion), "region where bucket should be created")
+	createBucketCmd.Flags().String("name", "", "name of bucket to create")
+	createBucketCmd.Flags().String("prefix", "antrea", "prefix to use for bucket name (with auto-generated suffix)")
+	createBucketCmd.MarkFlagsMutuallyExclusive("name", "prefix")
+}

snowflake/cmd/createKmsKey.go

@@ -0,0 +1,80 @@
+// Copyright 2022 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/kms"
+	"github.com/spf13/cobra"
+
+	kmsclient "antrea.io/theia/snowflake/pkg/aws/client/kms"
+)
+
+// createKmsKeyCmd represents the create-kms-key command
+var createKmsKeyCmd = &cobra.Command{
+	Use:   "create-kms-key",
+	Short: "Create an AWS KMS key",
+	Long: `This command creates a new KMS key in your AWS account. This key
+can be used to encrypt infrastructure state in the backend (S3 bucket). If you
+already have a KMS key that you want to use, you won't need this command.
+
+Before calling this command, ensure that AWS credentials are available:
+https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/#specifying-credentials
+
+To create a KMS key and obtain its ID:
+"theia-sf create-kms-key"`,
+	Args: cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		region, _ := cmd.Flags().GetString("region")
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region))
+		if err != nil {
+			return fmt.Errorf("unable to load AWS SDK config: %w", err)
+
+		}
+		kmsClient := kmsclient.GetClient(awsCfg)
+		keyID, err := createKey(ctx, kmsClient)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Key ID: %s\n", keyID)
+		return nil
+	},
+}
+
+func createKey(ctx context.Context, kmsClient kmsclient.Interface) (string, error) {
+	logger.Info("Creating key")
+	description := "This key was created by theia-sf; it is used to encrypt infrastructure state"
+	output, err := kmsClient.CreateKey(ctx, &kms.CreateKeyInput{
+		Description: &description,
+		// we use default parameters for everything else
+	})
+	if err != nil {
+		return "", fmt.Errorf("error when creating key: %w", err)
+	}
+	logger.Info("Created key")
+	return *output.KeyMetadata.KeyId, nil
+}
+
+func init() {
+	rootCmd.AddCommand(createKmsKeyCmd)
+
+	createKmsKeyCmd.Flags().String("region", GetEnv("AWS_REGION", defaultRegion), "region where key should be created")
+}

snowflake/cmd/defaults.go

@@ -0,0 +1,19 @@
+// Copyright 2022 Antrea Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+const (
+	defaultRegion = "us-west-2"
+)