feat: support bitnamic etcd chart and add charts cache (#135)

* feat: use bitnami etcd chart

Signed-off-by: zyy17 <zyylsxm@gmail.com>

* refactor: add 'HelmManager' and simplify the interface

* fix: e2e error

* refactor: expand e2e timeout value

* refactor: use fixed kind node image for k8s api compatible

* chore: fix format error

---------

Signed-off-by: zyy17 <zyylsxm@gmail.com>

Makefile

@@ -48,7 +48,7 @@ setup-e2e: ## Setup e2e test environment.
 
 .PHONY: e2e
 e2e: gtctl setup-e2e ## Run e2e.
-	go test -timeout 8m -v ./tests/e2e/... && kind delete clusters $(CLUSTER)
+	go test -timeout 10m -v ./tests/e2e/... && kind delete clusters $(CLUSTER)
 
 .PHONY: lint
 lint: golangci-lint gtctl ## Run lint.

hack/e2e/setup-e2e-env.sh

@@ -21,6 +21,7 @@ set -o pipefail
 CLUSTER=e2e-cluster
 REGISTRY_NAME=kind-registry
 REGISTRY_PORT=5001
+KIND_NODE_IMAGE=kindest/node:v1.24.15
 
 function check_prerequisites() {
     if ! hash docker 2>/dev/null; then
@@ -59,7 +60,7 @@ function create_kind_cluster() {
     done
 
     # create a cluster with the local registry enabled in containerd
-    cat <<EOF | kind create cluster --name "${CLUSTER}" --config=-
+    cat <<EOF | kind create cluster --image ${KIND_NODE_IMAGE} --name "${CLUSTER}" --config=-
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 containerdConfigPatches:

pkg/cmd/gtctl/cluster/create/create.go

@@ -51,7 +51,7 @@ type ClusterCliOptions struct {
 	EtcdChartVersion               string
 	EtcdStorageClassName           string
 	EtcdStorageSize                string
-	EtcdDataDir                    string
+	EtcdClusterSize                string
 
 	// The options for deploying GreptimeDBCluster in bare-metal.
 	BareMetal          bool
@@ -96,7 +96,7 @@ func NewCreateClusterCommand(l logger.Logger) *cobra.Command {
 	cmd.Flags().StringVar(&options.EtcdNamespace, "etcd-namespace", "default", "The namespace of etcd cluster.")
 	cmd.Flags().StringVar(&options.EtcdStorageClassName, "etcd-storage-class-name", "standard", "The etcd storage class name.")
 	cmd.Flags().StringVar(&options.EtcdStorageSize, "etcd-storage-size", "10Gi", "the etcd persistent volume size.")
-	cmd.Flags().StringVar(&options.EtcdDataDir, "etcd-data-dir", "/var/lib/etcd", "the etcd data directory.")
+	cmd.Flags().StringVar(&options.EtcdClusterSize, "etcd-cluster-size", "1", "the etcd cluster size.")
 	cmd.Flags().BoolVar(&options.BareMetal, "bare-metal", false, "Deploy the greptimedb cluster on bare-metal environment.")
 	cmd.Flags().StringVar(&options.GreptimeBinVersion, "greptime-bin-version", "", "The version of greptime binary(can be override by config file).")
 	cmd.Flags().StringVar(&options.Config, "config", "", "Configuration to deploy the greptimedb cluster on bare-metal environment.")
@@ -261,7 +261,7 @@ func deployEtcdCluster(ctx context.Context, l logger.Logger, options *ClusterCli
 		EtcdChartVersion:     options.EtcdChartVersion,
 		EtcdStorageClassName: options.EtcdStorageClassName,
 		EtcdStorageSize:      options.EtcdStorageSize,
-		EtcdDataDir:          options.EtcdDataDir,
+		EtcdClusterSize:      options.EtcdClusterSize,
 		ConfigValues:         options.Set.etcdConfig,
 	}
 

pkg/deployer/k8s/deployer.go

@@ -29,31 +29,33 @@ import (
 )
 
 type deployer struct {
-	render  *helm.Render
-	client  *kube.Client
-	timeout time.Duration
-	logger  logger.Logger
-	dryRun  bool
+	helmManager *helm.Manager
+	client      *kube.Client
+	timeout     time.Duration
+	logger      logger.Logger
+	dryRun      bool
 }
 
 var _ Interface = &deployer{}
 
 type Option func(*deployer)
 
 func NewDeployer(l logger.Logger, opts ...Option) (Interface, error) {
+	hm, err := helm.NewManager(l)
+	if err != nil {
+		return nil, err
+	}
+
 	d := &deployer{
-		render: &helm.Render{},
-		logger: l,
+		helmManager: hm,
+		logger:      l,
 	}
 
 	for _, opt := range opts {
 		opt(d)
 	}
 
-	var (
-		client *kube.Client
-		err    error
-	)
+	var client *kube.Client
 	if !d.dryRun {
 		client, err = kube.NewClient("")
 		if err != nil {
@@ -116,28 +118,11 @@ func (d *deployer) CreateGreptimeDBCluster(ctx context.Context, name string, opt
 		return err
 	}
 
-	values, err := d.render.GenerateHelmValues(*options)
-	if err != nil {
-		return err
-	}
-	d.logger.V(3).Infof("create greptimedb cluster with values: %v", values)
-
-	downloadURL, err := d.getChartDownloadURL(ctx, GreptimeDBChartName, options.GreptimeDBChartVersion)
-	if err != nil {
-		return err
-	}
-
-	chart, err := d.render.LoadChartFromRemoteCharts(ctx, downloadURL)
+	manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, helm.GreptimeDBChartName, options.GreptimeDBChartVersion, *options)
 	if err != nil {
 		return err
 	}
 
-	manifests, err := d.render.GenerateManifests(ctx, resourceName, resourceNamespace, chart, values)
-	if err != nil {
-		return err
-	}
-	d.logger.V(3).Infof("create greptimedb cluster with manifests: %s", string(manifests))
-
 	if d.dryRun {
 		d.logger.V(0).Info(string(manifests))
 		return nil
@@ -182,27 +167,16 @@ func (d *deployer) CreateEtcdCluster(ctx context.Context, name string, options *
 		return err
 	}
 
-	values, err := d.render.GenerateHelmValues(*options)
-	if err != nil {
-		return err
-	}
-	d.logger.V(3).Infof("create etcd cluster with values: %v", values)
-
-	downloadURL, err := d.getChartDownloadURL(ctx, GreptimeDBEtcdChartName, options.EtcdChartVersion)
-	if err != nil {
-		return err
-	}
-
-	chart, err := d.render.LoadChartFromRemoteCharts(ctx, downloadURL)
-	if err != nil {
-		return err
-	}
+	// TODO(zyy17): Maybe we can set this in the top level configures.
+	const (
+		disableRBACConfig = "auth.rbac.create=false,auth.rbac.token.enabled=false"
+	)
+	options.ConfigValues += disableRBACConfig
 
-	manifests, err := d.render.GenerateManifests(ctx, resourceName, resourceNamespace, chart, values)
+	manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, helm.EtcdBitnamiOCIRegistry, helm.DefaultEtcdChartVersion, *options)
 	if err != nil {
 		return err
 	}
-	d.logger.V(3).Infof("create etcd cluster with manifests: %s", string(manifests))
 
 	if d.dryRun {
 		d.logger.V(0).Info(string(manifests))
@@ -231,27 +205,10 @@ func (d *deployer) CreateGreptimeDBOperator(ctx context.Context, name string, op
 		return err
 	}
 
-	values, err := d.render.GenerateHelmValues(*options)
+	manifests, err := d.helmManager.LoadAndRenderChart(ctx, resourceName, resourceNamespace, helm.GreptimeDBOperatorChartName, options.GreptimeDBOperatorChartVersion, *options)
 	if err != nil {
 		return err
 	}
-	d.logger.V(3).Infof("create greptimedb-operator with values: %v", values)
-
-	downloadURL, err := d.getChartDownloadURL(ctx, GreptimeDBOperatorChartName, options.GreptimeDBOperatorChartVersion)
-	if err != nil {
-		return err
-	}
-
-	chart, err := d.render.LoadChartFromRemoteCharts(ctx, downloadURL)
-	if err != nil {
-		return err
-	}
-
-	manifests, err := d.render.GenerateManifests(ctx, GreptimeDBOperatorChartName, resourceNamespace, chart, values)
-	if err != nil {
-		return err
-	}
-	d.logger.V(3).Infof("create greptimedb-operator with manifests: %s", string(manifests))
 
 	if d.dryRun {
 		d.logger.V(0).Info(string(manifests))
@@ -277,29 +234,3 @@ func (d *deployer) splitNamescapedName(name string) (string, string, error) {
 
 	return split[0], split[1], nil
 }
-
-func (d *deployer) getChartDownloadURL(ctx context.Context, chartName, version string) (string, error) {
-	indexFile, err := d.render.GetIndexFile(ctx, GreptimeChartIndexURL)
-	if err != nil {
-		return "", err
-	}
-
-	var downloadURL string
-	if version == "" {
-		chartVersion, err := d.render.GetLatestChart(indexFile, chartName)
-		if err != nil {
-			return "", err
-		}
-		downloadURL = chartVersion.URLs[0]
-		d.logger.V(3).Infof("get latest chart '%s', version '%s', url: '%s'",
-			chartName, chartVersion.Version, downloadURL)
-	} else {
-		// The download URL example: 'https://github.com/GreptimeTeam/helm-charts/releases/download/greptimedb-0.1.1-alpha.3/greptimedb-0.1.1-alpha.3.tgz'.
-		chartName := chartName + "-" + version
-		downloadURL = fmt.Sprintf("%s/%s/%s.tgz", GreptimeChartReleaseDownloadURL, chartName, chartName)
-		d.logger.V(3).Infof("get given version chart '%s', version '%s', url: '%s'",
-			chartName, version, downloadURL)
-	}
-
-	return downloadURL, nil
-}

pkg/deployer/types.go

@@ -91,10 +91,11 @@ type DeleteGreptimeDBClusterOption struct{}
 type CreateEtcdClusterOptions struct {
 	EtcdChartVersion string
 
+	// The parameters reference: https://artifacthub.io/packages/helm/bitnami/etcd.
+	EtcdClusterSize      string `helm:"replicaCount"`
 	ImageRegistry        string `helm:"image.registry"`
-	EtcdStorageClassName string `helm:"storage.storageClassName"`
-	EtcdStorageSize      string `helm:"storage.volumeSize"`
-	EtcdDataDir          string `helm:"storage.dataDir"`
+	EtcdStorageClassName string `helm:"persistence.storageClass"`
+	EtcdStorageSize      string `helm:"persistence.size"`
 	ConfigValues         string `helm:"*"`
 }
 

pkg/deployer/k8s/constants.go → pkg/helm/constants.go

@@ -12,13 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package k8s
+package helm
 
 const (
-	GreptimeDBChartName         = "greptimedb"
-	GreptimeDBEtcdChartName     = "greptimedb-etcd"
-	GreptimeDBOperatorChartName = "greptimedb-operator"
+	DefaultChartsCache = ".gtctl/charts-cache"
 
 	GreptimeChartIndexURL           = "https://raw.githubusercontent.com/GreptimeTeam/helm-charts/gh-pages/index.yaml"
 	GreptimeChartReleaseDownloadURL = "https://github.com/GreptimeTeam/helm-charts/releases/download"
+
+	GreptimeDBChartName         = "greptimedb"
+	GreptimeDBOperatorChartName = "greptimedb-operator"
+	EtcdBitnamiOCIRegistry      = "oci://registry-1.docker.io/bitnamicharts/etcd"
+	DefaultEtcdChartVersion     = "9.2.0"
 )