feat: use bitnami etcd chart

Signed-off-by: zyy17 <zyylsxm@gmail.com>

pkg/cmd/gtctl/cluster/create/create.go

@@ -51,7 +51,7 @@ type ClusterCliOptions struct {
 	EtcdChartVersion               string
 	EtcdStorageClassName           string
 	EtcdStorageSize                string
-	EtcdDataDir                    string
+	EtcdClusterSize                string
 
 	// The options for deploying GreptimeDBCluster in bare-metal.
 	BareMetal          bool
@@ -96,7 +96,7 @@ func NewCreateClusterCommand(l logger.Logger) *cobra.Command {
 	cmd.Flags().StringVar(&options.EtcdNamespace, "etcd-namespace", "default", "The namespace of etcd cluster.")
 	cmd.Flags().StringVar(&options.EtcdStorageClassName, "etcd-storage-class-name", "standard", "The etcd storage class name.")
 	cmd.Flags().StringVar(&options.EtcdStorageSize, "etcd-storage-size", "10Gi", "the etcd persistent volume size.")
-	cmd.Flags().StringVar(&options.EtcdDataDir, "etcd-data-dir", "/var/lib/etcd", "the etcd data directory.")
+	cmd.Flags().StringVar(&options.EtcdClusterSize, "etcd-cluster-size", "1", "the etcd cluster size.")
 	cmd.Flags().BoolVar(&options.BareMetal, "bare-metal", false, "Deploy the greptimedb cluster on bare-metal environment.")
 	cmd.Flags().StringVar(&options.GreptimeBinVersion, "greptime-bin-version", "", "The version of greptime binary(can be override by config file).")
 	cmd.Flags().StringVar(&options.Config, "config", "", "Configuration to deploy the greptimedb cluster on bare-metal environment.")
@@ -261,7 +261,7 @@ func deployEtcdCluster(ctx context.Context, l logger.Logger, options *ClusterCli
 		EtcdChartVersion:     options.EtcdChartVersion,
 		EtcdStorageClassName: options.EtcdStorageClassName,
 		EtcdStorageSize:      options.EtcdStorageSize,
-		EtcdDataDir:          options.EtcdDataDir,
+		EtcdClusterSize:      options.EtcdClusterSize,
 		ConfigValues:         options.Set.etcdConfig,
 	}
 

pkg/deployer/k8s/constants.go

@@ -16,9 +16,11 @@ package k8s
 
 const (
 	GreptimeDBChartName         = "greptimedb"
-	GreptimeDBEtcdChartName     = "greptimedb-etcd"
 	GreptimeDBOperatorChartName = "greptimedb-operator"
 
 	GreptimeChartIndexURL           = "https://raw.githubusercontent.com/GreptimeTeam/helm-charts/gh-pages/index.yaml"
 	GreptimeChartReleaseDownloadURL = "https://github.com/GreptimeTeam/helm-charts/releases/download"
+
+	EtcdBitnamiOCIRegistry  = "oci://registry-1.docker.io/bitnamicharts/etcd"
+	DefaultEtcdChartVersion = "9.2.0"
 )

pkg/deployer/k8s/deployer.go

@@ -41,19 +41,21 @@ var _ Interface = &deployer{}
 type Option func(*deployer)
 
 func NewDeployer(l logger.Logger, opts ...Option) (Interface, error) {
+	r, err := helm.NewRender()
+	if err != nil {
+		return nil, err
+	}
+
 	d := &deployer{
-		render: &helm.Render{},
+		render: r,
 		logger: l,
 	}
 
 	for _, opt := range opts {
 		opt(d)
 	}
 
-	var (
-		client *kube.Client
-		err    error
-	)
+	var client *kube.Client
 	if !d.dryRun {
 		client, err = kube.NewClient("")
 		if err != nil {
@@ -182,18 +184,19 @@ func (d *deployer) CreateEtcdCluster(ctx context.Context, name string, options *
 		return err
 	}
 
+	// TODO(zyy17): Maybe we can set this in the top level configures.
+	const (
+		disableRBACConfig = "auth.rbac.create=false,auth.rbac.token.enabled=false"
+	)
+	options.ConfigValues += disableRBACConfig
+
 	values, err := d.render.GenerateHelmValues(*options)
 	if err != nil {
 		return err
 	}
 	d.logger.V(3).Infof("create etcd cluster with values: %v", values)
 
-	downloadURL, err := d.getChartDownloadURL(ctx, GreptimeDBEtcdChartName, options.EtcdChartVersion)
-	if err != nil {
-		return err
-	}
-
-	chart, err := d.render.LoadChartFromRemoteCharts(ctx, downloadURL)
+	chart, err := d.render.Pull(ctx, EtcdBitnamiOCIRegistry, DefaultEtcdChartVersion)
 	if err != nil {
 		return err
 	}

pkg/deployer/types.go

@@ -91,10 +91,11 @@ type DeleteGreptimeDBClusterOption struct{}
 type CreateEtcdClusterOptions struct {
 	EtcdChartVersion string
 
+	// The parameters reference: https://artifacthub.io/packages/helm/bitnami/etcd.
+	EtcdClusterSize      string `helm:"replicaCount"`
 	ImageRegistry        string `helm:"image.registry"`
-	EtcdStorageClassName string `helm:"storage.storageClassName"`
-	EtcdStorageSize      string `helm:"storage.volumeSize"`
-	EtcdDataDir          string `helm:"storage.dataDir"`
+	EtcdStorageClassName string `helm:"persistence.storageClass"`
+	EtcdStorageSize      string `helm:"persistence.size"`
 	ConfigValues         string `helm:"*"`
 }
 

pkg/helm/helm.go

@@ -21,33 +21,83 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
 	"reflect"
 	"strings"
 
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/registry"
 	. "helm.sh/helm/v3/pkg/repo"
 	"helm.sh/helm/v3/pkg/strvals"
 	"sigs.k8s.io/yaml"
+
+	fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file"
 )
 
 const (
 	helmFieldTag = "helm"
 )
 
+const (
+	defaultChartsCache = ".gtctl/charts-cache"
+)
+
 type TemplateRender interface {
 	GenerateManifests(ctx context.Context, releaseName, namespace string, chart *chart.Chart, values map[string]interface{}) ([]byte, error)
 }
 
 type Render struct {
 	// indexFile is the index file of the remote charts.
 	indexFile *IndexFile
+
+	// chartCache is the cache directory for the charts.
+	chartsCacheDir string
 }
 
 var _ TemplateRender = &Render{}
 
+func NewRender() (*Render, error) {
+	homeDir, err := os.UserHomeDir()
+	if err != nil {
+		return nil, err
+	}
+
+	chartsCacheDir := filepath.Join(homeDir, defaultChartsCache)
+
+	if err := fileutils.CreateDirIfNotExists(chartsCacheDir); err != nil {
+		return nil, err
+	}
+
+	return &Render{
+		chartsCacheDir: chartsCacheDir,
+	}, nil
+}
+
 func (r *Render) LoadChartFromRemoteCharts(ctx context.Context, downloadURL string) (*chart.Chart, error) {
+	parsedURL, err := url.Parse(downloadURL)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		packageName = path.Base(parsedURL.Path)
+		cachePath   = filepath.Join(r.chartsCacheDir, packageName)
+	)
+
+	if r.isInChartsCache(packageName) {
+		data, err := os.ReadFile(cachePath)
+		if err != nil {
+			return nil, err
+		}
+		return loader.LoadArchive(bytes.NewReader(data))
+	}
+
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
 		return nil, err
@@ -64,6 +114,10 @@ func (r *Render) LoadChartFromRemoteCharts(ctx context.Context, downloadURL stri
 		return nil, err
 	}
 
+	if err := os.WriteFile(cachePath, body, 0644); err != nil {
+		return nil, err
+	}
+
 	return loader.LoadArchive(bytes.NewReader(body))
 }
 
@@ -172,6 +226,47 @@ func (r *Render) GetIndexFile(ctx context.Context, indexURL string) (*IndexFile,
 	return indexFile, nil
 }
 
+// Pull pulls the chart from the remote OCI registry, for example, oci://registry-1.docker.io/bitnamicharts/etcd.
+func (r *Render) Pull(_ context.Context, OCIRegistry, version string) (*chart.Chart, error) {
+	packageName := r.packageName(path.Base(OCIRegistry), version)
+	if !r.isInChartsCache(packageName) {
+		registryClient, err := registry.NewClient(
+			registry.ClientOptDebug(false),
+			registry.ClientOptEnableCache(false),
+			registry.ClientOptCredentialsFile(""),
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		cfg := new(action.Configuration)
+		cfg.RegistryClient = registryClient
+
+		// Create a pull action
+		client := action.NewPullWithOpts(action.WithConfig(cfg))
+		client.Settings = cli.New()
+		client.Version = version
+		client.DestDir = r.chartsCacheDir
+
+		// Execute the pull action
+		if _, err := client.Run(OCIRegistry); err != nil {
+			return nil, err
+		}
+	}
+
+	data, err := os.ReadFile(filepath.Join(r.chartsCacheDir, packageName))
+	if err != nil {
+		return nil, err
+	}
+
+	return loader.LoadArchive(bytes.NewReader(data))
+}
+
+func (r *Render) isInChartsCache(packageName string) bool {
+	res, _ := fileutils.IsFileExists(filepath.Join(r.chartsCacheDir, packageName))
+	return res
+}
+
 func (r *Render) newHelmClient(releaseName, namespace string) (*action.Install, error) {
 	helmClient := action.NewInstall(new(action.Configuration))
 	helmClient.DryRun = true
@@ -184,6 +279,10 @@ func (r *Render) newHelmClient(releaseName, namespace string) (*action.Install,
 	return helmClient, nil
 }
 
+func (r *Render) packageName(chartName, version string) string {
+	return fmt.Sprintf("%s-%s.tgz", chartName, version)
+}
+
 // loadIndex is from 'helm/pkg/index.go'.
 func loadIndex(data []byte, source string) (*IndexFile, error) {
 	i := &IndexFile{}

pkg/helm/helm_test.go

@@ -173,7 +173,7 @@ func TestRender_GenerateEtcdHelmValues(t *testing.T) {
 		ImageRegistry:        "registry.cn-hangzhou.aliyuncs.com",
 		EtcdStorageClassName: "ebs-sc",
 		EtcdStorageSize:      "11Gi",
-		EtcdDataDir:          "/var/etcd",
+		EtcdClusterSize:      "3",
 		ConfigValues:         "image.tag=latest",
 	}
 
@@ -185,9 +185,9 @@ func TestRender_GenerateEtcdHelmValues(t *testing.T) {
 
 	ArgsStr := []string{
 		"image.registry=registry.cn-hangzhou.aliyuncs.com",
-		"storage.storageClassName=ebs-sc",
-		"storage.volumeSize=11Gi",
-		"storage.dataDir=/var/etcd",
+		"persistence.storageClass=ebs-sc",
+		"persistence.size=11Gi",
+		"replicaCount=3",
 		"image.tag=latest",
 	}