Skip to content

Commit

Permalink
Merge branch 'master' into YDBOPS-9692
Browse files Browse the repository at this point in the history
  • Loading branch information
kobzonega committed Jun 14, 2024
2 parents 6659d67 + 11716fc commit b8a7aec
Show file tree
Hide file tree
Showing 23 changed files with 1,398 additions and 731 deletions.
6 changes: 6 additions & 0 deletions api/v1alpha1/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ const (
DefaultDatabaseDomain = "Root"
DefaultDatabaseEncryptionPin = "EmptyPin"

LabelDeploymentKey = "deployment"
LabelDeploymentValueKubernetes = "kubernetes"
LabelSharedDatabaseKey = "shared"
LabelSharedDatabaseValueTrue = "true"
LabelSharedDatabaseValueFalse = "false"

AnnotationUpdateStrategyOnDelete = "ydb.tech/update-strategy-on-delete"
AnnotationUpdateDNSPolicy = "ydb.tech/update-dns-policy"
AnnotationSkipInitialization = "ydb.tech/skip-initialization"
Expand Down
4 changes: 2 additions & 2 deletions deploy/ydb-operator/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.5.7
version: 0.5.13

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "0.5.7"
appVersion: "0.5.13"
25 changes: 19 additions & 6 deletions internal/controllers/constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,23 @@ const (
OldStorageInitializedCondition = "StorageReady"
OldDatabaseInitializedCondition = "TenantInitialized"

StoragePausedCondition = "StoragePaused"
StorageInitializedCondition = "StorageInitialized"
StorageNodeSetReadyCondition = "StorageNodeSetReady"
DatabasePausedCondition = "DatabasePaused"
DatabaseInitializedCondition = "DatabaseInitialized"
DatabaseNodeSetReadyCondition = "DatabaseNodeSetReady"
StoragePreparedCondition = "StoragePrepared"
StorageInitializedCondition = "StorageInitialized"
StorageProvisionedCondition = "StorageProvisioned"
StoragePausedCondition = "StoragePaused"
StorageReadyCondition = "StorageReady"

DatabasePreparedCondition = "DatabasePrepared"
DatabaseInitializedCondition = "DatabaseInitialized"
DatabaseProvisionedCondition = "DatabaseProvisioned"
DatabasePausedCondition = "DatabasePaused"
DatabaseReadyCondition = "DatabaseReady"

NodeSetPreparedCondition = "NodeSetPrepared"
NodeSetProvisionedCondition = "NodeSetProvisioned"
NodeSetReadyCondition = "NodeSetReady"
NodeSetPausedCondition = "NodeSetPaused"

RemoteResourceSyncedCondition = "ResourceSynced"

Stop = true
Expand All @@ -48,6 +59,7 @@ const (
DatabasePaused ClusterState = "Paused"

DatabaseNodeSetPending ClusterState = "Pending"
DatabaseNodeSetPreparing ClusterState = "Preparing"
DatabaseNodeSetProvisioning ClusterState = "Provisioning"
DatabaseNodeSetReady ClusterState = "Ready"
DatabaseNodeSetPaused ClusterState = "Paused"
Expand All @@ -60,6 +72,7 @@ const (
StoragePaused ClusterState = "Paused"

StorageNodeSetPending ClusterState = "Pending"
StorageNodeSetPreparing ClusterState = "Preparing"
StorageNodeSetProvisioning ClusterState = "Provisioning"
StorageNodeSetReady ClusterState = "Ready"
StorageNodeSetPaused ClusterState = "Paused"
Expand Down
146 changes: 146 additions & 0 deletions internal/controllers/database/controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package database_test

import (
"context"
"errors"
"path/filepath"
"strings"
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"

"github.com/ydb-platform/ydb-kubernetes-operator/api/v1alpha1"
testobjects "github.com/ydb-platform/ydb-kubernetes-operator/e2e/tests/test-objects"
. "github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/constants"
"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/database"
"github.com/ydb-platform/ydb-kubernetes-operator/internal/controllers/storage"
"github.com/ydb-platform/ydb-kubernetes-operator/internal/test"
)

var (
k8sClient client.Client
ctx context.Context
)

func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)

test.SetupK8STestManager(&ctx, &k8sClient, func(mgr *manager.Manager) []test.Reconciler {
return []test.Reconciler{
&storage.Reconciler{
Client: k8sClient,
Scheme: (*mgr).GetScheme(),
},
&database.Reconciler{
Client: k8sClient,
Scheme: (*mgr).GetScheme(),
},
}
})

RunSpecs(t, "Database controller medium tests suite")
}

var _ = Describe("Database controller medium tests", func() {
var namespace corev1.Namespace
var storageSample v1alpha1.Storage

BeforeEach(func() {
namespace = corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: testobjects.YdbNamespace,
},
}
Expect(k8sClient.Create(ctx, &namespace)).Should(Succeed())
storageSample = *testobjects.DefaultStorage(filepath.Join("..", "..", "..", "e2e", "tests", "data", "storage-block-4-2-config.yaml"))
Expect(k8sClient.Create(ctx, &storageSample)).Should(Succeed())

By("checking that Storage created on local cluster...")
foundStorage := v1alpha1.Storage{}
Eventually(func() bool {
Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: storageSample.Name,
Namespace: testobjects.YdbNamespace,
}, &foundStorage))
return foundStorage.Status.State == StorageInitializing
}, test.Timeout, test.Interval).Should(BeTrue())

By("set condition Initialized to Storage...")
Eventually(func() error {
foundStorage := v1alpha1.Storage{}
Expect(k8sClient.Get(ctx, types.NamespacedName{
Name: storageSample.Name,
Namespace: testobjects.YdbNamespace,
}, &foundStorage))
meta.SetStatusCondition(&foundStorage.Status.Conditions, metav1.Condition{
Type: StorageInitializedCondition,
Status: metav1.ConditionTrue,
Reason: ReasonCompleted,
})
return k8sClient.Status().Update(ctx, &foundStorage)
}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())
})

AfterEach(func() {
Expect(k8sClient.Delete(ctx, &storageSample)).Should(Succeed())
Expect(k8sClient.Delete(ctx, &namespace)).Should(Succeed())
})

It("Checking field propagation to objects", func() {
By("Check that Shared Database was created...")
databaseSample := *testobjects.DefaultDatabase()
databaseSample.Spec.SharedResources = &v1alpha1.DatabaseResources{
StorageUnits: []v1alpha1.StorageUnit{
{
UnitKind: "ssd",
Count: 1,
},
},
}
Expect(k8sClient.Create(ctx, &databaseSample)).Should(Succeed())

By("Check that StatefulSet was created...")
databaseStatefulSet := appsv1.StatefulSet{}
foundStatefulSets := appsv1.StatefulSetList{}
Eventually(func() error {
err := k8sClient.List(ctx, &foundStatefulSets, client.InNamespace(
testobjects.YdbNamespace))
if err != nil {
return err
}
for idx, statefulSet := range foundStatefulSets.Items {
if statefulSet.Name == testobjects.DatabaseName {
databaseStatefulSet = foundStatefulSets.Items[idx]
return nil
}
}
return errors.New("failed to find StatefulSet")
}, test.Timeout, test.Interval).ShouldNot(HaveOccurred())

By("Check that args `--label` propagated to pods...", func() {
podContainerArgs := databaseStatefulSet.Spec.Template.Spec.Containers[0].Args
var labelArgKey string
var labelArgValue string
for idx, arg := range podContainerArgs {
if arg == "--label" {
labelArgKey = strings.Split(podContainerArgs[idx+1], "=")[0]
labelArgValue = strings.Split(podContainerArgs[idx+1], "=")[1]
if labelArgKey == v1alpha1.LabelDeploymentKey {
Expect(labelArgValue).Should(BeEquivalentTo(v1alpha1.LabelDeploymentValueKubernetes))
}
if labelArgKey == v1alpha1.LabelSharedDatabaseKey {
Expect(labelArgValue).Should(BeEquivalentTo(v1alpha1.LabelSharedDatabaseValueTrue))
}
}
}
})
})
})
97 changes: 29 additions & 68 deletions internal/controllers/database/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,62 +17,35 @@ import (
"github.com/ydb-platform/ydb-kubernetes-operator/internal/resources"
)

func (r *Reconciler) processSkipInitPipeline(
func (r *Reconciler) setInitPipelineStatus(
ctx context.Context,
database *resources.DatabaseBuilder,
) (bool, ctrl.Result, error) {
r.Log.Info("running step processSkipInitPipeline")
r.Log.Info("Database initialization disabled (with annotation), proceed with caution")

r.Recorder.Event(
database,
corev1.EventTypeWarning,
"SkippingInit",
"Skipping database creation due to skip annotation present, be careful!",
)

return r.setInitDatabaseCompleted(
ctx,
database,
"Database creation not performed because initialization is skipped",
)
}

func (r *Reconciler) setInitialStatus(
ctx context.Context,
database *resources.DatabaseBuilder,
) (bool, ctrl.Result, error) {
r.Log.Info("running step setInitialStatus")

if meta.IsStatusConditionTrue(database.Status.Conditions, OldDatabaseInitializedCondition) {
if database.Status.State == DatabasePreparing {
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
Type: DatabaseInitializedCondition,
Status: "True",
Reason: ReasonCompleted,
Message: "Database initialized successfully",
Status: metav1.ConditionFalse,
Reason: ReasonInProgress,
Message: "Database has not been initialized yet",
})
database.Status.State = DatabaseReady
return r.updateStatus(ctx, database)
database.Status.State = DatabaseInitializing
return r.updateStatus(ctx, database, StatusUpdateRequeueDelay)
}

// This block is special internal logic that skips all Database initialization.
if value, ok := database.Annotations[v1alpha1.AnnotationSkipInitialization]; ok && value == v1alpha1.AnnotationValueTrue {
if meta.FindStatusCondition(database.Status.Conditions, DatabaseInitializedCondition) == nil ||
meta.IsStatusConditionFalse(database.Status.Conditions, DatabaseInitializedCondition) {
return r.processSkipInitPipeline(ctx, database)
}
return Stop, ctrl.Result{RequeueAfter: DefaultRequeueDelay}, nil
r.Log.Info("Database initialization disabled (with annotation), proceed with caution")
r.Recorder.Event(
database,
corev1.EventTypeWarning,
"SkippingInit",
"Skipping initialization due to skip annotation present, be careful!",
)
return r.setInitDatabaseCompleted(ctx, database, "Database initialization not performed because initialization is skipped")
}

if database.Status.State == DatabasePending ||
meta.FindStatusCondition(database.Status.Conditions, DatabaseInitializedCondition) == nil {
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
Type: DatabaseInitializedCondition,
Status: "False",
Reason: ReasonInProgress,
Message: "Database has not been initialized yet",
})
database.Status.State = DatabasePreparing
return r.updateStatus(ctx, database)
if meta.IsStatusConditionTrue(database.Status.Conditions, OldDatabaseInitializedCondition) {
return r.setInitDatabaseCompleted(ctx, database, "Database initialized successfully")
}

return Continue, ctrl.Result{Requeue: false}, nil
Expand All @@ -85,26 +58,17 @@ func (r *Reconciler) setInitDatabaseCompleted(
) (bool, ctrl.Result, error) {
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
Type: DatabaseInitializedCondition,
Status: "True",
Status: metav1.ConditionTrue,
Reason: ReasonCompleted,
Message: message,
})
database.Status.State = DatabaseProvisioning

return r.updateStatus(ctx, database)
return r.updateStatus(ctx, database, StatusUpdateRequeueDelay)
}

func (r *Reconciler) initializeDatabase(
func (r *Reconciler) initializeTenant(
ctx context.Context,
database *resources.DatabaseBuilder,
) (bool, ctrl.Result, error) {
r.Log.Info("running step initializeDatabase")

if database.Status.State == DatabasePreparing {
database.Status.State = DatabaseInitializing
return r.updateStatus(ctx, database)
}

path := database.GetDatabasePath()
var storageUnits []v1alpha1.StorageUnit
var shared bool
Expand Down Expand Up @@ -150,16 +114,15 @@ func (r *Reconciler) initializeDatabase(
return Stop, ctrl.Result{RequeueAfter: SharedDatabaseAwaitRequeueDelay}, err
}

if sharedDatabaseCr.Status.State != "Ready" {
if !meta.IsStatusConditionTrue(sharedDatabaseCr.Status.Conditions, DatabaseProvisionedCondition) {
r.Recorder.Event(
database,
corev1.EventTypeWarning,
"Pending",
fmt.Sprintf(
"Referenced shared Database (%s, %s) in a bad state: %s != Ready",
"Referenced shared Database (%s, %s) is not Provisioned",
database.Spec.ServerlessResources.SharedDatabaseRef.Name,
database.Spec.ServerlessResources.SharedDatabaseRef.Namespace,
sharedDatabaseCr.Status.State,
),
)
return Stop, ctrl.Result{RequeueAfter: SharedDatabaseAwaitRequeueDelay}, err
Expand Down Expand Up @@ -213,7 +176,12 @@ func (r *Reconciler) initializeDatabase(
"InitializingFailed",
fmt.Sprintf("Error creating tenant %s: %s", tenant.Path, err),
)
return Stop, ctrl.Result{RequeueAfter: DatabaseInitializationRequeueDelay}, err
meta.SetStatusCondition(&database.Status.Conditions, metav1.Condition{
Type: DatabaseInitializedCondition,
Status: metav1.ConditionFalse,
Reason: ReasonInProgress,
})
return r.updateStatus(ctx, database, DatabaseInitializationRequeueDelay)
}
r.Recorder.Event(
database,
Expand All @@ -222,12 +190,5 @@ func (r *Reconciler) initializeDatabase(
fmt.Sprintf("Tenant %s created", tenant.Path),
)

r.Recorder.Event(
database,
corev1.EventTypeNormal,
"DatabaseReady",
"Database is initialized",
)

return r.setInitDatabaseCompleted(ctx, database, "Database initialized successfully")
}
Loading

0 comments on commit b8a7aec

Please sign in to comment.