feat: Creating Gardener Cluster CR as part of cluster provisioning

api/v1/runtime_types.go

@@ -17,6 +17,8 @@ limitations under the License.
 package v1
 
 import (
+	"fmt"
+
 	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,6 +35,21 @@ const (
 	AnnotationGardenerCloudDelConfirmation = "confirmation.gardener.cloud/deletion"
 )
 
+const (
+	LabelKymaInstanceID      = "kyma-project.io/instance-id"
+	LabelKymaRuntimeID       = "kyma-project.io/runtime-id"
+	LabelKymaShootName       = "kyma-project.io/shootName"
+	LabelKymaRegion          = "kyma-project.io/region"
+	LabelKymaName            = "operator.kyma-project.io/kyma-name"
+	LabelKymaBrokerPlanID    = "kyma-project.io/broker-plan-id"
+	LabelKymaBrokerPlanName  = "kyma-project.io/broker-plan-name"
+	LabelKymaGlobalAccountID = "kyma-project.io/global-account-id"
+	LabelKymaSubaccountID    = "kyma-project.io/subaccount-id"
+	LabelKymaManagedBy       = "operator.kyma-project.io/managed-by"
+	LabelKymaInternal        = "operator.kyma-project.io/internal"
+	LabelKymaPlatformRegion  = "kyma-project.io/platform-region"
+)
+
 const (
 	RuntimeStateReady       = "Ready"
 	RuntimeStateFailed      = "Failed"
@@ -59,7 +76,8 @@ const (
 	ConditionReasonShootCreationPending   = RuntimeConditionReason("Pending")
 	ConditionReasonShootCreationCompleted = RuntimeConditionReason("ShootCreationCompleted")
 
-	ConditionReasonConfigurationStarted   = RuntimeConditionReason("ConfigurationStarted")
+	ConditionReasonGardenerCRCreated      = RuntimeConditionReason("GardenerClusterCRCreated")
+	ConditionReasonGardenerCRReady        = RuntimeConditionReason("GardenerClusterCRReady")
 	ConditionReasonConfigurationCompleted = RuntimeConditionReason("ConfigurationCompleted")
 	ConditionReasonConfigurationErr       = RuntimeConditionReason("ConfigurationError")
 
@@ -68,6 +86,7 @@ const (
 	ConditionReasonConversionError    = RuntimeConditionReason("ConversionErr")
 	ConditionReasonCreationError      = RuntimeConditionReason("CreationErr")
 	ConditionReasonGardenerError      = RuntimeConditionReason("GardenerErr")
+	ConditionReasonKubernetesAPIErr   = RuntimeConditionReason("KubernetesErr")
 	ConditionReasonSerializationError = RuntimeConditionReason("SerializationErr")
 	ConditionReasonDeleted            = RuntimeConditionReason("Deleted")
 )
@@ -255,3 +274,23 @@ func (k *Runtime) IsConditionSetWithStatus(c RuntimeConditionType, r RuntimeCond
 	}
 	return false
 }
+
+func (k *Runtime) ValidateRequiredLabels() error {
+	var requiredLabelKeys = []string{
+		LabelKymaInstanceID,
+		LabelKymaRuntimeID,
+		LabelKymaRegion,
+		LabelKymaName,
+		LabelKymaBrokerPlanID,
+		LabelKymaBrokerPlanName,
+		LabelKymaGlobalAccountID,
+		LabelKymaSubaccountID,
+	}
+
+	for _, key := range requiredLabelKeys {
+		if k.Labels[key] == "" {
+			return fmt.Errorf("missing required label %s", key)
+		}
+	}
+	return nil
+}

internal/controller/runtime/fsm/runtime_fsm.go

@@ -20,7 +20,8 @@ import (
 )
 
 const (
-	gardenerRequeueDuration = 15 * time.Second
+	gardenerRequeueDuration     = 15 * time.Second
+	controlPlaneRequeueDuration = 10 * time.Second
 )
 
 type stateFn func(context.Context, *fsm, *systemState) (stateFn, *ctrl.Result, error)

internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig.go

@@ -0,0 +1,106 @@
+package fsm
+
+import (
+	"context"
+	"fmt"
+
+	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+func sFnCreateKubeconfig(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
+	m.log.Info("Create Gardener Cluster CR state")
+
+	runtimeID := s.instance.Labels[imv1.LabelKymaRuntimeID]
+
+	var cluster imv1.GardenerCluster
+	err := m.Get(ctx, types.NamespacedName{
+		Namespace: s.instance.Namespace,
+		Name:      runtimeID,
+	}, &cluster)
+
+	if err != nil {
+		if !k8serrors.IsNotFound(err) {
+			m.log.Error(err, "GardenerCluster CR read error", "name", runtimeID)
+			s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeKubeconfigReady, imv1.ConditionReasonKubernetesAPIErr, "False", err.Error())
+			return updateStatusAndStop()
+		}
+
+		m.log.Info("GardenerCluster CR not found, creating a new one", "Name", runtimeID)
+		err = m.Create(ctx, makeGardenerClusterForRuntime(s.instance, s.shoot))
+		if err != nil {
+			m.log.Error(err, "GardenerCluster CR create error", "name", runtimeID)
+			s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeKubeconfigReady, imv1.ConditionReasonKubernetesAPIErr, "False", err.Error())
+			return updateStatusAndStop()
+		}
+
+		m.log.Info("Gardener Cluster CR created, waiting for readiness", "Name", runtimeID)
+		s.instance.UpdateStatePending(imv1.ConditionTypeRuntimeKubeconfigReady, imv1.ConditionReasonGardenerCRCreated, "Unknown", "Gardener Cluster CR created, waiting for readiness")
+		return updateStatusAndRequeueAfter(controlPlaneRequeueDuration)
+	}
+
+	if cluster.Status.State != imv1.ReadyState {
+		m.log.Info("GardenerCluster CR is not ready yet, requeue", "Name", runtimeID, "State", cluster.Status.State)
+		return requeueAfter(controlPlaneRequeueDuration)
+	}
+
+	m.log.Info("GardenerCluster CR is ready", "Name", runtimeID)
+
+	return ensureStatusConditionIsSetAndContinue(&s.instance,
+		imv1.ConditionTypeRuntimeKubeconfigReady,
+		imv1.ConditionReasonGardenerCRReady,
+		"Gardener Cluster CR is ready.",
+		sFnProcessShoot)
+}
+
+func makeGardenerClusterForRuntime(runtime imv1.Runtime, shoot *gardener.Shoot) *imv1.GardenerCluster {
+	gardenCluster := &imv1.GardenerCluster{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "GardenerCluster",
+			APIVersion: imv1.GroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      runtime.Labels[imv1.LabelKymaRuntimeID],
+			Namespace: runtime.Namespace,
+			Annotations: map[string]string{
+				"skr-domain": *shoot.Spec.DNS.Domain,
+			},
+			Labels: map[string]string{
+				imv1.LabelKymaInstanceID:      runtime.Labels[imv1.LabelKymaInstanceID],
+				imv1.LabelKymaRuntimeID:       runtime.Labels[imv1.LabelKymaRuntimeID],
+				imv1.LabelKymaBrokerPlanID:    runtime.Labels[imv1.LabelKymaBrokerPlanID],
+				imv1.LabelKymaBrokerPlanName:  runtime.Labels[imv1.LabelKymaBrokerPlanName],
+				imv1.LabelKymaGlobalAccountID: runtime.Labels[imv1.LabelKymaGlobalAccountID],
+				imv1.LabelKymaSubaccountID:    runtime.Labels[imv1.LabelKymaSubaccountID], // BTW most likely this value will be missing
+				imv1.LabelKymaName:            runtime.Labels[imv1.LabelKymaName],
+
+				// values from Runtime CR fields
+				imv1.LabelKymaPlatformRegion: runtime.Spec.Shoot.PlatformRegion,
+				imv1.LabelKymaRegion:         runtime.Spec.Shoot.Region,
+				imv1.LabelKymaShootName:      shoot.Name,
+
+				// hardcoded values
+				imv1.LabelKymaManagedBy: "infrastructure-manager",
+				imv1.LabelKymaInternal:  "true",
+			},
+		},
+		Spec: imv1.GardenerClusterSpec{
+			Shoot: imv1.Shoot{
+				Name: shoot.Name,
+			},
+			Kubeconfig: imv1.Kubeconfig{
+				Secret: imv1.Secret{
+					Name:      fmt.Sprintf("kubeconfig-%s", runtime.Labels[imv1.LabelKymaRuntimeID]),
+					Namespace: runtime.Namespace,
+					Key:       "config",
+				},
+			},
+		},
+	}
+
+	return gardenCluster
+}

internal/controller/runtime/fsm/runtime_fsm_create_kubeconfig_test.go

@@ -0,0 +1,166 @@
+package fsm
+
+/*
+import (
+	"context"
+	"time"
+
+	gardener "github.com/gardener/gardener/pkg/apis/core/v1beta1"
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	. "github.com/onsi/ginkgo/v2" //nolint:revive
+	. "github.com/onsi/gomega"    //nolint:revive
+	"github.com/onsi/gomega/types"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	util "k8s.io/apimachinery/pkg/util/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("KIM sFnCreateKubeconfig", func() {
+	now := metav1.NewTime(time.Now())
+
+	testCtx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	// GIVEN
+
+	testScheme := runtime.NewScheme()
+	util.Must(imv1.AddToScheme(testScheme))
+
+	withTestSchemeAndObjects := func(objs ...client.Object) fakeFSMOpt {
+		return func(fsm *fsm) error {
+			return withFakedK8sClient(testScheme, objs...)(fsm)
+		}
+	}
+
+	testRtWithLables := imv1.Runtime{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-instance",
+			Namespace: "default",
+			Labels: map[string]string{
+				imv1.LabelKymaRuntimeID:          "059dbc39-fd2b-4186-b0e5-8a1bc8ede5b8",
+				imv1.LabelKymaInstanceID:         "test-instance",
+				imv1.LabelKymaBrokerPlanID:       "broker-plan-id",
+				imv1.LabelKymaGlobalAccountID:    "461f6292-8085-41c8-af0c-e185f39b5e18",
+				imv1.LabelKymaGlobalSubaccountID: "c5ad84ae-3d1b-4592-bee1-f022661f7b30",
+				imv1.LabelKymaRegion:             "region",
+				imv1.LabelKymaBrokerPlanName:     "aws",
+				imv1.LabelKymaName:               "caadafae-1234-1234-1234-123456789abc",
+			},
+		},
+	}
+
+	testRtWithFinalizerNoProvisioningCondition := imv1.Runtime{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-instance",
+			Namespace:  "default",
+			Finalizers: []string{"test-me-plz"},
+		},
+	}
+
+	testRtWithFinalizerAndProvisioningCondition := imv1.Runtime{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-instance",
+			Namespace:  "default",
+			Finalizers: []string{"test-me-plz"},
+		},
+	}
+
+	provisioningCondition := metav1.Condition{
+		Type:               string(imv1.ConditionTypeRuntimeProvisioned),
+		Status:             metav1.ConditionUnknown,
+		LastTransitionTime: now,
+		Reason:             "Test reason",
+		Message:            "Test message",
+	}
+	meta.SetStatusCondition(&testRtWithFinalizerAndProvisioningCondition.Status.Conditions, provisioningCondition)
+
+	testShoot := gardener.Shoot{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-instance",
+			Namespace: "default",
+		},
+	}
+
+	testFunction := buildTestFunction(sFnCreateKubeconfig)
+
+	// WHEN/THAN
+
+	DescribeTable(
+		"transition graph validation",
+		testFunction,
+		Entry(
+			"should return nothing when CR is being deleted without finalizer and shoot is missing",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer),
+			&systemState{instance: testRtWithDeletionTimestamp},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: BeNil(),
+			},
+		),
+		Entry(
+			"should return sFnUpdateStatus when CR is being deleted with finalizer and shoot is missing - Remove finalizer",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer, withTestSchemeAndObjects(&testRtWithLables)),
+			&systemState{instance: testRtWithDeletionTimestampAndFinalizer},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: haveName("sFnProcessShoot"),
+			},
+		),
+		Entry(
+			"should return sFnDeleteShoot and no error when CR is being deleted with finalizer and shoot exists",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer),
+			&systemState{instance: testRtWithDeletionTimestampAndFinalizer, shoot: &testShoot},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: haveName("sFnDeleteShoot"),
+			},
+		),
+		Entry(
+			"should return sFnUpdateStatus and no error when CR has been created without finalizer - Add finalizer",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer, withTestSchemeAndObjects(&testRt)),
+			&systemState{instance: testRt},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: BeNil(),
+				StateMatch:       []types.GomegaMatcher{haveFinalizer("test-me-plz")},
+			},
+		),
+		Entry(
+			"should return sFnUpdateStatus and no error when there is no Provisioning Condition - Add condition",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer),
+			&systemState{instance: testRtWithFinalizerNoProvisioningCondition},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: haveName("sFnUpdateStatus"),
+			},
+		),
+		Entry(
+			"should return sFnCreateStatus and no error when exists Provisioning Condition and shoot is missing",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer),
+			&systemState{instance: testRtWithFinalizerAndProvisioningCondition},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: haveName("sFnCreateShoot"),
+			},
+		),
+		Entry(
+			"should return sFnSelectShootProcessing and no error when exists Provisioning Condition and shoot exists",
+			testCtx,
+			must(newFakeFSM, withTestFinalizer),
+			&systemState{instance: testRtWithFinalizerAndProvisioningCondition, shoot: &testShoot},
+			testOpts{
+				MatchExpectedErr: BeNil(),
+				MatchNextFnState: haveName("sFnSelectShootProcessing"),
+			},
+		),
+	)
+})
+*/

internal/controller/runtime/fsm/runtime_fsm_create_shoot.go

@@ -8,7 +8,7 @@ import (
 )
 
 func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl.Result, error) {
-	m.log.Info("Create shoot")
+	m.log.Info("Create shoot state")
 	newShoot, err := convertShoot(&s.instance, m.ConverterConfig)
 	if err != nil {
 		m.log.Error(err, "Failed to convert Runtime instance to shoot object")
@@ -37,6 +37,7 @@ func sFnCreateShoot(ctx context.Context, m *fsm, s *systemState) (stateFn, *ctrl
 		"Shoot is pending",
 	)
 
+	// it will be executed only once because created shoot is executed only once
 	shouldPersistShoot := m.PVCPath != ""
 	if shouldPersistShoot {
 		s.shoot = newShoot.DeepCopy()

internal/controller/runtime/fsm/runtime_fsm_patch_shoot.go

@@ -57,8 +57,12 @@ func sFnPatchExistingShoot(ctx context.Context, m *fsm, s *systemState) (stateFn
 }
 
 func convertShoot(instance *imv1.Runtime, cfg shoot.ConverterConfig) (gardener.Shoot, error) {
+	if err := instance.ValidateRequiredLabels(); err != nil {
+		return gardener.Shoot{}, err
+	}
+
 	converter := gardener_shoot.NewConverter(cfg)
-	shoot, err := converter.ToShoot(*instance) // returned error is always nil BTW
+	shoot, err := converter.ToShoot(*instance)
 
 	if err == nil {
 		setObjectFields(&shoot)