Deprovisioning runtime resource

cmd/broker/deprovisioning.go

@@ -61,6 +61,12 @@ func NewDeprovisioningProcessingQueue(ctx context.Context, workersAmount int, de
 		{
 			step: deprovisioning.NewCheckRuntimeRemovalStep(db.Operations(), db.Instances(), provisionerClient, cfg.Provisioner.DeprovisioningTimeout),
 		},
+		{
+			step: deprovisioning.NewDeleteRuntimeResourceStep(db.Operations(), cli),
+		},
+		{
+			step: deprovisioning.NewCheckRuntimeResourceDeletionStep(db.Operations(), cli),
+		},
 		{
 			step: deprovisioning.NewReleaseSubscriptionStep(db.Operations(), db.Instances(), accountProvider),
 		},

cmd/broker/deprovisioning_suite_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 	"time"
 
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+
 	"github.com/kyma-project/kyma-environment-broker/internal/kubeconfig"
 
 	kebConfig "github.com/kyma-project/kyma-environment-broker/internal/config"
@@ -111,6 +113,8 @@ func NewDeprovisioningSuite(t *testing.T) *DeprovisioningSuite {
 	assert.NoError(t, err)
 	err = corev1.AddToScheme(scheme)
 	assert.NoError(t, err)
+	err = imv1.AddToScheme(scheme)
+	require.NoError(t, err)
 	fakeK8sSKRClient := fake.NewClientBuilder().WithScheme(scheme).Build()
 
 	sch := internal.NewSchemeForTests(t)

internal/process/deprovisioning/check_runtime_resource_deletion_step.go

@@ -0,0 +1,81 @@
+package deprovisioning
+
+import (
+	"context"
+	"time"
+
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	"github.com/kyma-project/kyma-environment-broker/internal"
+	"github.com/kyma-project/kyma-environment-broker/internal/process"
+	"github.com/kyma-project/kyma-environment-broker/internal/storage"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type CheckRuntimeResourceDeletionStep struct {
+	operationManager *process.OperationManager
+	kcpClient        client.Client
+}
+
+func NewCheckRuntimeResourceDeletionStep(operations storage.Operations, kcpClient client.Client) *CheckRuntimeResourceDeletionStep {
+	return &CheckRuntimeResourceDeletionStep{
+		operationManager: process.NewOperationManager(operations),
+		kcpClient:        kcpClient,
+	}
+}
+
+func (step *CheckRuntimeResourceDeletionStep) Name() string {
+	return "Check_RuntimeResource_Deletion"
+}
+
+func (step *CheckRuntimeResourceDeletionStep) Run(operation internal.Operation, logger logrus.FieldLogger) (internal.Operation, time.Duration, error) {
+	namespace := operation.KymaResourceNamespace
+	if namespace == "" {
+		logger.Warnf("namespace for Kyma resource not specified, setting 'kcp-system'")
+		namespace = "kcp-system"
+	}
+	resourceName := operation.RuntimeResourceName
+	if resourceName == "" {
+		logger.Infof("Runtime resource name is empty, using runtime-id")
+		resourceName = operation.RuntimeID
+	}
+	if resourceName == "" {
+		logger.Infof("Empty runtime ID, skipping")
+		return operation, 0, nil
+	}
+
+	runtime := &imv1.Runtime{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      resourceName,
+			Namespace: namespace,
+		},
+	}
+
+	err := step.kcpClient.Get(context.Background(), client.ObjectKey{
+		Namespace: namespace,
+		Name:      resourceName,
+	}, runtime)
+
+	if err == nil {
+		logger.Infof("Runtime resource still exists")
+		//TODO: extract the timeout as a configuration setting
+		return step.operationManager.RetryOperationWithoutFail(operation, step.Name(), "Runtime resource still exists", 20*time.Second, 15*time.Minute, logger, nil)
+	}
+
+	if !errors.IsNotFound(err) {
+		if meta.IsNoMatchError(err) {
+			logger.Info("No CRD installed, skipping")
+			return operation, 0, nil
+		}
+
+		logger.Warnf("unable to check Runtime resource existence: %s", err)
+		return step.operationManager.RetryOperationWithoutFail(operation, step.Name(), "unable to check Runtime resource existence", backoffForK8SOperation, timeoutForK8sOperation, logger, err)
+	}
+
+	return step.operationManager.UpdateOperation(operation, func(op *internal.Operation) {
+		op.RuntimeResourceName = ""
+	}, logger)
+}

internal/process/deprovisioning/check_runtime_resource_deletion_step_test.go

@@ -0,0 +1,55 @@
+package deprovisioning
+
+import (
+	"testing"
+
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+	"github.com/kyma-project/kyma-environment-broker/internal/fixture"
+	"github.com/kyma-project/kyma-environment-broker/internal/logger"
+	"github.com/kyma-project/kyma-environment-broker/internal/storage"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+)
+
+func TestCheckRuntimeResourceDeletionStep_ResourceNotExists(t *testing.T) {
+	// given
+	err := imv1.AddToScheme(scheme.Scheme)
+	assert.NoError(t, err)
+	op := fixture.FixDeprovisioningOperationAsOperation(fixOperationID, fixInstanceID)
+	op.RuntimeResourceName = "runtime-name"
+	op.KymaResourceNamespace = "kyma-ns"
+	memoryStorage := storage.NewMemoryStorage()
+	assert.NoError(t, memoryStorage.Operations().InsertOperation(op))
+	kcpClient := fake.NewClientBuilder().Build()
+	log := logger.NewLogDummy()
+
+	// when
+	step := NewCheckRuntimeResourceDeletionStep(memoryStorage.Operations(), kcpClient)
+	_, backoff, err := step.Run(op, log)
+
+	// then
+	assert.NoError(t, err)
+	assert.Zero(t, backoff)
+}
+
+func TestCheckRuntimeResourceDeletionStep_Run(t *testing.T) {
+	// given
+	err := imv1.AddToScheme(scheme.Scheme)
+	assert.NoError(t, err)
+	op := fixture.FixDeprovisioningOperationAsOperation(fixOperationID, fixInstanceID)
+	op.RuntimeResourceName = "runtime-name"
+	op.KymaResourceNamespace = "kyma-ns"
+	memoryStorage := storage.NewMemoryStorage()
+	assert.NoError(t, memoryStorage.Operations().InsertOperation(op))
+	kcpClient := fake.NewClientBuilder().WithRuntimeObjects(fixRuntimeResource("kyma-ns", "runtime-name")).Build()
+	log := logger.NewLogDummy()
+
+	// when
+	step := NewCheckRuntimeResourceDeletionStep(memoryStorage.Operations(), kcpClient)
+	_, backoff, err := step.Run(op, log)
+
+	// then
+	assert.NoError(t, err)
+	assert.NotZero(t, backoff)
+}

internal/process/deprovisioning/delete_runtime_resource_step.go

@@ -0,0 +1,79 @@
+package deprovisioning
+
+import (
+	"context"
+	"time"
+
+	"github.com/kyma-project/kyma-environment-broker/internal"
+	"github.com/kyma-project/kyma-environment-broker/internal/process"
+	"github.com/kyma-project/kyma-environment-broker/internal/storage"
+	"github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+)
+
+const (
+	timeoutForRuntimeDeletion = 10 * time.Minute
+)
+
+type DeleteRuntimeResourceStep struct {
+	operationManager *process.OperationManager
+	kcpClient        client.Client
+}
+
+func NewDeleteRuntimeResourceStep(operations storage.Operations, kcpClient client.Client) *DeleteRuntimeResourceStep {
+	return &DeleteRuntimeResourceStep{
+		operationManager: process.NewOperationManager(operations),
+		kcpClient:        kcpClient,
+	}
+}
+
+func (step *DeleteRuntimeResourceStep) Name() string {
+	return "Delete_Runtime_Resource"
+}
+
+func (step *DeleteRuntimeResourceStep) Run(operation internal.Operation, logger logrus.FieldLogger) (internal.Operation, time.Duration, error) {
+	resourceName := operation.RuntimeResourceName
+	resourceNamespace := operation.KymaResourceNamespace
+
+	if resourceName == "" {
+		logger.Infof("Runtime resource name is empty, skipping")
+		return operation, 0, nil
+	}
+	if resourceNamespace == "" {
+		logger.Warnf("Namespace for Runtime resource not specified")
+		return operation, 0, nil
+	}
+
+	runtime := &imv1.Runtime{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      resourceName,
+			Namespace: resourceNamespace,
+		},
+	}
+
+	err := step.kcpClient.Delete(context.Background(), runtime)
+
+	// check the error
+	if err != nil {
+		if meta.IsNoMatchError(err) {
+			logger.Info("No CRD installed, skipping")
+			return operation, 0, nil
+		}
+
+		// if the resource is not found, log it and return (it is not a problem)
+		if errors.IsNotFound(err) {
+			logger.Info("Runtime resource deleted")
+			return operation, 0, nil
+		} else {
+			logger.Warnf("unable to delete the Runtime resource: %s", err)
+			return step.operationManager.RetryOperationWithoutFail(operation, step.Name(), "unable to delete the Runtime resource", backoffForK8SOperation, timeoutForK8sOperation, logger, err)
+		}
+	}
+
+	return operation, 0, nil
+}

internal/process/deprovisioning/delete_runtime_resource_step_test.go

@@ -0,0 +1,77 @@
+package deprovisioning
+
+import (
+	"context"
+	"testing"
+
+	"github.com/kyma-project/kyma-environment-broker/internal/fixture"
+	"github.com/kyma-project/kyma-environment-broker/internal/logger"
+	"github.com/kyma-project/kyma-environment-broker/internal/storage"
+	"github.com/stretchr/testify/assert"
+	"k8s.io/apimachinery/pkg/api/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+)
+
+func TestDeleteRuntimeResourceStep_RuntimeResourceDoesNotExists(t *testing.T) {
+	// given
+	err := imv1.AddToScheme(scheme.Scheme)
+	assert.NoError(t, err)
+	kcpClient := fake.NewClientBuilder().Build()
+	op := fixture.FixDeprovisioningOperationAsOperation(fixOperationID, fixInstanceID)
+	op.RuntimeResourceName = "runtime-name"
+	op.KymaResourceNamespace = "kyma-ns"
+	memoryStorage := storage.NewMemoryStorage()
+	log := logger.NewLogDummy()
+
+	// when
+	step := NewDeleteRuntimeResourceStep(memoryStorage.Operations(), kcpClient)
+	_, backoff, err := step.Run(op, log)
+
+	// then
+
+	assert.NoError(t, err)
+	assert.Zero(t, backoff)
+	assertRuntimeDoesNotExists(t, kcpClient, "kyma-ns", "runtime-name")
+}
+
+func TestDeleteRuntimeResourceStep_RuntimeResourceExists(t *testing.T) {
+	// given
+	err := imv1.AddToScheme(scheme.Scheme)
+	assert.NoError(t, err)
+	op := fixture.FixDeprovisioningOperationAsOperation(fixOperationID, fixInstanceID)
+	op.RuntimeResourceName = "runtime-name"
+	op.KymaResourceNamespace = "kyma-ns"
+	memoryStorage := storage.NewMemoryStorage()
+	kcpClient := fake.NewClientBuilder().WithRuntimeObjects(fixRuntimeResource("kyma-ns", "runtime-name")).Build()
+	log := logger.NewLogDummy()
+
+	// when
+	step := NewDeleteRuntimeResourceStep(memoryStorage.Operations(), kcpClient)
+	_, backoff, err := step.Run(op, log)
+
+	// then
+	assert.NoError(t, err)
+	assert.Zero(t, backoff)
+	assertRuntimeDoesNotExists(t, kcpClient, "kyma-ns", "runtime-name")
+}
+
+func assertRuntimeDoesNotExists(t *testing.T, kcpClient client.WithWatch, namespace string, name string) {
+	err := kcpClient.Get(context.Background(), client.ObjectKey{Name: name, Namespace: namespace}, &imv1.Runtime{})
+	assert.Error(t, err)
+	assert.True(t, errors.IsNotFound(err))
+}
+
+func fixRuntimeResource(namespace string, name string) runtime.Object {
+	return &imv1.Runtime{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+	}
+}

internal/test_utils.go

@@ -12,6 +12,8 @@ import (
 	"sync"
 	"testing"
 
+	imv1 "github.com/kyma-project/infrastructure-manager/api/v1"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
@@ -36,6 +38,8 @@ func NewSchemeForTests(t *testing.T) *k8sruntime.Scheme {
 	assert.NoError(t, err)
 	err = apiextensionsv1.AddToScheme(sch)
 	assert.NoError(t, err)
+	err = imv1.AddToScheme(sch)
+	assert.NoError(t, err)
 	return sch
 }