feat(kumacp) Delete related mesh resources

Makefile

@@ -365,7 +365,7 @@ test/kuma-injector: test ## Dev: Run 'kuma injector' tests only
 
 integration: ## Dev: Run integration tests
 	mkdir -p "$(shell dirname "$(COVERAGE_INTEGRATION_PROFILE)")"
-	tools/test/run-integration-tests.sh '$(GO_TEST) -race -covermode=atomic -tags=integration -count=1 -coverpkg=./... -coverprofile=$(COVERAGE_INTEGRATION_PROFILE) $(PKG_LIST)'
+	tools/test/run-integration-tests.sh '$(GO_TEST) $(GO_TEST_OPTS) -race -covermode=atomic -tags=integration -count=1 -coverpkg=./... -coverprofile=$(COVERAGE_INTEGRATION_PROFILE) $(PKG_LIST)'
 	go tool cover -html="$(COVERAGE_INTEGRATION_PROFILE)" -o "$(COVERAGE_INTEGRATION_REPORT_HTML)"
 
 build: build/kuma-cp build/kuma-dp build/kumactl build/kuma-injector build/kuma-tcp-echo ## Dev: Build all binaries

pkg/core/managers/apis/mesh/mesh_manager.go

@@ -75,6 +75,8 @@ func (m *meshManager) Create(ctx context.Context, resource core_model.Resource,
 }
 
 func (m *meshManager) Delete(ctx context.Context, resource core_model.Resource, fs ...core_store.DeleteOptionsFunc) error {
+	opts := core_store.NewDeleteOptions(fs...)
+
 	mesh, err := m.mesh(resource)
 	if err != nil {
 		return err
@@ -89,6 +91,10 @@ func (m *meshManager) Delete(ctx context.Context, resource core_model.Resource,
 	if err := m.builtinCaManager.Delete(ctx, name); err != nil {
 		return errors.Wrapf(err, "failed to delete Builtin CA for a given mesh")
 	}
+	// delete associated resources
+	if err := m.store.DeleteMany(ctx, core_store.DeleteManyByMesh(opts.Name)); err != nil {
+		return errors.Wrapf(err, "failed to delete associated resources")
+	}
 	return nil
 }
 

pkg/core/managers/apis/mesh/mesh_manager_suite_test.go

@@ -0,0 +1,13 @@
+package mesh
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func TestMeshManager(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Mesh Manager Suite")
+}

pkg/core/managers/apis/mesh/mesh_manager_test.go

@@ -0,0 +1,87 @@
+package mesh
+
+import (
+	"context"
+	"github.com/Kong/kuma/pkg/core/ca/builtin"
+	core_mesh "github.com/Kong/kuma/pkg/core/resources/apis/mesh"
+	"github.com/Kong/kuma/pkg/core/resources/manager"
+	"github.com/Kong/kuma/pkg/core/resources/model"
+	"github.com/Kong/kuma/pkg/core/resources/store"
+	"github.com/Kong/kuma/pkg/core/secrets/cipher"
+	secrets_manager "github.com/Kong/kuma/pkg/core/secrets/manager"
+	secrets_store "github.com/Kong/kuma/pkg/core/secrets/store"
+	"github.com/Kong/kuma/pkg/plugins/resources/memory"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Mesh Manager", func() {
+
+	const namespace = "default"
+
+	var resManager manager.ResourceManager
+	var resStore store.ResourceStore
+	var caManager builtin.BuiltinCaManager
+
+	BeforeEach(func() {
+		resStore = memory.NewStore()
+		caManager = builtin.NewBuiltinCaManager(secrets_manager.NewSecretManager(secrets_store.NewSecretStore(resStore), cipher.None()))
+		resManager = NewMeshManager(resStore, caManager)
+	})
+
+	It("Create() should also create a built-in CA", func() {
+		// given
+		meshName := "mesh-1"
+		resKey := model.ResourceKey{
+			Mesh:      meshName,
+			Namespace: namespace,
+			Name:      meshName,
+		}
+
+		// when
+		mesh := core_mesh.MeshResource{}
+		err := resManager.Create(context.Background(), &mesh, store.CreateBy(resKey))
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+
+		// and built-in CA is created
+		certs, err := caManager.GetRootCerts(context.Background(), meshName)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(certs).To(HaveLen(1))
+	})
+
+	It("Delete() should delete all associated resources", func() {
+		// given mesh
+		meshName := "mesh-1"
+
+		mesh := core_mesh.MeshResource{}
+		resKey := model.ResourceKey{
+			Mesh:      meshName,
+			Namespace: namespace,
+			Name:      meshName,
+		}
+		err := resManager.Create(context.Background(), &mesh, store.CreateBy(resKey))
+		Expect(err).ToNot(HaveOccurred())
+
+		// and resource associated with it
+		dp := core_mesh.DataplaneResource{}
+		err = resStore.Create(context.Background(), &dp, store.CreateByKey(namespace, "dp-1", meshName))
+		Expect(err).ToNot(HaveOccurred())
+
+		// when mesh is deleted
+		err = resManager.Delete(context.Background(), &mesh, store.DeleteBy(resKey))
+
+		// then
+		Expect(err).ToNot(HaveOccurred())
+
+		// and resource is deleted
+		err = resStore.Get(context.Background(), &core_mesh.DataplaneResource{}, store.GetByKey(namespace, "dp-1", meshName))
+		Expect(store.IsResourceNotFound(err)).To(BeTrue())
+
+		// and built-in mesh CA is deleted
+		_, err = caManager.GetRootCerts(context.Background(), meshName)
+		Expect(err).ToNot(BeNil())
+		Expect(err.Error()).To(Equal("failed to load CA key pair for Mesh \"mesh-1\": Resource not found: type=\"Secret\" namespace=\"default\" name=\"builtinca.mesh-1\" mesh=\"mesh-1\""))
+	})
+})

pkg/core/resources/store/options.go

@@ -73,6 +73,26 @@ func DeleteByKey(ns, name, mesh string) DeleteOptionsFunc {
 	}
 }
 
+type DeleteManyOptions struct {
+	Mesh string
+}
+
+type DeleteManyOptionsFunc func(*DeleteManyOptions)
+
+func NewDeleteManyOptions(fs ...DeleteManyOptionsFunc) *DeleteManyOptions {
+	opts := &DeleteManyOptions{}
+	for _, f := range fs {
+		f(opts)
+	}
+	return opts
+}
+
+func DeleteManyByMesh(mesh string) DeleteManyOptionsFunc {
+	return func(opts *DeleteManyOptions) {
+		opts.Mesh = mesh
+	}
+}
+
 type GetOptions struct {
 	Namespace string
 	Name      string

pkg/core/resources/store/store.go

@@ -13,6 +13,7 @@ type ResourceStore interface {
 	Create(context.Context, model.Resource, ...CreateOptionsFunc) error
 	Update(context.Context, model.Resource, ...UpdateOptionsFunc) error
 	Delete(context.Context, model.Resource, ...DeleteOptionsFunc) error
+	DeleteMany(context.Context, ...DeleteManyOptionsFunc) error
 	Get(context.Context, model.Resource, ...GetOptionsFunc) error
 	List(context.Context, model.ResourceList, ...ListOptionsFunc) error
 }
@@ -88,6 +89,9 @@ func (s *strictResourceStore) Delete(ctx context.Context, r model.Resource, fs .
 	}
 	return s.delegate.Delete(ctx, r, fs...)
 }
+func (s *strictResourceStore) DeleteMany(ctx context.Context, fs ...DeleteManyOptionsFunc) error {
+	return s.delegate.DeleteMany(ctx, fs...)
+}
 func (s *strictResourceStore) Get(ctx context.Context, r model.Resource, fs ...GetOptionsFunc) error {
 	if r == nil {
 		return fmt.Errorf("ResourceStore.Get() requires a non-nil resource")

pkg/core/resources/store/store_test_template.go

@@ -2,6 +2,8 @@ package store
 
 import (
 	"context"
+	mesh_proto "github.com/Kong/kuma/api/mesh/v1alpha1"
+	core_mesh "github.com/Kong/kuma/pkg/core/resources/apis/mesh"
 	sample_proto "github.com/Kong/kuma/pkg/test/apis/sample/v1alpha1"
 	sample_model "github.com/Kong/kuma/pkg/test/resources/apis/sample"
 	. "github.com/onsi/ginkgo"
@@ -176,6 +178,68 @@ func ExecuteStoreTests(
 		})
 	})
 
+	Describe("DeleteMany()", func() {
+		BeforeEach(func() {
+			trRes := sample_model.TrafficRouteResource{
+				Spec: sample_proto.TrafficRoute{
+					Path: "demo",
+				},
+			}
+			err := s.Create(context.Background(), &trRes, CreateByKey(namespace, "tr-1", "mesh-1"))
+			Expect(err).ToNot(HaveOccurred())
+
+			dpRes := core_mesh.DataplaneResource{
+				Spec: mesh_proto.Dataplane{},
+			}
+			err = s.Create(context.Background(), &dpRes, CreateByKey(namespace, "dp-1", "mesh-2"))
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		It("should delete all resources", func() {
+			// when
+			err := s.DeleteMany(context.Background())
+
+			// then
+			Expect(err).ToNot(HaveOccurred())
+
+			// when query for deleted resource
+			resource := sample_model.TrafficRouteResource{}
+			err = s.Get(context.Background(), &resource, GetByKey(namespace, "tr-1", "mesh-1"))
+
+			// then resource cannot be found
+			Expect(err).To(Equal(ErrorResourceNotFound(resource.GetType(), namespace, "tr-1", "mesh-1")))
+
+			// when query for deleted resource
+			dpResource := core_mesh.DataplaneResource{}
+			err = s.Get(context.Background(), &dpResource, GetByKey(namespace, "dp-1", "mesh-2"))
+
+			// then resource cannot be found
+			Expect(err).To(Equal(ErrorResourceNotFound(dpResource.GetType(), namespace, "dp-1", "mesh-2")))
+		})
+
+		It("should delete resources by mesh", func() {
+			// when
+			err := s.DeleteMany(context.Background(), DeleteManyByMesh("mesh-1"))
+
+			// then
+			Expect(err).ToNot(HaveOccurred())
+
+			// when query for deleted resource in given mesh
+			resource := sample_model.TrafficRouteResource{}
+			err = s.Get(context.Background(), &resource, GetByKey(namespace, "tr-1", "mesh-1"))
+
+			// then resource cannot be found
+			Expect(err).To(Equal(ErrorResourceNotFound(resource.GetType(), namespace, "tr-1", "mesh-1")))
+
+			// when query for resource in another mesh
+			dpResource := core_mesh.DataplaneResource{}
+			err = s.Get(context.Background(), &dpResource, GetByKey(namespace, "dp-1", "mesh-2"))
+
+			// then resource is not deleted
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+
 	Describe("Get()", func() {
 		It("should return an error if resource is not found", func() {
 			// given

pkg/plugins/resources/k8s/k8s_suite_test.go

@@ -30,7 +30,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	// +kubebuilder:scaffold:imports
-	sample_v1alpha1 "github.com/Kong/kuma/pkg/plugins/resources/k8s/native/test/api/sample/v1alpha1"
+	mesh_proto "github.com/Kong/kuma/pkg/plugins/resources/k8s/native/api/v1alpha1"
+	sample_proto "github.com/Kong/kuma/pkg/plugins/resources/k8s/native/test/api/sample/v1alpha1"
 )
 
 var k8sClient client.Client
@@ -50,14 +51,20 @@ var _ = BeforeSuite(func(done Done) {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths: []string{filepath.Join("native", "test", "config", "crd", "bases")},
+		CRDDirectoryPaths: []string{
+			filepath.Join("native", "test", "config", "crd", "bases"),
+			filepath.Join("native", "config", "crd", "bases"),
+		},
 	}
 
 	cfg, err := testEnv.Start()
 	Expect(err).ToNot(HaveOccurred())
 	Expect(cfg).ToNot(BeNil())
 
-	err = sample_v1alpha1.AddToScheme(k8sClientScheme)
+	err = sample_proto.AddToScheme(k8sClientScheme)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = mesh_proto.AddToScheme(k8sClientScheme)
 	Expect(err).NotTo(HaveOccurred())
 
 	// +kubebuilder:scaffold:scheme

pkg/plugins/resources/k8s/native/pkg/registry/interfaces.go

@@ -10,6 +10,8 @@ type ResourceType = proto.Message
 type TypeRegistry interface {
 	RegisterObjectType(ResourceType, model.KubernetesObject) error
 	RegisterListType(ResourceType, model.KubernetesList) error
+	RegisteredObjects() []model.KubernetesObject
+	RegisteredLists() []model.KubernetesList
 
 	NewObject(ResourceType) (model.KubernetesObject, error)
 	NewList(ResourceType) (model.KubernetesList, error)

pkg/plugins/resources/k8s/native/pkg/registry/registry.go

@@ -20,6 +20,22 @@ type typeRegistry struct {
 	objectListTypes map[string]model.KubernetesList
 }
 
+func (r *typeRegistry) RegisteredObjects() []model.KubernetesObject {
+	var objs []model.KubernetesObject
+	for _, obj := range r.objectTypes {
+		objs = append(objs, obj.DeepCopyObject().(model.KubernetesObject))
+	}
+	return objs
+}
+
+func (r *typeRegistry) RegisteredLists() []model.KubernetesList {
+	var lists []model.KubernetesList
+	for _, list := range r.objectListTypes {
+		lists = append(lists, list.DeepCopyObject().(model.KubernetesList))
+	}
+	return lists
+}
+
 func (r *typeRegistry) RegisterObjectType(typ ResourceType, obj model.KubernetesObject) error {
 	name := proto.MessageName(typ)
 	if previous, ok := r.objectTypes[name]; ok {

pkg/plugins/resources/k8s/store.go

@@ -3,7 +3,6 @@ package k8s
 import (
 	"context"
 	"fmt"
-
 	core_model "github.com/Kong/kuma/pkg/core/resources/model"
 	"github.com/Kong/kuma/pkg/core/resources/store"
 	k8s_model "github.com/Kong/kuma/pkg/plugins/resources/k8s/native/pkg/model"
@@ -18,14 +17,16 @@ import (
 var _ store.ResourceStore = &KubernetesStore{}
 
 type KubernetesStore struct {
-	Client    kube_client.Client
-	Converter Converter
+	Client       kube_client.Client
+	Converter    Converter
+	TypeRegistry k8s_registry.TypeRegistry
 }
 
 func NewStore(client kube_client.Client) (store.ResourceStore, error) {
 	return &KubernetesStore{
-		Client:    client,
-		Converter: DefaultConverter(),
+		Client:       client,
+		Converter:    DefaultConverter(),
+		TypeRegistry: k8s_registry.Global(),
 	}, nil
 }
 
@@ -93,6 +94,23 @@ func (s *KubernetesStore) Delete(ctx context.Context, r core_model.Resource, fs
 	}
 	return nil
 }
+func (s *KubernetesStore) DeleteMany(ctx context.Context, fs ...store.DeleteManyOptionsFunc) error {
+	opts := store.NewDeleteManyOptions(fs...)
+
+	for _, list := range s.TypeRegistry.RegisteredLists() {
+		if err := s.Client.List(ctx, list); err != nil {
+			return errors.Wrapf(err, "failed to list resources of type %s", list.GetObjectKind().GroupVersionKind().String())
+		}
+		for _, obj := range list.GetItems() {
+			if opts.Mesh == "" || opts.Mesh == obj.GetMesh() {
+				if err := s.Client.Delete(ctx, obj); err != nil {
+					return errors.Wrap(err, "failed to delete k8s resource")
+				}
+			}
+		}
+	}
+	return nil
+}
 func (s *KubernetesStore) Get(ctx context.Context, r core_model.Resource, fs ...store.GetOptionsFunc) error {
 	opts := store.NewGetOptions(fs...)
 	obj, err := s.Converter.ToKubernetesObject(r)