Reduce code and fix webhook cache latency (#361)

* add

* add

* add

* fix test

* fix comments

* fix test

* fix helm template

* fix helm template generate

---------

Co-authored-by: Jiawei Du <59427055+msftcoderdjw@users.noreply.github.com>

k8s/apis/containers/v1/generic_container_types.go

@@ -0,0 +1,10 @@
+package v1
+
+// +kubebuilder:object:generate=true
+type ContainerStatus struct {
+	Properties map[string]string `json:"properties"`
+}
+
+// +kubebuilder:object:generate=true
+type ContainerSpec struct {
+}

k8s/apis/containers/v1/generic_webhook.go

@@ -0,0 +1,97 @@
+package v1
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+
+	"github.com/eclipse-symphony/symphony/k8s/apis/metrics/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+type GetSubResourceNums func() (n int, err error)
+
+var commoncontainerlog = logf.Log.WithName("commoncontainer-resource")
+var cacheClient client.Client
+var readerClient client.Reader
+var commoncontainermetrics *metrics.Metrics
+
+func InitCommonContainerWebHook(mgr ctrl.Manager) error {
+	if commoncontainermetrics == nil {
+		initmetrics, err := metrics.New()
+		if err != nil {
+			return err
+		}
+		commoncontainermetrics = initmetrics
+	}
+	cacheClient = mgr.GetClient()
+	readerClient = mgr.GetAPIReader()
+	return nil
+}
+
+func SetupWebhookWithManager(mgr ctrl.Manager, resource client.Object) error {
+	mgr.GetFieldIndexer().IndexField(context.Background(), resource, ".metadata.name", func(rawObj client.Object) []string {
+		return []string{rawObj.GetName()}
+	})
+
+	return ctrl.NewWebhookManagedBy(mgr).
+		For(resource).
+		Complete()
+}
+
+// Default implements webhook.Defaulter so a webhook will be registered for the type
+func DefaultImpl(r client.Object) {
+	commoncontainerlog.Info("default", "name", r.GetName(), "kind", r.GetObjectKind())
+}
+
+func ValidateCreateImpl(r client.Object) (admission.Warnings, error) {
+	commoncontainerlog.Info("validate create", "name", r.GetName(), "kind", r.GetObjectKind())
+	return nil, nil
+}
+func ValidateUpdateImpl(r client.Object, old runtime.Object) (admission.Warnings, error) {
+	commoncontainerlog.Info("validate update", "name", r.GetName(), "kind", r.GetObjectKind())
+	return nil, nil
+}
+
+func ValidateDeleteImpl(r client.Object, getSubResourceNums GetSubResourceNums) (admission.Warnings, error) {
+
+	commoncontainerlog.Info("validate delete", "name", r.GetName(), "kind", r.GetObjectKind())
+
+	validateDeleteTime := time.Now()
+	validationError := validateDeleteContainerImpl(r, getSubResourceNums)
+	if validationError != nil {
+		commoncontainermetrics.ControllerValidationLatency(
+			validateDeleteTime,
+			metrics.CreateOperationType,
+			metrics.InvalidResource,
+			metrics.ContainerResourceType)
+	} else {
+		commoncontainermetrics.ControllerValidationLatency(
+			validateDeleteTime,
+			metrics.CreateOperationType,
+			metrics.ValidResource,
+			metrics.ContainerResourceType)
+	}
+
+	return nil, validationError
+}
+
+func validateDeleteContainerImpl(r client.Object, getSubResourceNums GetSubResourceNums) error {
+	itemsNum, err := getSubResourceNums()
+	if err != nil {
+		commoncontainerlog.Error(err, "could not list nested resources ", "name", r.GetName(), "kind", r.GetObjectKind())
+		return apierrors.NewBadRequest(fmt.Sprintf("%s could not list nested resources for %s.", r.GetObjectKind(), r.GetName()))
+	}
+	if itemsNum > 0 {
+		commoncontainerlog.Error(err, "nested resources are not empty", "name", r.GetName(), "kind", r.GetObjectKind())
+		return apierrors.NewBadRequest(fmt.Sprintf("%s nested resources with root resource '%s' are not empty", r.GetObjectKind(), r.GetName()))
+	}
+
+	return nil
+}

k8s/apis/federation/v1/catalog_types.go

@@ -7,9 +7,11 @@
 package v1
 
 import (
+	commoncontainers "gopls-workspace/apis/containers/v1"
 	k8smodel "gopls-workspace/apis/model/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
 type CatalogStatus struct {
@@ -35,6 +37,38 @@ type CatalogList struct {
 	Items           []Catalog `json:"items"`
 }
 
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// CatalogContainer is the Schema for the CatalogContainers API
+type CatalogContainer struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   commoncontainers.ContainerSpec   `json:"spec,omitempty"`
+	Status commoncontainers.ContainerStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+// CatalogContainerList contains a list of CatalogContainer
+type CatalogContainerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []CatalogContainer `json:"items"`
+}
+
+// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
+
+//+kubebuilder:webhook:path=/validate-federation-symphony-v1-catalogcontainer,mutating=false,failurePolicy=fail,sideEffects=None,groups=federation.symphony,resources=catalogcontainers,verbs=create;update;delete,versions=v1,name=vcatalogcontainer.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Validator = &CatalogContainer{}
+
+// TODO(user): EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+
+//+kubebuilder:webhook:path=/mutate-federation-symphony-v1-catalogcontainer,mutating=true,failurePolicy=fail,sideEffects=None,groups=federation.symphony,resources=catalogcontainers,verbs=create;update,versions=v1,name=mcatalogcontainer.kb.io,admissionReviewVersions=v1
+
+var _ webhook.Defaulter = &CatalogContainer{}
+
 func init() {
 	SchemeBuilder.Register(&Catalog{}, &CatalogList{})
+	SchemeBuilder.Register(&CatalogContainer{}, &CatalogContainerList{})
 }

k8s/apis/federation/v1/catalog_webhook.go

@@ -9,6 +9,7 @@ package v1
 import (
 	"context"
 	"encoding/json"
+	commoncontainer "gopls-workspace/apis/containers/v1"
 	"gopls-workspace/apis/metrics/v1"
 	"gopls-workspace/configutils"
 	"time"
@@ -28,19 +29,15 @@ import (
 
 // log is for logging in this package.
 var cataloglog = logf.Log.WithName("catalog-resource")
-var myCatalogClient client.Client
+var myCatalogReaderClient client.Reader
 var catalogWebhookValidationMetrics *metrics.Metrics
 
 func (r *Catalog) SetupWebhookWithManager(mgr ctrl.Manager) error {
-	myCatalogClient = mgr.GetClient()
+	myCatalogReaderClient = mgr.GetAPIReader()
 	mgr.GetFieldIndexer().IndexField(context.Background(), &Catalog{}, ".metadata.name", func(rawObj client.Object) []string {
 		catalog := rawObj.(*Catalog)
 		return []string{catalog.Name}
 	})
-	mgr.GetFieldIndexer().IndexField(context.Background(), &Catalog{}, ".spec.rootResource", func(rawObj client.Object) []string {
-		catalog := rawObj.(*Catalog)
-		return []string{catalog.Spec.RootResource}
-	})
 
 	// initialize the controller operation metrics
 	if catalogWebhookValidationMetrics == nil {
@@ -68,20 +65,25 @@ func (r *Catalog) Default() {
 
 	if r.Spec.RootResource != "" {
 		var catalogContainer CatalogContainer
-		err := myCatalogClient.Get(context.Background(), client.ObjectKey{Name: r.Spec.RootResource, Namespace: r.Namespace}, &catalogContainer)
+		err := myCatalogReaderClient.Get(context.Background(), client.ObjectKey{Name: r.Spec.RootResource, Namespace: r.Namespace}, &catalogContainer)
 		if err != nil {
 			cataloglog.Error(err, "failed to get catalog container", "name", r.Spec.RootResource)
 		} else {
 			ownerReference := metav1.OwnerReference{
-				APIVersion: catalogContainer.APIVersion,
-				Kind:       catalogContainer.Kind,
+				APIVersion: GroupVersion.String(), //catalogContainer.APIVersion
+				Kind:       "CatalogContainer",    //catalogContainer.Kind
 				Name:       catalogContainer.Name,
 				UID:        catalogContainer.UID,
 			}
 
 			if !configutils.CheckOwnerReferenceAlreadySet(r.OwnerReferences, ownerReference) {
 				r.OwnerReferences = append(r.OwnerReferences, ownerReference)
 			}
+
+			if r.Labels == nil {
+				r.Labels = make(map[string]string)
+			}
+			r.Labels["rootResource"] = r.Spec.RootResource
 		}
 	}
 }
@@ -170,7 +172,7 @@ func (r *Catalog) checkSchema() *field.Error {
 		if schemaName, ok := r.Spec.Metadata["schema"]; ok {
 			cataloglog.Info("Find schema name", "name", schemaName)
 			var catalogs CatalogList
-			err := myCatalogClient.List(context.Background(), &catalogs, client.InNamespace(r.ObjectMeta.Namespace), client.MatchingFields{".metadata.name": schemaName})
+			err := myCatalogReaderClient.List(context.Background(), &catalogs, client.InNamespace(r.ObjectMeta.Namespace), client.MatchingFields{"metadata.name": schemaName}, client.Limit(1))
 			if err != nil || len(catalogs.Items) == 0 {
 				cataloglog.Error(err, "Could not find the required schema.", "name", schemaName)
 				return field.Invalid(field.NewPath("spec").Child("Metadata"), schemaName, "could not find the required schema")
@@ -236,7 +238,7 @@ func (r *Catalog) validateNameOnCreate() *field.Error {
 
 func (r *Catalog) validateRootResource() *field.Error {
 	var catalogContainer CatalogContainer
-	err := myCatalogClient.Get(context.Background(), client.ObjectKey{Name: r.Spec.RootResource, Namespace: r.Namespace}, &catalogContainer)
+	err := myCatalogReaderClient.Get(context.Background(), client.ObjectKey{Name: r.Spec.RootResource, Namespace: r.Namespace}, &catalogContainer)
 	if err != nil {
 		return field.Invalid(field.NewPath("spec").Child("rootResource"), r.Spec.RootResource, "rootResource must be a valid catalog container")
 	}
@@ -247,3 +249,28 @@ func (r *Catalog) validateRootResource() *field.Error {
 
 	return nil
 }
+
+func (r *CatalogContainer) Default() {
+	commoncontainer.DefaultImpl(r)
+}
+
+func (r *CatalogContainer) ValidateCreate() (admission.Warnings, error) {
+	return commoncontainer.ValidateCreateImpl(r)
+}
+func (r *CatalogContainer) ValidateUpdate(old runtime.Object) (admission.Warnings, error) {
+	return commoncontainer.ValidateUpdateImpl(r, old)
+}
+
+func (r *CatalogContainer) ValidateDelete() (admission.Warnings, error) {
+	cataloglog.Info("validate delete catalog container", "name", r.Name)
+	getSubResourceNums := func() (int, error) {
+		var catalogList CatalogList
+		err := myCatalogReaderClient.List(context.Background(), &catalogList, client.InNamespace(r.Namespace), client.MatchingLabels{"rootResource": r.Name}, client.Limit(1))
+		if err != nil {
+			return 0, err
+		} else {
+			return len(catalogList.Items), nil
+		}
+	}
+	return commoncontainer.ValidateDeleteImpl(r, getSubResourceNums)
+}