feat(kuma-cp) validation synced resources on K8S (#919)

app/kumactl/cmd/install/testdata/install-control-plane.overrides.golden.yaml

@@ -5168,3 +5168,79 @@ webhooks:
         resources:
           - pods
     sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: kuma-validating-webhook-configuration
+webhooks:
+  - clientConfig:
+      caBundle: QWRtaXNzaW9uQ2VydA==
+      service:
+        name: kuma-ctrl-plane
+        namespace: kuma
+        path: /validate-kuma-io-v1alpha1
+    failurePolicy: Fail
+    name: validator.kuma-admission.kuma.io
+    rules:
+      - apiGroups:
+          - kuma.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - circuitbreakers
+          - faultinjections
+          - trafficlogs
+          - trafficpermissions
+          - trafficroutes
+          - traffictraces
+          - dataplanes
+          - healthchecks
+          - meshes
+          - proxytemplates
+  - clientConfig:
+      caBundle: QWRtaXNzaW9uQ2VydA==
+      service:
+        name: kuma-ctrl-plane
+        namespace: kuma
+        path: /validate-v1-service
+    failurePolicy: Fail
+    name: service.validator.kuma-admission.kuma.io
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - services
+    sideEffects: None
+  - clientConfig:
+      caBundle: QWRtaXNzaW9uQ2VydA==
+      service:
+        name: kuma-ctrl-plane
+        namespace: kuma
+        path: /validate-v1-secret
+    failurePolicy: Ignore
+    name: secret.validator.kuma-admission.kuma.io
+    namespaceSelector:
+      matchLabels:
+        kuma.io/system-namespace: "true"
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - secrets
+    sideEffects: None

app/kumactl/cmd/install/testdata/install-control-plane.remote.golden.yaml

@@ -5168,4 +5168,80 @@ webhooks:
         - CREATE
       resources:
         - pods
-  sideEffects: None
+  sideEffects: None
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: kuma-validating-webhook-configuration
+webhooks:
+  - name: validator.kuma-admission.kuma.io
+    failurePolicy: Fail
+    clientConfig:
+      caBundle: Q0VSVA==
+      service:
+        namespace: kuma-system
+        name: kuma-control-plane
+        path: /validate-kuma-io-v1alpha1
+    rules:
+      - apiGroups:
+          - kuma.io
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - circuitbreakers
+          - faultinjections
+          - trafficlogs
+          - trafficpermissions
+          - trafficroutes
+          - traffictraces
+          - dataplanes
+          - healthchecks
+          - meshes
+          - proxytemplates
+  - name: service.validator.kuma-admission.kuma.io
+    failurePolicy: Fail
+    clientConfig:
+      caBundle: Q0VSVA==
+      service:
+        namespace: kuma-system
+        name: kuma-control-plane
+        path: /validate-v1-service
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - services
+    sideEffects: None
+  - name: secret.validator.kuma-admission.kuma.io
+    namespaceSelector:
+      matchLabels:
+        kuma.io/system-namespace: "true"
+    failurePolicy: Ignore
+    clientConfig:
+      caBundle: Q0VSVA==
+      service:
+        namespace: kuma-system
+        name: kuma-control-plane
+        path: /validate-v1-secret
+    rules:
+      - apiGroups:
+          - ""
+        apiVersions:
+          - v1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - secrets
+    sideEffects: None

app/kumactl/data/install/k8s/control-plane/kuma-cp/app.yaml

@@ -182,8 +182,8 @@ spec:
 {{- if ne .KumaCpMode "remote" }}
         - containerPort: 5683
 {{- end }}
-{{- if ne .KumaCpMode "global" }}
         - containerPort: 5443
+{{- if ne .KumaCpMode "global" }}
         - containerPort: 5677
         - containerPort: 5678
         - containerPort: 5679
@@ -311,7 +311,6 @@ webhooks:
         - pods
   sideEffects: None
 ---
-{{- if ne .KumaCpMode "remote" }}
 apiVersion: admissionregistration.k8s.io/v1beta1
 kind: ValidatingWebhookConfiguration
 metadata:
@@ -387,4 +386,3 @@ webhooks:
         resources:
           - secrets
     sideEffects: None
-{{- end }}

pkg/core/resources/store/options.go

@@ -12,6 +12,7 @@ type CreateOptions struct {
 	Mesh         string
 	CreationTime time.Time
 	Owner        core_model.Resource
+	Synced       bool
 }
 
 type CreateOptionsFunc func(*CreateOptions)
@@ -47,8 +48,15 @@ func CreateWithOwner(owner core_model.Resource) CreateOptionsFunc {
 	}
 }
 
+func CreateSynced() CreateOptionsFunc {
+	return func(opts *CreateOptions) {
+		opts.Synced = true
+	}
+}
+
 type UpdateOptions struct {
 	ModificationTime time.Time
+	Synced           bool
 }
 
 func ModifiedAt(modificationTime time.Time) UpdateOptionsFunc {
@@ -57,6 +65,12 @@ func ModifiedAt(modificationTime time.Time) UpdateOptionsFunc {
 	}
 }
 
+func UpdateSynced() UpdateOptionsFunc {
+	return func(opts *UpdateOptions) {
+		opts.Synced = true
+	}
+}
+
 type UpdateOptionsFunc func(*UpdateOptions)
 
 func NewUpdateOptions(fs ...UpdateOptionsFunc) *UpdateOptions {

pkg/kds/global/components.go

@@ -141,3 +141,12 @@ func dedupIngresses(rs model.ResourceList) model.ResourceList {
 	}
 	return rv
 }
+
+func ConsumesType(typ model.ResourceType) bool {
+	for _, consumedTyp := range consumedTypes {
+		if consumedTyp == typ {
+			return true
+		}
+	}
+	return false
+}

pkg/kds/remote/components.go

@@ -93,3 +93,12 @@ func Callbacks(syncer sync_store.ResourceSyncer, k8sStore bool, localZone string
 		},
 	}
 }
+
+func ConsumesType(typ model.ResourceType) bool {
+	for _, consumedTyp := range consumedTypes {
+		if consumedTyp == typ {
+			return true
+		}
+	}
+	return false
+}

pkg/kds/store/sync.go

@@ -121,7 +121,7 @@ func (s *syncResourceStore) Sync(upstream model.ResourceList, fs ...SyncOptionFu
 		creationTime := r.GetMeta().GetCreationTime()
 		// some Stores try to cast ResourceMeta to own Store type that's why we have to set meta to nil
 		r.SetMeta(nil)
-		if err := s.resourceStore.Create(ctx, r, store.CreateBy(rk), store.CreatedAt(creationTime)); err != nil {
+		if err := s.resourceStore.Create(ctx, r, store.CreateBy(rk), store.CreatedAt(creationTime), store.CreateSynced()); err != nil {
 			return err
 		}
 	}
@@ -132,7 +132,7 @@ func (s *syncResourceStore) Sync(upstream model.ResourceList, fs ...SyncOptionFu
 		// some stores manage ModificationTime time on they own (Kubernetes), in order to be consistent
 		// we set ModificationTime when we add to downstream store. This time is almost the same with ModificationTime
 		// from upstream store, because we update downstream only when resource have changed in upstream
-		if err := s.resourceStore.Update(ctx, r, store.ModifiedAt(now)); err != nil {
+		if err := s.resourceStore.Update(ctx, r, store.ModifiedAt(now), store.UpdateSynced()); err != nil {
 			return err
 		}
 	}

pkg/plugins/common/k8s/names.go

@@ -14,6 +14,9 @@ const (
 	// The value is considered a part of user-facing Kuma API and should not be changed lightly.
 	// The value has a format of a Kubernetes label name.
 	k8sNameComponent = "k8s.kuma.io/name"
+
+	// k8sSynced identifies that resource was synced
+	K8sSynced = "k8s.kuma.io/synced"
 )
 
 func ResourceNameExtensions(namespace, name string) core_model.ResourceNameExtensions {

pkg/plugins/resources/k8s/store.go

@@ -62,6 +62,10 @@ func (s *KubernetesStore) Create(ctx context.Context, r core_model.Resource, fs
 		}
 	}
 
+	if opts.Synced {
+		markAsSynced(obj)
+	}
+
 	if err := s.Client.Create(ctx, obj); err != nil {
 		if kube_apierrs.IsAlreadyExists(err) {
 			return store.ErrorResourceAlreadyExists(r.GetType(), opts.Name, opts.Mesh)
@@ -75,11 +79,27 @@ func (s *KubernetesStore) Create(ctx context.Context, r core_model.Resource, fs
 	return nil
 }
 
+func markAsSynced(obj k8s_model.KubernetesObject) {
+	annotations := obj.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[common_k8s.K8sSynced] = "true"
+	obj.SetAnnotations(annotations)
+}
+
 func (s *KubernetesStore) Update(ctx context.Context, r core_model.Resource, fs ...store.UpdateOptionsFunc) error {
+	opts := store.NewUpdateOptions(fs...)
+
 	obj, err := s.Converter.ToKubernetesObject(r)
 	if err != nil {
 		return errors.Wrapf(err, "failed to convert core model of type %s into k8s counterpart", r.GetType())
 	}
+
+	if opts.Synced {
+		markAsSynced(obj)
+	}
+
 	if err := s.Client.Update(ctx, obj); err != nil {
 		if kube_apierrs.IsConflict(err) {
 			return store.ErrorResourceConflict(r.GetType(), r.GetMeta().GetName(), r.GetMeta().GetMesh())

pkg/plugins/runtime/k8s/plugin.go

@@ -53,10 +53,8 @@ func (p *plugin) Customize(rt core_runtime.Runtime) error {
 		return err
 	}
 
-	if rt.Config().Mode.Mode != mode.Remote {
-		if err := addValidators(mgr, rt); err != nil {
-			return err
-		}
+	if err := addValidators(mgr, rt); err != nil {
+		return err
 	}
 
 	addMutators(mgr, rt)
@@ -128,7 +126,7 @@ func generateDefaulterPath(gvk kube_schema.GroupVersionKind) string {
 func addValidators(mgr kube_ctrl.Manager, rt core_runtime.Runtime) error {
 	composite := k8s_webhooks.CompositeValidator{}
 
-	handler := k8s_webhooks.NewValidatingWebhook(k8s_resources.DefaultConverter(), core_registry.Global(), k8s_registry.Global())
+	handler := k8s_webhooks.NewValidatingWebhook(k8s_resources.DefaultConverter(), core_registry.Global(), k8s_registry.Global(), rt.Config().Mode.Mode)
 	composite.AddValidator(handler)
 
 	coreMeshValidator := managers_mesh.MeshValidator{CaManagers: rt.CaManagers()}