From ea011b11a8cec768424368f7fb76f9db9078cc1e Mon Sep 17 00:00:00 2001 From: jwcesign Date: Wed, 13 Dec 2023 15:16:32 +0800 Subject: [PATCH] feat: fix conflicts with pp Signed-off-by: jwcesign --- api/openapi-spec/swagger.json | 32 +- ...rking.karmada.io_multiclusterservices.yaml | 34 +- pkg/apis/networking/v1alpha1/service_types.go | 22 +- .../v1alpha1/well_known_constants.go | 8 + .../v1alpha1/zz_generated.deepcopy.go | 26 ++ .../work/v1alpha2/well_known_constants.go | 3 + .../endpointslice_collect_controller.go | 4 +- .../endpointslice_dispatch_controller.go | 18 +- .../multiclusterservice/mcs_controller.go | 403 +++++++++++------- pkg/detector/detector.go | 6 + pkg/events/events.go | 10 +- pkg/generated/openapi/zz_generated.openapi.go | 57 ++- pkg/scheduler/event_handler.go | 3 +- pkg/util/constants.go | 6 + pkg/util/helper/mcs.go | 30 +- pkg/util/names/names.go | 4 - pkg/webhook/multiclusterservice/validating.go | 12 +- .../multiclusterservice/validating_test.go | 48 ++- test/e2e/mcs_test.go | 47 +- test/helper/resource.go | 12 +- 20 files changed, 545 insertions(+), 240 deletions(-) diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index e2334ee9a3c4..3ea3db4d2083 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -20975,6 +20975,16 @@ } } }, + "com.github.karmada-io.karmada.pkg.apis.networking.v1alpha1.ClusterSelector": { + "description": "ClusterSelector specifies the cluster to be selected.", + "type": "object", + "properties": { + "name": { + "description": "Name is the name of the cluster to be selected.", + "type": "string" + } + } + }, "com.github.karmada-io.karmada.pkg.apis.networking.v1alpha1.ExposurePort": { "description": "ExposurePort describes which port will be exposed.", "type": "object", @@ -21160,6 +21170,14 @@ "types" ], "properties": { + "consumerClusters": { + "description": "ConsumerClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the service will be exposed to all clusters.", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.networking.v1alpha1.ClusterSelector" + } + }, "ports": { "description": "Ports is the list of ports that are exposed by this MultiClusterService. No specified port will be filtered out during the service exposure and discovery process. All ports in the referencing service will be exposed by default.", "type": "array", @@ -21168,13 +21186,21 @@ "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.networking.v1alpha1.ExposurePort" } }, + "providerClusters": { + "description": "ProviderClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ConsumerClusters.", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.networking.v1alpha1.ClusterSelector" + } + }, "range": { - "description": "Range specifies the ranges where the referencing service should be exposed. Only valid and optional in case of Types contains CrossCluster. If not set and Types contains CrossCluster, all clusters will be selected, that means the referencing service will be exposed across all registered clusters. Deprecated: in favor of ServiceProvisionClusters/ServiceConsumptionClusters.", + "description": "Range specifies the ranges where the referencing service should be exposed. Only valid and optional in case of Types contains CrossCluster. If not set and Types contains CrossCluster, all clusters will be selected, that means the referencing service will be exposed across all registered clusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.", "default": {}, "$ref": "#/definitions/com.github.karmada-io.karmada.pkg.apis.networking.v1alpha1.ExposureRange" }, "serviceConsumptionClusters": { - "description": "ServiceConsumptionClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the service will be exposed to all clusters.", + "description": "ServiceConsumptionClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the service will be exposed to all clusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.", "type": "array", "items": { "type": "string", @@ -21182,7 +21208,7 @@ } }, "serviceProvisionClusters": { - "description": "ServiceProvisionClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ServiceConsumptionClusters.", + "description": "ServiceProvisionClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ServiceConsumptionClusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.", "type": "array", "items": { "type": "string", diff --git a/charts/karmada/_crds/bases/networking/networking.karmada.io_multiclusterservices.yaml b/charts/karmada/_crds/bases/networking/networking.karmada.io_multiclusterservices.yaml index ef13571dfb51..058d7be58ad7 100644 --- a/charts/karmada/_crds/bases/networking/networking.karmada.io_multiclusterservices.yaml +++ b/charts/karmada/_crds/bases/networking/networking.karmada.io_multiclusterservices.yaml @@ -44,6 +44,18 @@ spec: spec: description: Spec is the desired state of the MultiClusterService. properties: + consumerClusters: + description: ConsumerClusters specifies the clusters where the service + will be exposed, for clients. If leave it empty, the service will + be exposed to all clusters. + items: + description: ClusterSelector specifies the cluster to be selected. + properties: + name: + description: Name is the name of the cluster to be selected. + type: string + type: object + type: array ports: description: Ports is the list of ports that are exposed by this MultiClusterService. No specified port will be filtered out during the service exposure @@ -65,12 +77,24 @@ spec: - port type: object type: array + providerClusters: + description: ProviderClusters specifies the clusters which will provision + the service backend. If leave it empty, we will collect the backend + endpoints from all clusters and sync them to the ConsumerClusters. + items: + description: ClusterSelector specifies the cluster to be selected. + properties: + name: + description: Name is the name of the cluster to be selected. + type: string + type: object + type: array range: description: 'Range specifies the ranges where the referencing service should be exposed. Only valid and optional in case of Types contains CrossCluster. If not set and Types contains CrossCluster, all clusters will be selected, that means the referencing service will be exposed - across all registered clusters. Deprecated: in favor of ServiceProvisionClusters/ServiceConsumptionClusters.' + across all registered clusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.' properties: clusterNames: description: ClusterNames is the list of clusters to be selected. @@ -79,16 +103,18 @@ spec: type: array type: object serviceConsumptionClusters: - description: ServiceConsumptionClusters specifies the clusters where + description: 'ServiceConsumptionClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the - service will be exposed to all clusters. + service will be exposed to all clusters. Deprecated: in favor of + ProviderClusters/ConsumerClusters.' items: type: string type: array serviceProvisionClusters: - description: ServiceProvisionClusters specifies the clusters which + description: 'ServiceProvisionClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ServiceConsumptionClusters. + Deprecated: in favor of ProviderClusters/ConsumerClusters.' items: type: string type: array diff --git a/pkg/apis/networking/v1alpha1/service_types.go b/pkg/apis/networking/v1alpha1/service_types.go index aac7272a8b26..f6079d876742 100644 --- a/pkg/apis/networking/v1alpha1/service_types.go +++ b/pkg/apis/networking/v1alpha1/service_types.go @@ -79,20 +79,40 @@ type MultiClusterServiceSpec struct { // If not set and Types contains CrossCluster, all clusters will // be selected, that means the referencing service will be exposed // across all registered clusters. - // Deprecated: in favor of ServiceProvisionClusters/ServiceConsumptionClusters. + // Deprecated: in favor of ProviderClusters/ConsumerClusters. // +optional Range ExposureRange `json:"range,omitempty"` // ServiceProvisionClusters specifies the clusters which will provision the service backend. // If leave it empty, we will collect the backend endpoints from all clusters and sync // them to the ServiceConsumptionClusters. + // Deprecated: in favor of ProviderClusters/ConsumerClusters. // +optional ServiceProvisionClusters []string `json:"serviceProvisionClusters,omitempty"` // ServiceConsumptionClusters specifies the clusters where the service will be exposed, for clients. // If leave it empty, the service will be exposed to all clusters. + // Deprecated: in favor of ProviderClusters/ConsumerClusters. // +optional ServiceConsumptionClusters []string `json:"serviceConsumptionClusters,omitempty"` + + // ProviderClusters specifies the clusters which will provision the service backend. + // If leave it empty, we will collect the backend endpoints from all clusters and sync + // them to the ConsumerClusters. + // +optional + ProviderClusters []ClusterSelector `json:"providerClusters,omitempty"` + + // ConsumerClusters specifies the clusters where the service will be exposed, for clients. + // If leave it empty, the service will be exposed to all clusters. + // +optional + ConsumerClusters []ClusterSelector `json:"consumerClusters,omitempty"` +} + +// ClusterSelector specifies the cluster to be selected. +type ClusterSelector struct { + // Name is the name of the cluster to be selected. + // +required + Name string `json:"name,omitempty"` } // ExposureType describes how to expose the service. diff --git a/pkg/apis/networking/v1alpha1/well_known_constants.go b/pkg/apis/networking/v1alpha1/well_known_constants.go index b037b7942676..8dff9cd910c0 100644 --- a/pkg/apis/networking/v1alpha1/well_known_constants.go +++ b/pkg/apis/networking/v1alpha1/well_known_constants.go @@ -23,4 +23,12 @@ const ( // The reason for generating a new unique identifier instead of simply using metadata.UUID is because: // In backup scenarios, when applying the backup resource manifest in a new cluster, the UUID may change. MultiClusterServicePermanentIDLabel = "multiclusterservice.karmada.io/permanent-id" + + // MultiClusterServiceNameAnnotation is the name of a MultiClusterService object. + // This annotation will be added to the resource template and ResoruceBinding + MultiClusterServiceNameAnnotation = "multiclusterservice.karmada.io/name" + + // MultiClusterServiceNamespaceAnnotation is the namespace of a MultiClusterService object. + // This annotation will be added to the resource template and ResoruceBinding + MultiClusterServiceNamespaceAnnotation = "multiclusterservice.karmada.io/namespace" ) diff --git a/pkg/apis/networking/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/networking/v1alpha1/zz_generated.deepcopy.go index f29caf91d6c5..da7493d70b1b 100644 --- a/pkg/apis/networking/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/networking/v1alpha1/zz_generated.deepcopy.go @@ -25,6 +25,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSelector) DeepCopyInto(out *ClusterSelector) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSelector. +func (in *ClusterSelector) DeepCopy() *ClusterSelector { + if in == nil { + return nil + } + out := new(ClusterSelector) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExposurePort) DeepCopyInto(out *ExposurePort) { *out = *in @@ -208,6 +224,16 @@ func (in *MultiClusterServiceSpec) DeepCopyInto(out *MultiClusterServiceSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ProviderClusters != nil { + in, out := &in.ProviderClusters, &out.ProviderClusters + *out = make([]ClusterSelector, len(*in)) + copy(*out, *in) + } + if in.ConsumerClusters != nil { + in, out := &in.ConsumerClusters, &out.ConsumerClusters + *out = make([]ClusterSelector, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/apis/work/v1alpha2/well_known_constants.go b/pkg/apis/work/v1alpha2/well_known_constants.go index 82c6c9c65667..9ba6d380cdde 100644 --- a/pkg/apis/work/v1alpha2/well_known_constants.go +++ b/pkg/apis/work/v1alpha2/well_known_constants.go @@ -82,6 +82,9 @@ const ( // WorkNameLabel is added to objects to specify associated Work's name. WorkNameLabel = "work.karmada.io/name" + + // BindingManagedByLabel is added to ResoruceBinding to represent what kind of resource manages this Binding. + BindingManagedByLabel = "binding.karmada.io/managed-by" ) // Define resource conflict resolution diff --git a/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go b/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go index ab4a283e5b28..d1ee6cc89c76 100644 --- a/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go +++ b/pkg/controllers/multiclusterservice/endpointslice_collect_controller.go @@ -376,7 +376,7 @@ func reportEndpointSlice(c client.Client, endpointSlice *unstructured.Unstructur workMeta := metav1.ObjectMeta{ // Karmada will synchronize this work to other cluster namespaces and add the cluster name to prevent conflicts. - Name: names.GenerateMCSWorkName(endpointSlice.GetKind(), endpointSlice.GetName(), endpointSlice.GetNamespace(), clusterName), + Name: names.GenerateWorkName(endpointSlice.GetKind(), endpointSlice.GetName(), endpointSlice.GetNamespace()), Namespace: executionSpace, Labels: map[string]string{ util.MultiClusterServiceNamespaceLabel: endpointSlice.GetNamespace(), @@ -400,7 +400,7 @@ func cleanupWorkWithEndpointSliceDelete(c client.Client, endpointSliceKey keys.F workNamespaceKey := types.NamespacedName{ Namespace: executionSpace, - Name: names.GenerateMCSWorkName(endpointSliceKey.Kind, endpointSliceKey.Name, endpointSliceKey.Namespace, endpointSliceKey.Cluster), + Name: names.GenerateWorkName(endpointSliceKey.Kind, endpointSliceKey.Name, endpointSliceKey.Namespace), } work := &workv1alpha1.Work{} if err := c.Get(context.TODO(), workNamespaceKey, work); err != nil { diff --git a/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go b/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go index 9fc7075c786f..ed4f7ed6338f 100644 --- a/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go +++ b/pkg/controllers/multiclusterservice/endpointslice_dispatch_controller.go @@ -27,7 +27,6 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" "k8s.io/klog/v2" @@ -192,7 +191,7 @@ func (c *EndpointsliceDispatchController) newClusterFunc() handler.MapFunc { var requests []reconcile.Request for _, mcs := range mcsList.Items { - clusterSet, err := helper.GetConsumptionClustres(c.Client, mcs.DeepCopy()) + clusterSet, _, err := helper.GetConsumerClustres(c.Client, mcs.DeepCopy()) if err != nil { klog.Errorf("Failed to get provision clusters, error: %v", err) continue @@ -284,7 +283,7 @@ func (c *EndpointsliceDispatchController) cleanOrphanDispatchedEndpointSlice(ctx continue } - consumptionClusters, err := helper.GetConsumptionClustres(c.Client, mcs) + consumptionClusters, _, err := helper.GetConsumerClustres(c.Client, mcs) if err != nil { klog.Errorf("Failed to get consumption clusters, error is: %v", err) return err @@ -316,15 +315,12 @@ func (c *EndpointsliceDispatchController) dispatchEndpointSlice(ctx context.Cont return err } - consumptionClusters := sets.New[string](mcs.Spec.ServiceConsumptionClusters...) - if len(consumptionClusters) == 0 { - consumptionClusters, err = util.GetClusterSet(c.Client) - if err != nil { - klog.Errorf("Failed to get cluster set, error is: %v", err) - return err - } + consumerClusters, _, err := helper.GetConsumerClustres(c.Client, mcs) + if err != nil { + klog.Errorf("Failed to get consumer clusters, error is: %v", err) + return err } - for clusterName := range consumptionClusters { + for clusterName := range consumerClusters { if clusterName == epsSourceCluster { continue } diff --git a/pkg/controllers/multiclusterservice/mcs_controller.go b/pkg/controllers/multiclusterservice/mcs_controller.go index 19c969dd9dde..bad4a9a50b5d 100644 --- a/pkg/controllers/multiclusterservice/mcs_controller.go +++ b/pkg/controllers/multiclusterservice/mcs_controller.go @@ -19,15 +19,15 @@ package multiclusterservice import ( "context" "fmt" + "strings" + "github.com/google/uuid" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -46,7 +46,9 @@ import ( clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1" networkingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1" + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" workv1alpha1 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha1" + workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2" "github.com/karmada-io/karmada/pkg/events" "github.com/karmada-io/karmada/pkg/sharedcli/ratelimiterflag" "github.com/karmada-io/karmada/pkg/util" @@ -64,10 +66,6 @@ type MCSController struct { RateLimiterOptions ratelimiterflag.Options } -var ( - serviceGVK = corev1.SchemeGroupVersion.WithKind("Service") -) - // Reconcile performs a full reconciliation for the object referred to by the Request. // The Controller will requeue the Request to be processed again if an error is non-nil or // Result.Requeue is true, otherwise upon completion it will remove the work from the queue. @@ -85,44 +83,61 @@ func (c *MCSController) Reconcile(ctx context.Context, req controllerruntime.Req } if !mcs.DeletionTimestamp.IsZero() { - return c.handleMCSDelete(ctx, mcs.DeepCopy()) + return c.handleMultiClusterServiceDelete(mcs.DeepCopy()) + } + + providerClusters, noneExistProviderClusters, err := helper.GetProviderClusters(c.Client, mcs) + if err != nil { + klog.Errorf("Failed to get provider clusters by MultiClusterService(%s/%s):%v", mcs.Namespace, mcs.Name, err) + return controllerruntime.Result{}, err + } + consumerClusters, noneExistConsumerClusters, err := helper.GetConsumerClustres(c.Client, mcs) + if err != nil { + klog.Errorf("Failed to get consumer clusters by MultiClusterService(%s/%s):%v", mcs.Namespace, mcs.Name, err) + return controllerruntime.Result{}, err + } + // Report event on + if len(noneExistProviderClusters) != 0 || len(noneExistConsumerClusters) != 0 { + msgProvider := strings.Join(noneExistProviderClusters.UnsortedList(), ",") + msgConsumer := strings.Join(noneExistConsumerClusters.UnsortedList(), ",") + c.EventRecorder.Eventf(mcs, corev1.EventTypeNormal, events.EventReasonConfigurationRedundant, + fmt.Sprintf("ProviderClusters(%s)/ConsumerClusters(%s) dont's exist, Karmada will ignore them", msgProvider, msgConsumer)) } - var err error defer func() { if err != nil { - _ = c.updateMCSStatus(mcs, metav1.ConditionFalse, "ServiceAppliedFailed", err.Error()) - c.EventRecorder.Eventf(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceWorkFailed, err.Error()) + _ = c.updateMultiClusterServiceStatus(mcs, metav1.ConditionFalse, "ServiceAppliedFailed", err.Error()) + c.EventRecorder.Eventf(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceFailed, err.Error()) return } - _ = c.updateMCSStatus(mcs, metav1.ConditionTrue, "ServiceAppliedSucceed", "Service is propagated to target clusters.") - c.EventRecorder.Eventf(mcs, corev1.EventTypeNormal, events.EventReasonSyncWorkSucceed, "Service is propagated to target clusters.") + _ = c.updateMultiClusterServiceStatus(mcs, metav1.ConditionTrue, "ServiceAppliedSucceed", "Service is propagated to target clusters.") + c.EventRecorder.Eventf(mcs, corev1.EventTypeNormal, events.EventReasonSyncServiceSucceed, "Service is propagated to target clusters.") }() - if err = c.handleMCSCreateOrUpdate(ctx, mcs.DeepCopy()); err != nil { + if err = c.handleMultiClusterServiceCreateOrUpdate(mcs.DeepCopy(), providerClusters, consumerClusters); err != nil { return controllerruntime.Result{}, err } return controllerruntime.Result{}, nil } -func (c *MCSController) handleMCSDelete(ctx context.Context, mcs *networkingv1alpha1.MultiClusterService) (controllerruntime.Result, error) { +func (c *MCSController) handleMultiClusterServiceDelete(mcs *networkingv1alpha1.MultiClusterService) (controllerruntime.Result, error) { klog.V(4).InfoS("Begin to handle MultiClusterService delete event", "namespace", mcs.Namespace, "name", mcs.Name) - if err := c.deleteServiceWork(mcs, sets.New[string]()); err != nil { - c.EventRecorder.Event(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceWorkFailed, + if err := c.retrieveService(mcs); err != nil { + c.EventRecorder.Event(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceFailed, fmt.Sprintf("failed to delete service work :%v", err)) return controllerruntime.Result{}, err } - if err := c.deleteMultiClusterServiceWork(mcs, true); err != nil { - c.EventRecorder.Event(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceWorkFailed, + if err := c.retrieveMultiClusterService(mcs, nil); err != nil { + c.EventRecorder.Event(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceFailed, fmt.Sprintf("failed to delete MultiClusterService work :%v", err)) return controllerruntime.Result{}, err } finalizersUpdated := controllerutil.RemoveFinalizer(mcs, util.MCSControllerFinalizer) if finalizersUpdated { - err := c.Client.Update(ctx, mcs) + err := c.Client.Update(context.Background(), mcs) if err != nil { klog.V(4).ErrorS(err, "failed to update MultiClusterService with finalizer", "namespace", mcs.Namespace, "name", mcs.Name) @@ -134,41 +149,7 @@ func (c *MCSController) handleMCSDelete(ctx context.Context, mcs *networkingv1al return controllerruntime.Result{}, nil } -func (c *MCSController) deleteServiceWork(mcs *networkingv1alpha1.MultiClusterService, retainClusters sets.Set[string]) error { - mcsID := util.GetLabelValue(mcs.Labels, networkingv1alpha1.MultiClusterServicePermanentIDLabel) - workList, err := helper.GetWorksByLabelsSet(c, labels.Set{ - networkingv1alpha1.MultiClusterServicePermanentIDLabel: mcsID, - }) - if err != nil { - klog.ErrorS(err, "failed to get work", "namespace", mcs.Namespace, "name", mcs.Name) - return err - } - - for _, work := range workList.Items { - if !helper.IsWorkContains(work.Spec.Workload.Manifests, serviceGVK) { - continue - } - clusterName, err := names.GetClusterName(work.Namespace) - if err != nil { - klog.Errorf("Failed to get member cluster name for work %s/%s", work.Namespace, work.Name) - continue - } - if retainClusters.Has(clusterName) { - continue - } - - if err = c.Client.Delete(context.TODO(), work.DeepCopy()); err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("Error while deleting work(%s/%s) deletion timestamp: %s", - work.Namespace, work.Name, err) - return err - } - } - - klog.V(4).InfoS("success to delete service work", "namespace", mcs.Namespace, "name", mcs.Name) - return nil -} - -func (c *MCSController) deleteMultiClusterServiceWork(mcs *networkingv1alpha1.MultiClusterService, deleteAll bool) error { +func (c *MCSController) retrieveMultiClusterService(mcs *networkingv1alpha1.MultiClusterService, providerClusters sets.Set[string]) error { mcsID := util.GetLabelValue(mcs.Labels, networkingv1alpha1.MultiClusterServicePermanentIDLabel) workList, err := helper.GetWorksByLabelsSet(c, labels.Set{ networkingv1alpha1.MultiClusterServicePermanentIDLabel: mcsID, @@ -178,12 +159,6 @@ func (c *MCSController) deleteMultiClusterServiceWork(mcs *networkingv1alpha1.Mu return err } - provisionClusters, err := helper.GetProvisionClusters(c.Client, mcs) - if err != nil { - klog.Errorf("Failed to get provision clusters by MultiClusterService(%s/%s):%v", mcs.Namespace, mcs.Name, err) - return err - } - for _, work := range workList.Items { if !helper.IsWorkContains(work.Spec.Workload.Manifests, multiClusterServiceGVK) { continue @@ -194,7 +169,7 @@ func (c *MCSController) deleteMultiClusterServiceWork(mcs *networkingv1alpha1.Mu continue } - if !deleteAll && provisionClusters.Has(clusterName) && c.IsClusterReady(clusterName) { + if providerClusters.Has(clusterName) && c.IsClusterReady(clusterName) { continue } @@ -209,7 +184,7 @@ func (c *MCSController) deleteMultiClusterServiceWork(mcs *networkingv1alpha1.Mu } } - klog.V(4).InfoS("Success to delete MultiClusterService work", "namespace", mcs.Namespace, "name", mcs.Name) + klog.V(4).InfoS("Success to delete MultiClusterService(%s/%s) work:%v", mcs.Namespace, mcs.Name, err) return nil } @@ -257,16 +232,15 @@ func (c *MCSController) cleanProvisionEndpointSliceWork(work *workv1alpha1.Work) return nil } -func (c *MCSController) handleMCSCreateOrUpdate(ctx context.Context, mcs *networkingv1alpha1.MultiClusterService) error { - klog.V(4).InfoS("Begin to handle MultiClusterService create or update event", - "namespace", mcs.Namespace, "name", mcs.Name) +func (c *MCSController) handleMultiClusterServiceCreateOrUpdate(mcs *networkingv1alpha1.MultiClusterService, providerClusters, consumerClusters sets.Set[string]) error { + klog.V(4).Infof("Begin to handle MultiClusterService(%s/%s) create or update event", mcs.Namespace, mcs.Name) // 1. if mcs not contain CrossCluster type, delete service work if needed if !helper.MultiClusterServiceCrossClusterEnabled(mcs) { - if err := c.deleteServiceWork(mcs, sets.New[string]()); err != nil { + if err := c.retrieveService(mcs); err != nil { return err } - if err := c.deleteMultiClusterServiceWork(mcs, true); err != nil { + if err := c.retrieveMultiClusterService(mcs, providerClusters); err != nil { return err } } @@ -274,70 +248,50 @@ func (c *MCSController) handleMCSCreateOrUpdate(ctx context.Context, mcs *networ // 2. add finalizer if needed finalizersUpdated := controllerutil.AddFinalizer(mcs, util.MCSControllerFinalizer) if finalizersUpdated { - err := c.Client.Update(ctx, mcs) + err := c.Client.Update(context.Background(), mcs) if err != nil { klog.V(4).ErrorS(err, "failed to update mcs with finalizer", "namespace", mcs.Namespace, "name", mcs.Name) return err } } - // 3. Generate the MCS work - if err := c.ensureMultiClusterServiceWork(ctx, mcs); err != nil { + // 3. Generate the MCS work in target clusters namespace + if err := c.propagateMultiClusterService(mcs, providerClusters); err != nil { return err } - // 4. make sure service exist - svc := &corev1.Service{} - err := c.Client.Get(ctx, types.NamespacedName{Namespace: mcs.Namespace, Name: mcs.Name}, svc) - if err != nil && !apierrors.IsNotFound(err) { - klog.ErrorS(err, "failed to get service", "namespace", mcs.Namespace, "name", mcs.Name) - return err - } - - // 5. if service not exist, delete service work if needed - if apierrors.IsNotFound(err) { - delErr := c.deleteServiceWork(mcs, sets.New[string]()) - if delErr != nil { - klog.ErrorS(err, "failed to delete service work", "namespace", mcs.Namespace, "name", mcs.Name) - c.EventRecorder.Event(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceWorkFailed, - fmt.Sprintf("failed to delete service work :%v", err)) - } - return err - } - - // 6. if service exist, create or update corresponding work in clusters - syncClusters, err := c.syncSVCWorkToClusters(ctx, mcs, svc) - if err != nil { + // 4. delete MultiClusterService work not in provision clusters and in the unready clusters + if err := c.retrieveMultiClusterService(mcs, providerClusters); err != nil { return err } - // 7. delete MultiClusterService work not in provision clusters and in the unready clusters - if err = c.deleteMultiClusterServiceWork(mcs, false); err != nil { + // 5. make sure service exist + svc := &corev1.Service{} + err := c.Client.Get(context.Background(), types.NamespacedName{Namespace: mcs.Namespace, Name: mcs.Name}, svc) + // If the Serivice are deleted, the Service's ResourceBinding will be cleaned by GC + if err != nil && !apierrors.IsNotFound(err) { + klog.Errorf("Failed to get service(%s/%s):%v", mcs.Namespace, mcs.Name, err) return err } - // 8. delete service work not in need sync clusters - if err = c.deleteServiceWork(mcs, syncClusters); err != nil { - return err + // 6. if service exists, create or update corresponding ResourceBinding + if err == nil { + if err := c.propagateService(context.Background(), mcs, svc, providerClusters, consumerClusters); err != nil { + return err + } } - klog.V(4).InfoS("success to ensure service work", "namespace", mcs.Namespace, "name", mcs.Name) + klog.V(4).Infof("Success to reconcile MultiClusterService(%s/%s)", mcs.Namespace, mcs.Name) return nil } -func (c *MCSController) ensureMultiClusterServiceWork(ctx context.Context, mcs *networkingv1alpha1.MultiClusterService) error { - provisionClusters, err := helper.GetProvisionClusters(c.Client, mcs) - if err != nil { - klog.Errorf("Failed to get provision clusters by MultiClusterService(%s/%s):%v", mcs.Namespace, mcs.Name, err) - return err - } - - for clusterName := range provisionClusters { +func (c *MCSController) propagateMultiClusterService(mcs *networkingv1alpha1.MultiClusterService, providerClusters sets.Set[string]) error { + for clusterName := range providerClusters { if !c.IsClusterReady(clusterName) { continue } workMeta := metav1.ObjectMeta{ - Name: names.GenerateMCSWorkName(mcs.Kind, mcs.Name, mcs.Namespace, clusterName), + Name: names.GenerateWorkName(mcs.Kind, mcs.Name, mcs.Namespace), Namespace: names.GenerateExecutionSpaceName(clusterName), Labels: map[string]string{ // We add this id in mutating webhook, let's just use it @@ -364,68 +318,184 @@ func (c *MCSController) ensureMultiClusterServiceWork(ctx context.Context, mcs * return nil } -func (c *MCSController) syncSVCWorkToClusters( - ctx context.Context, - mcs *networkingv1alpha1.MultiClusterService, - svc *corev1.Service, -) (sets.Set[string], error) { - syncClusters := sets.New[string]() - clusters, err := util.GetClusterSet(c.Client) - if err != nil { - klog.ErrorS(err, "failed to list clusters") - return syncClusters, err +func (c *MCSController) retrieveService(mcs *networkingv1alpha1.MultiClusterService) error { + svc := &corev1.Service{} + err := c.Client.Get(context.Background(), types.NamespacedName{Namespace: mcs.Namespace, Name: mcs.Name}, svc) + if err != nil && !apierrors.IsNotFound(err) { + klog.ErrorS(err, "failed to get service", "namespace", mcs.Namespace, "name", mcs.Name) + return err + } + + if apierrors.IsNotFound(err) { + return nil } - serverLocations := sets.New[string](mcs.Spec.ServiceProvisionClusters...) - clientLocations := sets.New[string](mcs.Spec.ServiceConsumptionClusters...) - for clusterName := range clusters { - // if ServerLocations or ClientLocations are empty, we will sync work to the all clusters - if len(serverLocations) == 0 || len(clientLocations) == 0 || - serverLocations.Has(clusterName) || clientLocations.Has(clusterName) { - syncClusters.Insert(clusterName) + svcCopy := svc.DeepCopy() + if svcCopy.Labels != nil { + delete(svcCopy.Labels, util.ResourceTemplateMatchedByLabel) + delete(svcCopy.Labels, networkingv1alpha1.MultiClusterServicePermanentIDLabel) + } + + if err = c.Client.Update(context.Background(), svc); err != nil { + klog.ErrorS(err, "failed to update service", "namespace", mcs.Namespace, "name", mcs.Name) + return err + } + + rb := &workv1alpha2.ResourceBinding{} + err = c.Client.Get(context.Background(), types.NamespacedName{Namespace: mcs.Namespace, Name: names.GenerateBindingName(svc.Kind, svc.Name)}, rb) + if err != nil { + if apierrors.IsNotFound(err) { + return nil } + klog.Errorf("Failed to get ResourceBinding(%s/%s):%v", mcs.Namespace, names.GenerateBindingName(svc.Kind, svc.Name), err) + return err + } + + if err := c.Client.Delete(context.Background(), rb); err != nil { + klog.Errorf("Failed to delete ResourceBinding(%s/%s):%v", mcs.Namespace, names.GenerateBindingName(svc.Kind, svc.Name), err) + return err + } + + return nil +} + +func (c *MCSController) propagateService(ctx context.Context, mcs *networkingv1alpha1.MultiClusterService, svc *corev1.Service, + providerClusters, consumerClusters sets.Set[string]) error { + if err := c.claimMultiClusterServiceForService(svc, mcs); err != nil { + klog.Errorf("Failed to claim for Service(%s/%s), err is %v", svc.Namespace, svc.Name, err) + return err } - svcObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(svc) + binding, err := c.buildResourceBinding(svc, mcs, providerClusters, consumerClusters) if err != nil { - return syncClusters, err + klog.Errorf("Failed to build ResourceBinding for Service(%s/%s), err is %v", svc.Namespace, svc.Name, err) + return err } - Unstructured := &unstructured.Unstructured{Object: svcObj} - var errs []error - for clusterName := range syncClusters { - workMeta := metav1.ObjectMeta{ - Name: names.GenerateMCSWorkName(svc.Kind, svc.Name, svc.Namespace, clusterName), - Namespace: names.GenerateExecutionSpaceName(clusterName), - Finalizers: []string{util.ExecutionControllerFinalizer}, + var operationResult controllerutil.OperationResult + bindingCopy := binding.DeepCopy() + err = retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { + operationResult, err = controllerutil.CreateOrUpdate(ctx, c.Client, bindingCopy, func() error { + // If this binding exists and its owner is not the input object, return error and let garbage collector + // delete this binding and try again later. See https://github.com/karmada-io/karmada/issues/2090. + if ownerRef := metav1.GetControllerOfNoCopy(bindingCopy); ownerRef != nil && ownerRef.UID != svc.GetUID() { + return fmt.Errorf("failed to update binding due to different owner reference UID, will " + + "try again later after binding is garbage collected, see https://github.com/karmada-io/karmada/issues/2090") + } + + if util.GetLabelValue(bindingCopy.Labels, workv1alpha2.ResourceBindingPermanentIDLabel) == "" { + binding.Labels = util.DedupeAndMergeLabels(binding.Labels, + map[string]string{workv1alpha2.ResourceBindingPermanentIDLabel: uuid.New().String()}) + } + // Just update necessary fields, especially avoid modifying Spec.Clusters which is scheduling result, if already exists. + bindingCopy.Annotations = binding.Annotations + bindingCopy.Labels = binding.Labels + bindingCopy.OwnerReferences = binding.OwnerReferences + bindingCopy.Finalizers = binding.Finalizers + bindingCopy.Spec.Placement = binding.Spec.Placement + bindingCopy.Spec.Resource = binding.Spec.Resource + bindingCopy.Spec.ConflictResolution = binding.Spec.ConflictResolution + return nil + }) + if err != nil { + return err + } + return nil + }) + if err != nil { + klog.Errorf("Failed to create/update ResourceBinding(%s/%s):%v", bindingCopy.Namespace, bindingCopy.Name, err) + return err + } + + if operationResult == controllerutil.OperationResultCreated { + klog.Infof("Create ResourceBinding(%s/%s) successfully.", binding.GetNamespace(), binding.GetName()) + } else if operationResult == controllerutil.OperationResultUpdated { + klog.Infof("Update ResourceBinding(%s/%s) successfully.", binding.GetNamespace(), binding.GetName()) + } else { + klog.V(2).Infof("ResourceBinding(%s/%s) is up to date.", binding.GetNamespace(), binding.GetName()) + } + + return nil +} + +func (c *MCSController) buildResourceBinding(svc *corev1.Service, mcs *networkingv1alpha1.MultiClusterService, + providerClusters, consumerClusters sets.Set[string]) (*workv1alpha2.ResourceBinding, error) { + propagateClusters := providerClusters.Clone().Insert(consumerClusters.Clone().UnsortedList()...).UnsortedList() + placement := &policyv1alpha1.Placement{ + ClusterAffinity: &policyv1alpha1.ClusterAffinity{ + ClusterNames: propagateClusters, + }, + } + propagationBinding := &workv1alpha2.ResourceBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.GenerateBindingName(svc.Kind, svc.Name), + Namespace: svc.Namespace, + OwnerReferences: []metav1.OwnerReference{ + {APIVersion: svc.APIVersion, Kind: svc.Kind, Name: svc.Name, UID: svc.UID}, + }, + Annotations: map[string]string{ + networkingv1alpha1.MultiClusterServiceNameAnnotation: mcs.Name, + networkingv1alpha1.MultiClusterServiceNamespaceAnnotation: mcs.Namespace, + }, Labels: map[string]string{ - // We add this id in mutating webhook, let's just use it + workv1alpha2.BindingManagedByLabel: util.MultiClusterServiceKind, networkingv1alpha1.MultiClusterServicePermanentIDLabel: util.GetLabelValue(mcs.Labels, networkingv1alpha1.MultiClusterServicePermanentIDLabel), - util.ManagedByKarmadaLabel: util.ManagedByKarmadaLabelValue, }, - } + Finalizers: []string{util.BindingControllerFinalizer}, + }, + Spec: workv1alpha2.ResourceBindingSpec{ + Placement: placement, + ConflictResolution: policyv1alpha1.ConflictOverwrite, + Resource: workv1alpha2.ObjectReference{ + APIVersion: svc.APIVersion, + Kind: svc.Kind, + Namespace: svc.Namespace, + Name: svc.Name, + UID: svc.UID, + ResourceVersion: svc.ResourceVersion, + }, + }, + } - // Add these two label as for work status synchronization - util.MergeLabel(Unstructured, workv1alpha1.WorkNameLabel, names.GenerateMCSWorkName(svc.Kind, svc.Name, svc.Namespace, clusterName)) - util.MergeLabel(Unstructured, workv1alpha1.WorkNamespaceLabel, names.GenerateExecutionSpaceName(clusterName)) + return propagationBinding, nil +} - if err = helper.CreateOrUpdateWork(c, workMeta, Unstructured); err != nil { - klog.Errorf("Failed to create or update resource(%v/%v) in the given member cluster %s, err is %v", - workMeta.GetNamespace(), workMeta.GetName(), clusterName, err) - c.EventRecorder.Event(mcs, corev1.EventTypeWarning, events.EventReasonSyncServiceWorkFailed, fmt.Sprintf( - "Failed to create or update resource(%v/%v) in member cluster(%s): %v", - workMeta.GetNamespace(), workMeta.GetName(), clusterName, err)) - errs = append(errs, err) - } +func (c *MCSController) claimMultiClusterServiceForService(svc *corev1.Service, mcs *networkingv1alpha1.MultiClusterService) error { + svcCopy := svc.DeepCopy() + if svcCopy.Labels == nil { + svcCopy.Labels = map[string]string{} } - if len(errs) != 0 { - return syncClusters, utilerrors.NewAggregate(errs) + if svcCopy.Annotations == nil { + svcCopy.Annotations = map[string]string{} } - return syncClusters, nil + // cleanup the policy labels + delete(svcCopy.Labels, policyv1alpha1.PropagationPolicyNameLabel) + delete(svcCopy.Labels, policyv1alpha1.PropagationPolicyNamespaceLabel) + delete(svcCopy.Labels, policyv1alpha1.PropagationPolicyPermanentIDLabel) + delete(svcCopy.Labels, policyv1alpha1.ClusterPropagationPolicyLabel) + delete(svcCopy.Labels, policyv1alpha1.ClusterPropagationPolicyPermanentIDLabel) + + svcCopy.Labels[util.ResourceTemplateMatchedByLabel] = util.MultiClusterServiceKind + svcCopy.Labels[networkingv1alpha1.MultiClusterServicePermanentIDLabel] = util.GetLabelValue(mcs.Labels, networkingv1alpha1.MultiClusterServicePermanentIDLabel) + + // cleanup policy annotations + delete(svcCopy.Annotations, policyv1alpha1.PropagationPolicyNameAnnotation) + delete(svcCopy.Annotations, policyv1alpha1.PropagationPolicyNamespaceAnnotation) + delete(svcCopy.Annotations, policyv1alpha1.ClusterPropagationPolicyAnnotation) + + svcCopy.Annotations[networkingv1alpha1.MultiClusterServiceNameAnnotation] = mcs.Name + svcCopy.Annotations[networkingv1alpha1.MultiClusterServiceNamespaceAnnotation] = mcs.Namespace + + if err := c.Client.Update(context.Background(), svcCopy); err != nil { + klog.Errorf("Failed to update service(%s/%s):%v ", svc.Namespace, svc.Name, err) + return err + } + + return nil } -func (c *MCSController) updateMCSStatus(mcs *networkingv1alpha1.MultiClusterService, status metav1.ConditionStatus, reason, message string) error { +func (c *MCSController) updateMultiClusterServiceStatus(mcs *networkingv1alpha1.MultiClusterService, status metav1.ConditionStatus, reason, message string) error { serviceAppliedCondition := metav1.Condition{ Type: networkingv1alpha1.MCSServiceAppliedConditionType, Status: status, @@ -502,6 +572,7 @@ func (c *MCSController) SetupWithManager(mgr controllerruntime.Manager) error { // We only care about the update events below: if equality.Semantic.DeepEqual(svcOld.Annotations, svcNew.Annotations) && + equality.Semantic.DeepEqual(svcOld.Labels, svcNew.Labels) && equality.Semantic.DeepEqual(svcOld.Spec, svcNew.Spec) { return false } @@ -552,7 +623,7 @@ func (c *MCSController) clusterMapFunc() handler.MapFunc { var requests []reconcile.Request for index := range mcsList.Items { - if !needSyncMultiClusterService(&mcsList.Items[index], clusterName) { + if need, err := c.needSyncMultiClusterService(&mcsList.Items[index], clusterName); err != nil || !need { continue } @@ -564,11 +635,27 @@ func (c *MCSController) clusterMapFunc() handler.MapFunc { } } -func needSyncMultiClusterService(mcs *networkingv1alpha1.MultiClusterService, clusterName string) bool { - if len(mcs.Spec.ServiceProvisionClusters) == 0 || len(mcs.Spec.ServiceConsumptionClusters) == 0 { - return true +func (c *MCSController) needSyncMultiClusterService(mcs *networkingv1alpha1.MultiClusterService, clusterName string) (bool, error) { + if len(mcs.Spec.ProviderClusters) == 0 || len(mcs.Spec.ConsumerClusters) == 0 { + return true, nil + } + + providerClusters, _, err := helper.GetProviderClusters(c.Client, mcs) + if err != nil { + klog.Errorf("Failed to get provider clusters by MultiClusterService(%s/%s):%v", mcs.Namespace, mcs.Name, err) + return false, err + } + if providerClusters.Has(clusterName) { + return true, nil + } + + consumerClusters, _, err := helper.GetConsumerClustres(c.Client, mcs) + if err != nil { + klog.Errorf("Failed to get consumer clusters by MultiClusterService(%s/%s):%v", mcs.Namespace, mcs.Name, err) + return false, err + } + if consumerClusters.Has(clusterName) { + return true, nil } - clusters := sets.New[string](mcs.Spec.ServiceProvisionClusters...) - clusters.Insert(mcs.Spec.ServiceConsumptionClusters...) - return clusters.Has(clusterName) + return false, nil } diff --git a/pkg/detector/detector.go b/pkg/detector/detector.go index 99bef8db9753..b46f0fb284bd 100644 --- a/pkg/detector/detector.go +++ b/pkg/detector/detector.go @@ -247,6 +247,12 @@ func (d *ResourceDetector) Reconcile(key util.QueueKey) error { return err } + resoruceTemplateMatchedBy := util.GetLabelValue(object.GetLabels(), util.ResourceTemplateMatchedByLabel) + if resoruceTemplateMatchedBy != "" && resoruceTemplateMatchedBy != util.PolicyKind { + d.RemoveWaiting(clusterWideKey) + return nil + } + return d.propagateResource(object, clusterWideKey) } diff --git a/pkg/events/events.go b/pkg/events/events.go index b6ec5f55e0d5..16c5e12e0a2a 100644 --- a/pkg/events/events.go +++ b/pkg/events/events.go @@ -132,12 +132,14 @@ const ( // Define events for MultiClusterService objects with CrossCluster type. const ( - // EventReasonSyncServiceWorkFailed is indicates that sync service work failed. - EventReasonSyncServiceWorkFailed string = "SyncServiceWorkFailed" - // EventReasonSyncServiceWorkSucceed is indicates that sync service work succeed. - EventReasonSyncServiceWorkSucceed string = "SyncServiceWorkSucceed" + // EventReasonSyncServiceFailed is indicates that sync service work failed. + EventReasonSyncServiceFailed string = "SyncServiceWorkFailed" + // EventReasonSyncServiceSucceed is indicates that sync service work succeed. + EventReasonSyncServiceSucceed string = "SyncServiceWorkSucceed" // EventReasonDispatchEndpointSliceFailed indicates that dispatch endpointslice failed. EventReasonDispatchEndpointSliceFailed = "DispatchEndpointSliceFailed" // EventReasonDispatchEndpointSliceSucceed indicates that dispatch endpointslice succeed. EventReasonDispatchEndpointSliceSucceed = "DispatchEndpointSliceSucceed" + // EventReasonConfigurationRedundant indicates that MultiClusterService configuration redundant. + EventReasonConfigurationRedundant = "ConfigurationRedundant" ) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index a383d7310209..bda25e5c5805 100755 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -79,6 +79,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1.RuleWithOperations": schema_pkg_apis_config_v1alpha1_RuleWithOperations(ref), "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1.StatusAggregation": schema_pkg_apis_config_v1alpha1_StatusAggregation(ref), "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1.StatusReflection": schema_pkg_apis_config_v1alpha1_StatusReflection(ref), + "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ClusterSelector": schema_pkg_apis_networking_v1alpha1_ClusterSelector(ref), "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposurePort": schema_pkg_apis_networking_v1alpha1_ExposurePort(ref), "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposureRange": schema_pkg_apis_networking_v1alpha1_ExposureRange(ref), "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.MultiClusterIngress": schema_pkg_apis_networking_v1alpha1_MultiClusterIngress(ref), @@ -2688,6 +2689,26 @@ func schema_pkg_apis_config_v1alpha1_StatusReflection(ref common.ReferenceCallba } } +func schema_pkg_apis_networking_v1alpha1_ClusterSelector(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ClusterSelector specifies the cluster to be selected.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name is the name of the cluster to be selected.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_pkg_apis_networking_v1alpha1_ExposurePort(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -2982,14 +3003,14 @@ func schema_pkg_apis_networking_v1alpha1_MultiClusterServiceSpec(ref common.Refe }, "range": { SchemaProps: spec.SchemaProps{ - Description: "Range specifies the ranges where the referencing service should be exposed. Only valid and optional in case of Types contains CrossCluster. If not set and Types contains CrossCluster, all clusters will be selected, that means the referencing service will be exposed across all registered clusters. Deprecated: in favor of ServiceProvisionClusters/ServiceConsumptionClusters.", + Description: "Range specifies the ranges where the referencing service should be exposed. Only valid and optional in case of Types contains CrossCluster. If not set and Types contains CrossCluster, all clusters will be selected, that means the referencing service will be exposed across all registered clusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.", Default: map[string]interface{}{}, Ref: ref("github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposureRange"), }, }, "serviceProvisionClusters": { SchemaProps: spec.SchemaProps{ - Description: "ServiceProvisionClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ServiceConsumptionClusters.", + Description: "ServiceProvisionClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ServiceConsumptionClusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -3004,7 +3025,7 @@ func schema_pkg_apis_networking_v1alpha1_MultiClusterServiceSpec(ref common.Refe }, "serviceConsumptionClusters": { SchemaProps: spec.SchemaProps{ - Description: "ServiceConsumptionClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the service will be exposed to all clusters.", + Description: "ServiceConsumptionClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the service will be exposed to all clusters. Deprecated: in favor of ProviderClusters/ConsumerClusters.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -3017,12 +3038,40 @@ func schema_pkg_apis_networking_v1alpha1_MultiClusterServiceSpec(ref common.Refe }, }, }, + "providerClusters": { + SchemaProps: spec.SchemaProps{ + Description: "ProviderClusters specifies the clusters which will provision the service backend. If leave it empty, we will collect the backend endpoints from all clusters and sync them to the ConsumerClusters.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ClusterSelector"), + }, + }, + }, + }, + }, + "consumerClusters": { + SchemaProps: spec.SchemaProps{ + Description: "ConsumerClusters specifies the clusters where the service will be exposed, for clients. If leave it empty, the service will be exposed to all clusters.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ClusterSelector"), + }, + }, + }, + }, + }, }, Required: []string{"types"}, }, }, Dependencies: []string{ - "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposurePort", "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposureRange"}, + "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ClusterSelector", "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposurePort", "github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1.ExposureRange"}, } } diff --git a/pkg/scheduler/event_handler.go b/pkg/scheduler/event_handler.go index 51f0880170c7..4b19649796fb 100644 --- a/pkg/scheduler/event_handler.go +++ b/pkg/scheduler/event_handler.go @@ -108,7 +108,8 @@ func (s *Scheduler) resourceBindingEventFilter(obj interface{}) bool { } return util.GetLabelValue(accessor.GetLabels(), policyv1alpha1.PropagationPolicyNameLabel) != "" || - util.GetLabelValue(accessor.GetLabels(), policyv1alpha1.ClusterPropagationPolicyLabel) != "" + util.GetLabelValue(accessor.GetLabels(), policyv1alpha1.ClusterPropagationPolicyLabel) != "" || + util.GetLabelValue(accessor.GetLabels(), workv1alpha2.BindingManagedByLabel) != "" } func (s *Scheduler) onResourceBindingAdd(obj interface{}) { diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 37cdf684083a..64f7bde26520 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -58,6 +58,8 @@ const ( // resourcetemplate.karmada.io/retain-replicas: true // with value `true` indicates retain // resourcetemplate.karmada.io/retain-replicas: false // with value `false` and others, indicates not retain RetainReplicasLabel = "resourcetemplate.karmada.io/retain-replicas" + + ResourceTemplateMatchedByLabel = "resorucetemplate.karmada.io/matched-by" ) const ( @@ -172,6 +174,10 @@ const ( ServiceExportKind = "ServiceExport" // ServiceImportKind indicates the target resource is a serviceimport crd ServiceImportKind = "ServiceImport" + + PolicyKind = "Policy" + + MultiClusterServiceKind = "MultiClusterService" ) // Define resource filed diff --git a/pkg/util/helper/mcs.go b/pkg/util/helper/mcs.go index be112ec3c268..7b174db2b573 100644 --- a/pkg/util/helper/mcs.go +++ b/pkg/util/helper/mcs.go @@ -106,28 +106,34 @@ func MultiClusterServiceCrossClusterEnabled(mcs *networkingv1alpha1.MultiCluster return false } -func GetProvisionClusters(client client.Client, mcs *networkingv1alpha1.MultiClusterService) (sets.Set[string], error) { - provisionClusters := sets.New[string](mcs.Spec.ServiceProvisionClusters...) - existClusters, err := util.GetClusterSet(client) +func GetProviderClusters(client client.Client, mcs *networkingv1alpha1.MultiClusterService) (existClusters, noneExistClusters sets.Set[string], err error) { + provisionClusters := sets.New[string]() + for _, p := range mcs.Spec.ProviderClusters { + provisionClusters.Insert(p.Name) + } + allClusters, err := util.GetClusterSet(client) if err != nil { klog.Errorf("Failed to get cluster set, Error: %v", err) - return nil, err + return nil, nil, err } if len(provisionClusters) == 0 { - return existClusters, nil + return allClusters, nil, nil } - return provisionClusters.Intersection(existClusters), nil + return provisionClusters.Clone().Intersection(allClusters), provisionClusters.Clone().Delete(allClusters.UnsortedList()...), nil } -func GetConsumptionClustres(client client.Client, mcs *networkingv1alpha1.MultiClusterService) (sets.Set[string], error) { - consumptionClusters := sets.New[string](mcs.Spec.ServiceConsumptionClusters...) - existClusters, err := util.GetClusterSet(client) +func GetConsumerClustres(client client.Client, mcs *networkingv1alpha1.MultiClusterService) (existClusters, noneExistClusters sets.Set[string], err error) { + consumptionClusters := sets.New[string]() + for _, c := range mcs.Spec.ConsumerClusters { + consumptionClusters.Insert(c.Name) + } + allClusters, err := util.GetClusterSet(client) if err != nil { klog.Errorf("Failed to get cluster set, Error: %v", err) - return nil, err + return nil, nil, err } if len(consumptionClusters) == 0 { - return existClusters, nil + return allClusters, nil, nil } - return consumptionClusters.Intersection(existClusters), nil + return consumptionClusters.Clone().Intersection(allClusters), consumptionClusters.Clone().Delete(allClusters.UnsortedList()...), nil } diff --git a/pkg/util/names/names.go b/pkg/util/names/names.go index e11bb5acfda1..30ca752f4e8c 100644 --- a/pkg/util/names/names.go +++ b/pkg/util/names/names.go @@ -89,10 +89,6 @@ func GenerateBindingReferenceKey(namespace, name string) string { return rand.SafeEncodeString(fmt.Sprint(hash.Sum32())) } -func GenerateMCSWorkName(kind, name, namespace, cluster string) string { - return GenerateWorkName(kind, name, cluster+"/"+namespace) -} - // GenerateWorkName will generate work name by its name and the hash of its namespace, kind and name. func GenerateWorkName(kind, name, namespace string) string { // The name of resources, like 'Role'/'ClusterRole'/'RoleBinding'/'ClusterRoleBinding', diff --git a/pkg/webhook/multiclusterservice/validating.go b/pkg/webhook/multiclusterservice/validating.go index d7506ae9fd35..13388b76dd3f 100644 --- a/pkg/webhook/multiclusterservice/validating.go +++ b/pkg/webhook/multiclusterservice/validating.go @@ -103,19 +103,19 @@ func (v *ValidatingAdmission) validateMultiClusterServiceSpec(mcs *networkingv1a exposureType := mcs.Spec.Types[i] allErrs = append(allErrs, v.validateExposureType(&exposureType, typePath)...) } - clusterNamesPath := specPath.Child("range").Child("serviceProvisionClusters") - for i := range mcs.Spec.ServiceProvisionClusters { + clusterNamesPath := specPath.Child("range").Child("providerClusters") + for i := range mcs.Spec.ProviderClusters { clusterNamePath := clusterNamesPath.Index(i) - clusterName := mcs.Spec.ServiceProvisionClusters[i] + clusterName := mcs.Spec.ProviderClusters[i].Name if errMegs := clustervalidation.ValidateClusterName(clusterName); len(errMegs) > 0 { allErrs = append(allErrs, field.Invalid(clusterNamePath, clusterName, strings.Join(errMegs, ","))) } } - clusterNamesPath = specPath.Child("range").Child("serviceConsumptionClusters") - for i := range mcs.Spec.ServiceConsumptionClusters { + clusterNamesPath = specPath.Child("range").Child("consumerClusters") + for i := range mcs.Spec.ConsumerClusters { clusterNamePath := clusterNamesPath.Index(i) - clusterName := mcs.Spec.ServiceConsumptionClusters[i] + clusterName := mcs.Spec.ConsumerClusters[i].Name if errMegs := clustervalidation.ValidateClusterName(clusterName); len(errMegs) > 0 { allErrs = append(allErrs, field.Invalid(clusterNamePath, clusterName, strings.Join(errMegs, ","))) } diff --git a/pkg/webhook/multiclusterservice/validating_test.go b/pkg/webhook/multiclusterservice/validating_test.go index d4d17eb5efc3..6645684dfe88 100755 --- a/pkg/webhook/multiclusterservice/validating_test.go +++ b/pkg/webhook/multiclusterservice/validating_test.go @@ -53,8 +53,14 @@ func TestValidateMultiClusterServiceSpec(t *testing.T) { networkingv1alpha1.ExposureTypeLoadBalancer, networkingv1alpha1.ExposureTypeCrossCluster, }, - ServiceProvisionClusters: []string{"member1", "member2"}, - ServiceConsumptionClusters: []string{"member1", "member2"}, + ProviderClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, + ConsumerClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, }, }, expectedErr: field.ErrorList{}, @@ -77,8 +83,14 @@ func TestValidateMultiClusterServiceSpec(t *testing.T) { networkingv1alpha1.ExposureTypeLoadBalancer, networkingv1alpha1.ExposureTypeLoadBalancer, }, - ServiceProvisionClusters: []string{"member1", "member2"}, - ServiceConsumptionClusters: []string{"member1", "member2"}, + ProviderClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, + ConsumerClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, }, }, expectedErr: field.ErrorList{field.Duplicate(specFld.Child("ports").Index(1).Child("name"), "foo")}, @@ -96,8 +108,14 @@ func TestValidateMultiClusterServiceSpec(t *testing.T) { Types: []networkingv1alpha1.ExposureType{ networkingv1alpha1.ExposureTypeLoadBalancer, }, - ServiceProvisionClusters: []string{"member1", "member2"}, - ServiceConsumptionClusters: []string{"member1", "member2"}, + ProviderClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, + ConsumerClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, }, }, expectedErr: field.ErrorList{field.Invalid(specFld.Child("ports").Index(0).Child("port"), int32(163121), validation.InclusiveRangeError(1, 65535))}, @@ -115,8 +133,14 @@ func TestValidateMultiClusterServiceSpec(t *testing.T) { Types: []networkingv1alpha1.ExposureType{ "", }, - ServiceProvisionClusters: []string{"member1", "member2"}, - ServiceConsumptionClusters: []string{"member1", "member2"}, + ProviderClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, + ConsumerClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "member1"}, + {Name: "member2"}, + }, }, }, expectedErr: field.ErrorList{field.Invalid(specFld.Child("types").Index(0), networkingv1alpha1.ExposureType(""), "ExposureType Error")}, @@ -134,11 +158,13 @@ func TestValidateMultiClusterServiceSpec(t *testing.T) { Types: []networkingv1alpha1.ExposureType{ networkingv1alpha1.ExposureTypeCrossCluster, }, - ServiceProvisionClusters: []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, - ServiceConsumptionClusters: []string{}, + ProviderClusters: []networkingv1alpha1.ClusterSelector{ + {Name: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, + }, + ConsumerClusters: []networkingv1alpha1.ClusterSelector{}, }, }, - expectedErr: field.ErrorList{field.Invalid(specFld.Child("range").Child("serviceProvisionClusters").Index(0), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "must be no more than 48 characters")}, + expectedErr: field.ErrorList{field.Invalid(specFld.Child("range").Child("providerClusters").Index(0), "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "must be no more than 48 characters")}, }, } for _, tt := range tests { diff --git a/test/e2e/mcs_test.go b/test/e2e/mcs_test.go index d17c59f6b777..c8247e882790 100644 --- a/test/e2e/mcs_test.go +++ b/test/e2e/mcs_test.go @@ -559,7 +559,7 @@ var _ = ginkgo.Describe("CrossCluster MultiClusterService testing", func() { }) gomega.Eventually(func() bool { - return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ServiceProvisionClusters, mcs.Spec.ServiceConsumptionClusters) + return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ProviderClusters, mcs.Spec.ConsumerClusters) }, pollTimeout, pollInterval).Should(gomega.BeTrue()) }) }) @@ -595,7 +595,7 @@ var _ = ginkgo.Describe("CrossCluster MultiClusterService testing", func() { }) gomega.Eventually(func() bool { - return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ServiceProvisionClusters, mcs.Spec.ServiceConsumptionClusters) + return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ProviderClusters, mcs.Spec.ConsumerClusters) }, pollTimeout, pollInterval).Should(gomega.BeTrue()) }) }) @@ -633,11 +633,15 @@ var _ = ginkgo.Describe("CrossCluster MultiClusterService testing", func() { }) gomega.Eventually(func() bool { - return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ServiceProvisionClusters, mcs.Spec.ServiceConsumptionClusters) + return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ProviderClusters, mcs.Spec.ConsumerClusters) }, pollTimeout, pollInterval).Should(gomega.BeTrue()) - mcs.Spec.ServiceProvisionClusters = []string{member2Name} - mcs.Spec.ServiceConsumptionClusters = []string{member1Name} + mcs.Spec.ProviderClusters = []networkingv1alpha1.ClusterSelector{ + {Name: member2Name}, + } + mcs.Spec.ConsumerClusters = []networkingv1alpha1.ClusterSelector{ + {Name: member1Name}, + } framework.UpdateMultiClusterService(karmadaClient, mcs) framework.WaitMultiClusterServicePresentOnClustersFitWith(karmadaClient, testNamespace, mcsName, func(mcs *networkingv1alpha1.MultiClusterService) bool { @@ -676,7 +680,7 @@ var _ = ginkgo.Describe("CrossCluster MultiClusterService testing", func() { }) gomega.Eventually(func() bool { - return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ServiceProvisionClusters, mcs.Spec.ServiceConsumptionClusters) + return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ProviderClusters, mcs.Spec.ConsumerClusters) }, pollTimeout, pollInterval).Should(gomega.BeTrue()) }) }) @@ -712,7 +716,7 @@ var _ = ginkgo.Describe("CrossCluster MultiClusterService testing", func() { }) gomega.Eventually(func() bool { - return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ServiceProvisionClusters, mcs.Spec.ServiceConsumptionClusters) + return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ProviderClusters, mcs.Spec.ConsumerClusters) }, pollTimeout, pollInterval).Should(gomega.BeTrue()) }) }) @@ -748,21 +752,30 @@ var _ = ginkgo.Describe("CrossCluster MultiClusterService testing", func() { }) gomega.Eventually(func() bool { - return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ServiceProvisionClusters, mcs.Spec.ServiceConsumptionClusters) + return checkEndpointSliceWithMultiClusterService(testNamespace, mcsName, mcs.Spec.ProviderClusters, mcs.Spec.ConsumerClusters) }, pollTimeout, pollInterval).Should(gomega.BeTrue()) }) }) }) -func checkEndpointSliceWithMultiClusterService(mcsNamespace, mcsName string, provisionClusters, consumptionClusters []string) bool { - if len(provisionClusters) == 0 { - provisionClusters = framework.ClusterNames() +func checkEndpointSliceWithMultiClusterService(mcsNamespace, mcsName string, provisionClusters, consumptionClusters []networkingv1alpha1.ClusterSelector) bool { + providerClusterNames := framework.ClusterNames() + if len(provisionClusters) != 0 { + providerClusterNames = []string{} + for _, provisionCluster := range provisionClusters { + providerClusterNames = append(providerClusterNames, provisionCluster.Name) + } } - if len(consumptionClusters) == 0 { - consumptionClusters = framework.ClusterNames() + + consumerClusterNames := framework.ClusterNames() + if len(consumptionClusters) != 0 { + consumerClusterNames = []string{} + for _, consumptionCluster := range consumptionClusters { + consumerClusterNames = append(consumerClusterNames, consumptionCluster.Name) + } } - for _, clusterName := range provisionClusters { + for _, clusterName := range providerClusterNames { provisonClusterClient := framework.GetClusterClient(clusterName) gomega.Expect(provisonClusterClient).ShouldNot(gomega.BeNil()) @@ -771,8 +784,8 @@ func checkEndpointSliceWithMultiClusterService(mcsNamespace, mcsName string, pro }) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - for _, consumptionCluster := range consumptionClusters { - consumptionClusterClient := framework.GetClusterClient(consumptionCluster) + for _, consumerCluster := range consumerClusterNames { + consumptionClusterClient := framework.GetClusterClient(consumerCluster) gomega.Expect(consumptionClusterClient).ShouldNot(gomega.BeNil()) consumptionEPSList, err := consumptionClusterClient.DiscoveryV1().EndpointSlices(mcsNamespace).List(context.TODO(), metav1.ListOptions{ @@ -780,7 +793,7 @@ func checkEndpointSliceWithMultiClusterService(mcsNamespace, mcsName string, pro }) gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) - if !checkEndpointSliceSynced(provisionEPSList, consumptionEPSList, clusterName, consumptionCluster) { + if !checkEndpointSliceSynced(provisionEPSList, consumptionEPSList, clusterName, consumerCluster) { return false } } diff --git a/test/helper/resource.go b/test/helper/resource.go index 1a2353e286d0..b6ac700334e8 100644 --- a/test/helper/resource.go +++ b/test/helper/resource.go @@ -262,6 +262,14 @@ func NewDaemonSet(namespace string, name string) *appsv1.DaemonSet { } func NewCrossClusterMultiClusterService(namespace, name string, provisionClusters, consumptionClusters []string) *networkingv1alpha1.MultiClusterService { + providerClusterSelector := []networkingv1alpha1.ClusterSelector{} + for _, cluster := range provisionClusters { + providerClusterSelector = append(providerClusterSelector, networkingv1alpha1.ClusterSelector{Name: cluster}) + } + consumerClusterSelector := []networkingv1alpha1.ClusterSelector{} + for _, cluster := range consumptionClusters { + consumerClusterSelector = append(consumerClusterSelector, networkingv1alpha1.ClusterSelector{Name: cluster}) + } return &networkingv1alpha1.MultiClusterService{ TypeMeta: metav1.TypeMeta{ APIVersion: "networking.karmada.io/v1alpha1", @@ -275,8 +283,8 @@ func NewCrossClusterMultiClusterService(namespace, name string, provisionCluster Types: []networkingv1alpha1.ExposureType{ networkingv1alpha1.ExposureTypeCrossCluster, }, - ServiceProvisionClusters: provisionClusters, - ServiceConsumptionClusters: consumptionClusters, + ProviderClusters: providerClusterSelector, + ConsumerClusters: consumerClusterSelector, }, } }