From a0a1060f65f4408e0b7b550c524b41fa8852f58b Mon Sep 17 00:00:00 2001 From: changzhen Date: Mon, 24 May 2021 10:47:01 +0800 Subject: [PATCH] ImageOverrider Implementation Signed-off-by: changzhen --- ...cy.karmada.io_clusteroverridepolicies.yaml | 9 +- .../policy.karmada.io_overridepolicies.yaml | 9 +- pkg/apis/policy/v1alpha1/override_types.go | 6 +- pkg/util/constants.go | 16 ++ pkg/util/detector/detector.go | 6 +- pkg/util/helper/binding.go | 6 +- pkg/util/objectwatcher/retain.go | 14 +- .../overridemanager/imageoverridemanager.go | 141 ++++++++++++++++++ pkg/util/overridemanager/overridemanager.go | 26 +++- 9 files changed, 201 insertions(+), 32 deletions(-) create mode 100644 pkg/util/overridemanager/imageoverridemanager.go diff --git a/artifacts/deploy/policy.karmada.io_clusteroverridepolicies.yaml b/artifacts/deploy/policy.karmada.io_clusteroverridepolicies.yaml index 32afff72b2fd..f95a70dff6c1 100644 --- a/artifacts/deploy/policy.karmada.io_clusteroverridepolicies.yaml +++ b/artifacts/deploy/policy.karmada.io_clusteroverridepolicies.yaml @@ -73,10 +73,11 @@ spec: automatically detect image fields if the resource type is Pod, ReplicaSet, Deployment or StatefulSet by following rule: - Pod: spec/containers//image - ReplicaSet: - spec/template/spec//image - Deployment: spec/template/spec//image - \ - StatefulSet: spec/template/spec//image In addition, - all images will be processed if the resource object has - more than one containers. \n If not nil, only images matches + spec/template/spec/containers//image - Deployment: + spec/template/spec/containers//image - StatefulSet: + spec/template/spec/containers//image In addition, all + images will be processed if the resource object has more + than one containers. \n If not nil, only images matches the filters will be processed." properties: path: diff --git a/artifacts/deploy/policy.karmada.io_overridepolicies.yaml b/artifacts/deploy/policy.karmada.io_overridepolicies.yaml index c961615ae55f..b2896ca8fe82 100644 --- a/artifacts/deploy/policy.karmada.io_overridepolicies.yaml +++ b/artifacts/deploy/policy.karmada.io_overridepolicies.yaml @@ -73,10 +73,11 @@ spec: automatically detect image fields if the resource type is Pod, ReplicaSet, Deployment or StatefulSet by following rule: - Pod: spec/containers//image - ReplicaSet: - spec/template/spec//image - Deployment: spec/template/spec//image - \ - StatefulSet: spec/template/spec//image In addition, - all images will be processed if the resource object has - more than one containers. \n If not nil, only images matches + spec/template/spec/containers//image - Deployment: + spec/template/spec/containers//image - StatefulSet: + spec/template/spec/containers//image In addition, all + images will be processed if the resource object has more + than one containers. \n If not nil, only images matches the filters will be processed." properties: path: diff --git a/pkg/apis/policy/v1alpha1/override_types.go b/pkg/apis/policy/v1alpha1/override_types.go index 631077d7d334..fb19ae6c4eb3 100644 --- a/pkg/apis/policy/v1alpha1/override_types.go +++ b/pkg/apis/policy/v1alpha1/override_types.go @@ -57,9 +57,9 @@ type ImageOverrider struct { // Defaults to nil, in that case, the system will automatically detect image fields if the resource type is // Pod, ReplicaSet, Deployment or StatefulSet by following rule: // - Pod: spec/containers//image - // - ReplicaSet: spec/template/spec//image - // - Deployment: spec/template/spec//image - // - StatefulSet: spec/template/spec//image + // - ReplicaSet: spec/template/spec/containers//image + // - Deployment: spec/template/spec/containers//image + // - StatefulSet: spec/template/spec/containers//image // In addition, all images will be processed if the resource object has more than one containers. // // If not nil, only images matches the filters will be processed. diff --git a/pkg/util/constants.go b/pkg/util/constants.go index 08a0a7cb6d81..f6d42ee45e5b 100644 --- a/pkg/util/constants.go +++ b/pkg/util/constants.go @@ -54,3 +54,19 @@ const ( // is deleted before Work itself is deleted. ExecutionControllerFinalizer = "karmada.io/execution-controller" ) + +// Define resource kind. +const ( + // DeploymentKind indicates the target resource is a deployment + DeploymentKind = "Deployment" + // ServiceKind indicates the target resource is a service + ServiceKind = "Service" + // PodKind indicates the target resource is a pod + PodKind = "Pod" + // ServiceAccountKind indicates the target resource is a serviceaccount + ServiceAccountKind = "ServiceAccount" + // ReplicaSetKind indicates the target resource is a replicaset + ReplicaSetKind = "ReplicaSet" + // StatefulSetKind indicates the target resource is a statefulset + StatefulSetKind = "StatefulSet" +) diff --git a/pkg/util/detector/detector.go b/pkg/util/detector/detector.go index 99697a1b73b8..c2963164acba 100644 --- a/pkg/util/detector/detector.go +++ b/pkg/util/detector/detector.go @@ -860,7 +860,7 @@ func (d *ResourceDetector) ReconcileResourceBinding(key util.QueueKey) error { klog.Infof("Reconciling resource binding(%s/%s)", binding.Namespace, binding.Name) switch binding.Spec.Resource.Kind { - case helper.DeploymentKind: + case util.DeploymentKind: return d.AggregateDeploymentStatus(binding.Spec.Resource, binding.Status.AggregatedStatus) default: // Unsupported resource type. @@ -925,7 +925,7 @@ func (d *ResourceDetector) ReconcileClusterResourceBinding(key util.QueueKey) er klog.Infof("Reconciling cluster resource binding(%s)", binding.Name) switch binding.Spec.Resource.Kind { - case helper.DeploymentKind: + case util.DeploymentKind: return d.AggregateDeploymentStatus(binding.Spec.Resource, binding.Status.AggregatedStatus) default: // Unsupported resource type. @@ -992,7 +992,7 @@ func (d *ResourceDetector) AggregateDeploymentStatus(objRef workv1alpha1.ObjectR // Note: Only limited resource type supported. func (d *ResourceDetector) CleanupResourceTemplateStatus(objRef workv1alpha1.ObjectReference) error { switch objRef.Kind { - case helper.DeploymentKind: + case util.DeploymentKind: return d.CleanupDeploymentStatus(objRef) } diff --git a/pkg/util/helper/binding.go b/pkg/util/helper/binding.go index 2cb01a2b6a2e..f88895ec443a 100644 --- a/pkg/util/helper/binding.go +++ b/pkg/util/helper/binding.go @@ -30,8 +30,6 @@ var resourceBindingKind = v1alpha1.SchemeGroupVersion.WithKind("ResourceBinding" var clusterResourceBindingKind = v1alpha1.SchemeGroupVersion.WithKind("ClusterResourceBinding") const ( - // DeploymentKind indicates the target resource is a deployment - DeploymentKind = "Deployment" // SpecField indicates the 'spec' field of a deployment SpecField = "spec" // ReplicasField indicates the 'replicas' field of a deployment @@ -196,7 +194,7 @@ func EnsureWork(c client.Client, workload *unstructured.Unstructured, clusterNam workLabel[util.ClusterResourceBindingLabel] = binding.GetName() } - if clonedWorkload.GetKind() == DeploymentKind && referenceRSP != nil { + if clonedWorkload.GetKind() == util.DeploymentKind && referenceRSP != nil { err = applyReplicaSchedulingPolicy(clonedWorkload, desireReplicaInfos[clusterName]) if err != nil { klog.Errorf("failed to apply ReplicaSchedulingPolicy for %s/%s/%s in cluster %s, err is: %v", @@ -290,7 +288,7 @@ func calculateReplicasIfNeeded(c client.Client, workload *unstructured.Unstructu var referenceRSP *v1alpha1.ReplicaSchedulingPolicy var desireReplicaInfos = make(map[string]int64) - if workload.GetKind() == DeploymentKind { + if workload.GetKind() == util.DeploymentKind { referenceRSP, err = matchReplicaSchedulingPolicy(c, workload) if err != nil { return nil, nil, err diff --git a/pkg/util/objectwatcher/retain.go b/pkg/util/objectwatcher/retain.go index 180f6e5d99f4..049066976eac 100644 --- a/pkg/util/objectwatcher/retain.go +++ b/pkg/util/objectwatcher/retain.go @@ -4,6 +4,8 @@ import ( "fmt" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/karmada-io/karmada/pkg/util" ) /* @@ -13,12 +15,6 @@ For reference: https://github.com/kubernetes-sigs/kubefed/blob/master/pkg/contro */ const ( - // ServiceKind indicates the target resource is a service - ServiceKind = "Service" - // PodKind indicates the target resource is a pod - PodKind = "Pod" - // ServiceAccountKind indicates the target resource is a serviceaccount - ServiceAccountKind = "ServiceAccount" // SecretsField indicates the 'secrets' field of a service account SecretsField = "secrets" ) @@ -36,13 +32,13 @@ func RetainClusterFields(desiredObj, clusterObj *unstructured.Unstructured) erro desiredObj.SetFinalizers(clusterObj.GetFinalizers()) desiredObj.SetAnnotations(clusterObj.GetAnnotations()) - if targetKind == PodKind { + if targetKind == util.PodKind { return retainPodFields(desiredObj, clusterObj) } - if targetKind == ServiceKind { + if targetKind == util.ServiceKind { return retainServiceFields(desiredObj, clusterObj) } - if targetKind == ServiceAccountKind { + if targetKind == util.ServiceAccountKind { return retainServiceAccountFields(desiredObj, clusterObj) } return nil diff --git a/pkg/util/overridemanager/imageoverridemanager.go b/pkg/util/overridemanager/imageoverridemanager.go new file mode 100644 index 000000000000..9e03bc21384a --- /dev/null +++ b/pkg/util/overridemanager/imageoverridemanager.go @@ -0,0 +1,141 @@ +package overridemanager + +import ( + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1" + "github.com/karmada-io/karmada/pkg/util" + "github.com/karmada-io/karmada/pkg/util/imageparser" +) + +const pathSplit = "/" + +func parseJSONPatchesByImageOverriders(rawObj *unstructured.Unstructured, imageOverriders []policyv1alpha1.ImageOverrider) ([]overrideOption, error) { + imagePatches := make([]overrideOption, 0) + for index := range imageOverriders { + patches, err := parseJSONPatchesByImageOverrider(rawObj, &imageOverriders[index]) + if err != nil { + return nil, err + } + + imagePatches = append(imagePatches, patches...) + } + return imagePatches, nil +} + +func parseJSONPatchesByImageOverrider(rawObj *unstructured.Unstructured, imageOverrider *policyv1alpha1.ImageOverrider) ([]overrideOption, error) { + if imageOverrider.Predicate == nil { + return parseJSONPatchesWithEmptyPredicate(rawObj, imageOverrider) + } + + return parseJSONPatchesWithPredicate(rawObj, imageOverrider) +} + +func parseJSONPatchesWithEmptyPredicate(rawObj *unstructured.Unstructured, imageOverrider *policyv1alpha1.ImageOverrider) ([]overrideOption, error) { + switch rawObj.GetKind() { + case util.PodKind: + return parseJSONPatchesWithSpecContainersPath("spec/containers", rawObj, imageOverrider) + case util.ReplicaSetKind: + return parseJSONPatchesWithSpecContainersPath("spec/template/spec/containers", rawObj, imageOverrider) + case util.DeploymentKind: + return parseJSONPatchesWithSpecContainersPath("spec/template/spec/containers", rawObj, imageOverrider) + case util.StatefulSetKind: + return parseJSONPatchesWithSpecContainersPath("spec/template/spec/containers", rawObj, imageOverrider) + } + + return nil, nil +} + +func parseJSONPatchesWithSpecContainersPath(specContainersPath string, rawObj *unstructured.Unstructured, imageOverrider *policyv1alpha1.ImageOverrider) ([]overrideOption, error) { + patches := make([]overrideOption, 0) + + containers, ok, err := unstructured.NestedSlice(rawObj.Object, strings.Split(specContainersPath, pathSplit)...) + if err != nil { + return nil, fmt.Errorf("failed to retrieves path(%s) from rawObj, error: %v", specContainersPath, err) + } + if !ok || len(containers) == 0 { + return nil, nil + } + + for index := range containers { + patch, err := parseJSONPatchWithPath(fmt.Sprintf("%s/%d/image", specContainersPath, index), rawObj, imageOverrider) + if err != nil { + return nil, err + } + patches = append(patches, patch) + } + + return patches, nil +} + +func parseJSONPatchesWithPredicate(rawObj *unstructured.Unstructured, imageOverrider *policyv1alpha1.ImageOverrider) ([]overrideOption, error) { + patches := make([]overrideOption, 0) + patch, err := parseJSONPatchWithPath(imageOverrider.Predicate.Path, rawObj, imageOverrider) + if err != nil { + return nil, err + } + + patches = append(patches, patch) + return patches, nil +} + +func parseJSONPatchWithPath(imagePath string, rawObj *unstructured.Unstructured, imageOverrider *policyv1alpha1.ImageOverrider) (overrideOption, error) { + imageValue, ok, err := unstructured.NestedString(rawObj.Object, strings.Split(imagePath, pathSplit)...) + if err != nil { + return overrideOption{}, fmt.Errorf("failed to retrieves path(%s) from rawObj, errpr: %v", imagePath, err) + } + if !ok { + return overrideOption{}, nil + } + + imageComponent, err := imageparser.Parse(imageValue) + if err != nil { + return overrideOption{}, fmt.Errorf("failed to parse image value with path(%s), error: %v", imagePath, err) + } + + return overrideOption{ + Op: string(policyv1alpha1.OverriderOpReplace), + Path: imageOverrider.Predicate.Path, + Value: parsedUpdatedImageValue(imageComponent, imageOverrider), + }, nil +} + +func parsedUpdatedImageValue(imageComponent *imageparser.Components, imageOverrider *policyv1alpha1.ImageOverrider) string { + switch imageOverrider.Component { + case policyv1alpha1.Registry: + switch imageOverrider.Operator { + case policyv1alpha1.OverriderOpAdd: + imageComponent.SetHostname(imageComponent.Hostname() + imageOverrider.Value) + case policyv1alpha1.OverriderOpReplace: + imageComponent.SetHostname(imageOverrider.Value) + case policyv1alpha1.OverriderOpRemove: + imageComponent.RemoveHostname() + } + return imageComponent.String() + case policyv1alpha1.Repository: + switch imageOverrider.Operator { + case policyv1alpha1.OverriderOpAdd: + imageComponent.SetRepository(imageComponent.Repository() + imageOverrider.Value) + case policyv1alpha1.OverriderOpReplace: + imageComponent.SetRepository(imageOverrider.Value) + case policyv1alpha1.OverriderOpRemove: + imageComponent.RemoveRepository() + } + return imageComponent.String() + case policyv1alpha1.Tag: + switch imageOverrider.Operator { + case policyv1alpha1.OverriderOpAdd: + imageComponent.SetTagOrDigest(imageComponent.TagOrDigest() + imageOverrider.Value) + case policyv1alpha1.OverriderOpReplace: + imageComponent.SetTagOrDigest(imageOverrider.Value) + case policyv1alpha1.OverriderOpRemove: + imageComponent.RemoveTagOrDigest() + } + return imageComponent.String() + } + + return "" +} diff --git a/pkg/util/overridemanager/overridemanager.go b/pkg/util/overridemanager/overridemanager.go index 32eb8b1ce987..6d09bb6dd762 100644 --- a/pkg/util/overridemanager/overridemanager.go +++ b/pkg/util/overridemanager/overridemanager.go @@ -96,7 +96,8 @@ func (o *overrideManagerImpl) applyClusterOverrides(rawObj *unstructured.Unstruc appliedList := &AppliedOverrides{} for _, p := range matchingPolicies { - if err := applyJSONPatch(rawObj, parseJSONPatch(p.Spec.Overriders.Plaintext)); err != nil { + if err := applyPolicyOverriders(rawObj, p.Spec.Overriders); err != nil { + klog.Errorf("Failed to apply cluster overrides(%s) for resource(%s/%s), error: %v", p.Name, rawObj.GetNamespace(), rawObj.GetName(), err) return nil, err } klog.V(2).Infof("Applied cluster overrides(%s) for %s/%s", p.Name, rawObj.GetNamespace(), rawObj.GetName()) @@ -121,16 +122,17 @@ func (o *overrideManagerImpl) applyNamespacedOverrides(rawObj *unstructured.Unst matchingPolicies := o.getMatchingOverridePolicies(policyList.Items, rawObj, cluster) if len(matchingPolicies) == 0 { - klog.V(2).Infof("No override policy for resource: %s/%s", rawObj.GetNamespace(), rawObj.GetName()) + klog.V(2).Infof("No override policy for resource(%s/%s)", rawObj.GetNamespace(), rawObj.GetName()) return nil, nil } appliedList := &AppliedOverrides{} for _, p := range matchingPolicies { - if err := applyJSONPatch(rawObj, parseJSONPatch(p.Spec.Overriders.Plaintext)); err != nil { + if err := applyPolicyOverriders(rawObj, p.Spec.Overriders); err != nil { + klog.Errorf("Failed to apply overrides(%s/%s) for resource(%s/%s), error: %v", p.Namespace, p.Name, rawObj.GetNamespace(), rawObj.GetName(), err) return nil, err } - klog.V(2).Infof("Applied overrides(%s/%s) for %s/%s", p.Namespace, p.Name, rawObj.GetNamespace(), rawObj.GetName()) + klog.V(2).Infof("Applied overrides(%s/%s) for resource(%s/%s)", p.Namespace, p.Name, rawObj.GetNamespace(), rawObj.GetName()) appliedList.Add(p.Name, p.Spec.Overriders) } @@ -207,7 +209,21 @@ func (o *overrideManagerImpl) getMatchingOverridePolicies(policies []policyv1alp return clusterMatchingPolicies } -func parseJSONPatch(overriders []policyv1alpha1.PlaintextOverrider) []overrideOption { +// applyPolicyOverriders applies OverridePolicy/ClusterOverridePolicy overriders to target object +func applyPolicyOverriders(rawObj *unstructured.Unstructured, overriders policyv1alpha1.Overriders) error { + patches, err := parseJSONPatchesByImageOverriders(rawObj, overriders.ImageOverrider) + if err != nil { + return err + } + + if err = applyJSONPatch(rawObj, patches); err != nil { + return err + } + + return applyJSONPatch(rawObj, parseJSONPatchesByPlaintext(overriders.Plaintext)) +} + +func parseJSONPatchesByPlaintext(overriders []policyv1alpha1.PlaintextOverrider) []overrideOption { patches := make([]overrideOption, 0, len(overriders)) for i := range overriders { patches = append(patches, overrideOption{