diff --git a/apis/kueue/v1beta1/clusterqueue_types.go b/apis/kueue/v1beta1/clusterqueue_types.go index dced114ca6..cfd8aa2b7d 100644 --- a/apis/kueue/v1beta1/clusterqueue_types.go +++ b/apis/kueue/v1beta1/clusterqueue_types.go @@ -243,6 +243,21 @@ const ( PreemptionPolicyLowerOrNewerEqualPriority PreemptionPolicy = "LowerOrNewerEqualPriority" ) +type FlavorFungibilityPolicy string + +const ( + Borrow FlavorFungibilityPolicy = "Borrow" + Preempt FlavorFungibilityPolicy = "Preempt" + TryNextFlavor FlavorFungibilityPolicy = "TryNextFlavor" +) + +type FlavorFungibility struct { + // +kubebuilder:validation:Enum="Borrow,TryNextFlavor" + WhenCanBorrow FlavorFungibilityPolicy `json:"whenCanBorrow"` + // +kubebuilder:validation:Enum="Preempt,TryNextFlavor" + WhenCanPreempt FlavorFungibilityPolicy `json:"whenCanPreempt"` +} + // ClusterQueuePreemption contains policies to preempt Workloads from this // ClusterQueue or the ClusterQueue's cohort. type ClusterQueuePreemption struct { @@ -276,6 +291,8 @@ type ClusterQueuePreemption struct { // +kubebuilder:default=Never // +kubebuilder:validation:Enum=Never;LowerPriority;LowerOrNewerEqualPriority WithinClusterQueue PreemptionPolicy `json:"withinClusterQueue,omitempty"` + + FlavorFungibility FlavorFungibility `json:"flavorFungibility"` } //+kubebuilder:object:root=true diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 97d45a56d9..860172ea15 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -31,6 +31,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utilindexer "sigs.k8s.io/kueue/pkg/controller/core/indexer" "sigs.k8s.io/kueue/pkg/metrics" @@ -141,6 +142,8 @@ type ClusterQueue struct { Preemption kueue.ClusterQueuePreemption Status metrics.ClusterQueueStatus + FlavorFungibility v1beta1.FlavorFungibility + // The following fields are not populated in a snapshot. // Key is localQueue's key (namespace/name). diff --git a/pkg/scheduler/flavorassigner/flavorassigner.go b/pkg/scheduler/flavorassigner/flavorassigner.go index 50ea8a50a8..30e084f556 100644 --- a/pkg/scheduler/flavorassigner/flavorassigner.go +++ b/pkg/scheduler/flavorassigner/flavorassigner.go @@ -32,6 +32,7 @@ import ( "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "k8s.io/utils/pointer" + "sigs.k8s.io/kueue/apis/kueue/v1beta1" kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" "sigs.k8s.io/kueue/pkg/cache" "sigs.k8s.io/kueue/pkg/workload" @@ -339,6 +340,9 @@ func (a *Assignment) findFlavorForResourceGroup( var bestAssignment ResourceAssignment bestAssignmentMode := NoFit + policyBorrow := cq.FlavorFungibility.WhenCanBorrow + policyPreempt := cq.FlavorFungibility.WhenCanPreempt + // We will only check against the flavors' labels for the resource. selector := flavorSelector(spec, rg.LabelKeys) for _, flvQuotas := range rg.Flavors { @@ -364,6 +368,7 @@ func (a *Assignment) findFlavorForResourceGroup( continue } + whetherNeedBorrowing := false assignments := make(ResourceAssignment, len(requests)) // Calculate representativeMode for this assignment as the worst mode among all requests. representativeMode := Fit @@ -376,6 +381,7 @@ func (a *Assignment) findFlavorForResourceGroup( } if mode < representativeMode { representativeMode = mode + whetherNeedBorrowing = whetherNeedBorrowing || borrow > 0 } if representativeMode == NoFit { // The flavor doesn't fit, no need to check other resources. @@ -389,13 +395,25 @@ func (a *Assignment) findFlavorForResourceGroup( } } + if representativeMode == Preempt && policyPreempt == v1beta1.Preempt { + return assignments, nil + } + + if representativeMode == Fit && whetherNeedBorrowing && policyBorrow == v1beta1.Borrow { + return assignments, nil + } + + if representativeMode == Fit && !whetherNeedBorrowing { + return assignments, nil + } + if representativeMode > bestAssignmentMode { bestAssignment = assignments bestAssignmentMode = representativeMode - if bestAssignmentMode == Fit { - // All the resources fit in the cohort, no need to check more flavors. - return bestAssignment, nil - } + // if bestAssignmentMode == Fit { + // // All the resources fit in the cohort, no need to check more flavors. + // return bestAssignment, nil + // } } } return bestAssignment, status