-
Notifications
You must be signed in to change notification settings - Fork 4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ScaleUp for check-capacity ProvisioningRequestClass
- Loading branch information
1 parent
13c5875
commit cba20c1
Showing
13 changed files
with
992 additions
and
38 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
cluster-autoscaler/core/scaleup/orchestrator/wrapper_orchestrator.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| /* | ||
| Copyright 2024 The Kubernetes Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package orchestrator | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| appsv1 "k8s.io/api/apps/v1" | ||
| apiv1 "k8s.io/api/core/v1" | ||
| "k8s.io/autoscaler/cluster-autoscaler/clusterstate" | ||
| "k8s.io/autoscaler/cluster-autoscaler/context" | ||
| "k8s.io/autoscaler/cluster-autoscaler/core/scaleup" | ||
| ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" | ||
| "k8s.io/autoscaler/cluster-autoscaler/processors/status" | ||
| "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/checkcapacity" | ||
| "k8s.io/autoscaler/cluster-autoscaler/utils/errors" | ||
| "k8s.io/autoscaler/cluster-autoscaler/utils/taints" | ||
| "k8s.io/client-go/rest" | ||
| schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" | ||
| ) | ||
|
|
||
| const ( | ||
| consumeProvReq = "cluster-autoscaler.kubernetes.io/consume-provisioning-request" | ||
| ) | ||
|
|
||
| // WrapperOrchestrator is an orchestrator which wrap scale up for ProvisioningRequests and regular pods. | ||
| type WrapperOrchestrator struct { | ||
| scaleUpRegularPods bool | ||
| scaleUpOrchestrator scaleup.Orchestrator | ||
| provReqOrchestrator scaleup.Orchestrator | ||
| } | ||
|
|
||
| // NewWrapperOrchestrator return WrapperOrchestrator | ||
| func NewWrapperOrchestrator(kubeConfig *rest.Config) (scaleup.Orchestrator, error) { | ||
| provReqOrchestrator, err := checkcapacity.New(kubeConfig) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed create ScaleUp orchestrator for ProvisioningRequests, error: %v", err) | ||
| } | ||
| return &WrapperOrchestrator{ | ||
| scaleUpOrchestrator: New(), | ||
| provReqOrchestrator: provReqOrchestrator, | ||
| }, nil | ||
| } | ||
|
|
||
| // Initialize initializes the orchestrator object with required fields. | ||
| func (o *WrapperOrchestrator) Initialize( | ||
| autoscalingContext *context.AutoscalingContext, | ||
| processors *ca_processors.AutoscalingProcessors, | ||
| clusterStateRegistry *clusterstate.ClusterStateRegistry, | ||
| taintConfig taints.TaintConfig, | ||
| ) { | ||
| o.scaleUpOrchestrator.Initialize(autoscalingContext, processors, clusterStateRegistry, taintConfig) | ||
| o.provReqOrchestrator.Initialize(autoscalingContext, processors, clusterStateRegistry, taintConfig) | ||
| } | ||
|
|
||
| // ScaleUp run scaleUp function for regular pods of pods from ProvisioningRequest. | ||
| func (o *WrapperOrchestrator) ScaleUp( | ||
| unschedulablePods []*apiv1.Pod, | ||
| nodes []*apiv1.Node, | ||
| daemonSets []*appsv1.DaemonSet, | ||
| nodeInfos map[string]*schedulerframework.NodeInfo, | ||
| ) (*status.ScaleUpStatus, errors.AutoscalerError) { | ||
| provReqPods, regularPods := sortOut(unschedulablePods) | ||
| if len(provReqPods) == 0 { | ||
| return o.scaleUpOrchestrator.ScaleUp(regularPods, nodes, daemonSets, nodeInfos) | ||
| } | ||
| if len(regularPods) == 0 { | ||
| return o.provReqOrchestrator.ScaleUp(provReqPods, nodes, daemonSets, nodeInfos) | ||
| } | ||
| if o.scaleUpRegularPods { | ||
| o.scaleUpRegularPods = false | ||
| return o.scaleUpOrchestrator.ScaleUp(regularPods, nodes, daemonSets, nodeInfos) | ||
| } | ||
| o.scaleUpRegularPods = true | ||
| return o.provReqOrchestrator.ScaleUp(provReqPods, nodes, daemonSets, nodeInfos) | ||
| } | ||
|
|
||
| func sortOut(unschedulablePods []*apiv1.Pod) (provReqPods, regularPods []*apiv1.Pod) { | ||
| for _, pod := range unschedulablePods { | ||
| if _, ok := pod.Annotations[consumeProvReq]; ok { | ||
| provReqPods = append(provReqPods, pod) | ||
| } else { | ||
| regularPods = append(regularPods, pod) | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| // ScaleUpToNodeGroupMinSize tries to scale up node groups that have less nodes | ||
| // than the configured min size. The source of truth for the current node group | ||
| // size is the TargetSize queried directly from cloud providers. Returns | ||
| // appropriate status or error if an unexpected error occurred. | ||
| func (o *WrapperOrchestrator) ScaleUpToNodeGroupMinSize( | ||
| nodes []*apiv1.Node, | ||
| nodeInfos map[string]*schedulerframework.NodeInfo, | ||
| ) (*status.ScaleUpStatus, errors.AutoscalerError) { | ||
| return o.scaleUpOrchestrator.ScaleUpToNodeGroupMinSize(nodes, nodeInfos) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 77 additions & 0 deletions
77
cluster-autoscaler/provisioningrequest/checkcapacity/condition.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| /* | ||
| Copyright 2024 The Kubernetes Authors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package checkcapacity | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| v1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/autoscaler/cluster-autoscaler/provisioningrequest/provreqwrapper" | ||
| ) | ||
|
|
||
| // ProvisioningRequestCondition is a type of Condition that ClusterAutoscaler appends to ProvisioningRequest. | ||
| type ProvisioningRequestCondition string | ||
|
|
||
| const ( | ||
| // BookCapacityCondition is appended if capacity for ProvisioningRequest was found in the cluster. | ||
| BookCapacityCondition = ProvisioningRequestCondition("BookCapacity") | ||
|
|
||
| // ExpiredCondition is append if the ProvisioningRequest has BookCapacity condition before | ||
| // and the reservation time is expired or the ProvisioningRequest has Pending condition before | ||
| // and expiration time is expired. | ||
| ExpiredCondition = ProvisioningRequestCondition("Expired") | ||
|
|
||
| // PendingCondition is append if no capacity for ProvisioningRequest was found in the cluster | ||
| // and ClusterAutoscaler will try to find capacity later. | ||
| PendingCondition = ProvisioningRequestCondition("Pending") | ||
|
|
||
| // RejectedCondition is append if ProvisioningRequest is invalid. | ||
| RejectedCondition = ProvisioningRequestCondition("Rejected") | ||
|
|
||
| // CheckCapacityClass is a name of ProvisioningRequestClass | ||
| CheckCapacityClass = "check-capacity.kubernetes.io" | ||
| defaultReservationTime = 10 * time.Minute | ||
| defaultExpirationTime = 7 * 24 * time.Hour // 7 days | ||
| ) | ||
|
|
||
| // HasBookCapacityCondition return if PR has BookCapacity condition | ||
| func HasBookCapacityCondition(pr *provreqwrapper.ProvisioningRequest) bool { | ||
| if pr.V1Beta1().Spec.ProvisioningClassName != CheckCapacityClass { | ||
| return false | ||
| } | ||
| if pr.Conditions() == nil || len(pr.Conditions()) == 0 { | ||
| return false | ||
| } | ||
| condition := pr.Conditions()[len(pr.Conditions())-1] | ||
| if condition.Type == string(BookCapacityCondition) && condition.Status == v1.ConditionTrue { | ||
| return true | ||
| } | ||
| return false | ||
| } | ||
|
|
||
| func setCondition(pr *provreqwrapper.ProvisioningRequest, conditionType ProvisioningRequestCondition, reason, message string) { | ||
| conditions := pr.Conditions() | ||
| conditions = append(conditions, v1.Condition{ | ||
| Type: string(conditionType), | ||
| Status: v1.ConditionTrue, | ||
| ObservedGeneration: pr.V1Beta1().GetObjectMeta().GetGeneration(), | ||
| LastTransitionTime: v1.Now(), | ||
| Reason: reason, | ||
| Message: message, | ||
| }) | ||
| pr.SetConditions(conditions) | ||
| } |
Oops, something went wrong.