From 987b99e884b4266988b30ebaeea0c8d0440749fd Mon Sep 17 00:00:00 2001 From: jiangkaihua Date: Tue, 6 Dec 2022 16:36:55 +0800 Subject: [PATCH 1/4] Update queue API. Signed-off-by: jiangkaihua --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- vendor/volcano.sh/apis/pkg/apis/scheduling/types.go | 5 ++++- .../volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go | 3 +++ .../volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go | 7 +++++-- .../pkg/apis/scheduling/v1beta1/zz_generated.conversion.go | 2 ++ .../pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go | 7 +++++++ .../apis/pkg/apis/scheduling/zz_generated.deepcopy.go | 7 +++++++ 9 files changed, 32 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index aa9a26fb26..e3324357f7 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed sigs.k8s.io/yaml v1.3.0 stathat.com/c/consistent v1.0.0 - volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c + volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d ) require ( diff --git a/go.sum b/go.sum index 7604a197e2..ee977493c5 100644 --- a/go.sum +++ b/go.sum @@ -827,5 +827,5 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= -volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c h1:s7K9ouTNVr6OWyH9AX4aG4crc7XvuT8Dcfeaqt6gzkg= -volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c/go.mod h1:xe38GChdXXam/g/FkQXIsR0vhqp4twoZdY2gaGkEP24= +volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d h1:3ThPUKyWZxJTOHBRHv1VGDzaLGl+Bb/g362xNG749zE= +volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d/go.mod h1:xe38GChdXXam/g/FkQXIsR0vhqp4twoZdY2gaGkEP24= diff --git a/vendor/modules.txt b/vendor/modules.txt index b7c5b3a4d6..8dfd1858eb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -970,7 +970,7 @@ sigs.k8s.io/yaml # stathat.com/c/consistent v1.0.0 ## explicit stathat.com/c/consistent -# volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c +# volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d ## explicit; go 1.19 volcano.sh/apis/pkg/apis/batch/v1alpha1 volcano.sh/apis/pkg/apis/bus/v1alpha1 diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go index 508cc33721..a7435e260b 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/types.go @@ -285,7 +285,10 @@ type QueueStatus struct { Completed int32 // Reservation is the profile of resource reservation for queue - Reservation Reservation `json:"reservation,omitempty" protobuf:"bytes,6,opt,name=reservation"` + Reservation Reservation + + // Allocated is allocated resources in queue + Allocated v1.ResourceList } // CluterSpec represents the template of Cluster diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go index eb923a33dc..b1961b224e 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/labels.go @@ -64,3 +64,6 @@ const NumaPolicyKey = "volcano.sh/numa-topology-policy" // TopologyDecisionAnnotation is the key of topology decision about pod request resource const TopologyDecisionAnnotation = "volcano.sh/topology-decision" + +// PodQosLevel is the key of pod qos level +const PodQosLevel = "volcano.sh/qos-level" diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go index 82abba681d..2e95e18a9c 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/types.go @@ -298,10 +298,13 @@ type QueueStatus struct { // The number of `Inqueue` PodGroup in this queue. Inqueue int32 `json:"inqueue,omitempty" protobuf:"bytes,5,opt,name=inqueue"` // The number of `Completed` PodGroup in this queue. - Completed int32 `json:"completed,omitempty" protobuf:"bytes,7,opt,name=completed"` + Completed int32 `json:"completed,omitempty" protobuf:"bytes,6,opt,name=completed"` // Reservation is the profile of resource reservation for queue - Reservation Reservation `json:"reservation,omitempty" protobuf:"bytes,6,opt,name=reservation"` + Reservation Reservation `json:"reservation,omitempty" protobuf:"bytes,7,opt,name=reservation"` + + // Allocated is allocated resources in queue + Allocated v1.ResourceList `json:"allocated" protobuf:"bytes,8,opt,name=allocated"` } // CluterSpec represents the template of Cluster diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go index 7562fe6cd0..30625cef48 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.conversion.go @@ -555,6 +555,7 @@ func autoConvert_v1beta1_QueueStatus_To_scheduling_QueueStatus(in *QueueStatus, if err := Convert_v1beta1_Reservation_To_scheduling_Reservation(&in.Reservation, &out.Reservation, s); err != nil { return err } + out.Allocated = *(*v1.ResourceList)(unsafe.Pointer(&in.Allocated)) return nil } @@ -573,6 +574,7 @@ func autoConvert_scheduling_QueueStatus_To_v1beta1_QueueStatus(in *scheduling.Qu if err := Convert_scheduling_Reservation_To_v1beta1_Reservation(&in.Reservation, &out.Reservation, s); err != nil { return err } + out.Allocated = *(*v1.ResourceList)(unsafe.Pointer(&in.Allocated)) return nil } diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go index 8a524df88d..56ffe201b1 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/v1beta1/zz_generated.deepcopy.go @@ -391,6 +391,13 @@ func (in *QueueSpec) DeepCopy() *QueueSpec { func (in *QueueStatus) DeepCopyInto(out *QueueStatus) { *out = *in in.Reservation.DeepCopyInto(&out.Reservation) + if in.Allocated != nil { + in, out := &in.Allocated, &out.Allocated + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } return } diff --git a/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go b/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go index 8d3e48ee06..773fa329cf 100644 --- a/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go +++ b/vendor/volcano.sh/apis/pkg/apis/scheduling/zz_generated.deepcopy.go @@ -391,6 +391,13 @@ func (in *QueueSpec) DeepCopy() *QueueSpec { func (in *QueueStatus) DeepCopyInto(out *QueueStatus) { *out = *in in.Reservation.DeepCopyInto(&out.Reservation) + if in.Allocated != nil { + in, out := &in.Allocated, &out.Allocated + *out = make(v1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } return } From 411e4af14d3369dfeb17a324317c957ac6ff4932 Mon Sep 17 00:00:00 2001 From: jiangkaihua Date: Tue, 15 Nov 2022 16:31:29 +0800 Subject: [PATCH 2/4] Update yamls. Signed-off-by: jiangkaihua --- config/crd/bases/scheduling.volcano.sh_queues.yaml | 11 +++++++++++ .../crd/v1beta1/scheduling.volcano.sh_queues.yaml | 11 +++++++++++ .../crd/bases/scheduling.volcano.sh_queues.yaml | 11 +++++++++++ .../crd/v1beta1/scheduling.volcano.sh_queues.yaml | 11 +++++++++++ .../helm/chart/volcano/templates/scheduler.yaml | 5 ++++- installer/volcano-development.yaml | 14 ++++++++++++++ 6 files changed, 62 insertions(+), 1 deletion(-) diff --git a/config/crd/bases/scheduling.volcano.sh_queues.yaml b/config/crd/bases/scheduling.volcano.sh_queues.yaml index 7a1f2474f9..bc932bf1b4 100644 --- a/config/crd/bases/scheduling.volcano.sh_queues.yaml +++ b/config/crd/bases/scheduling.volcano.sh_queues.yaml @@ -133,6 +133,15 @@ spec: status: description: The status of queue. properties: + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocated is allocated resources in queue + type: object completed: description: The number of `Completed` PodGroup in this queue. format: int32 @@ -176,6 +185,8 @@ spec: description: The number of 'Unknown' PodGroup in this queue. format: int32 type: integer + required: + - allocated type: object type: object served: true diff --git a/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml b/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml index edc22cb2e7..18abfe7d11 100644 --- a/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml +++ b/config/crd/v1beta1/scheduling.volcano.sh_queues.yaml @@ -132,6 +132,15 @@ spec: status: description: The status of queue. properties: + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocated is allocated resources in queue + type: object completed: description: The number of `Completed` PodGroup in this queue. format: int32 @@ -175,6 +184,8 @@ spec: description: The number of 'Unknown' PodGroup in this queue. format: int32 type: integer + required: + - allocated type: object type: object version: v1beta1 diff --git a/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml b/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml index 1e21b02103..1f9b14faa6 100644 --- a/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml +++ b/installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml @@ -131,6 +131,15 @@ spec: status: description: The status of queue. properties: + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocated is allocated resources in queue + type: object completed: description: The number of `Completed` PodGroup in this queue. format: int32 @@ -174,6 +183,8 @@ spec: description: The number of 'Unknown' PodGroup in this queue. format: int32 type: integer + required: + - allocated type: object type: object served: true diff --git a/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml b/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml index b07530f1c3..888ea125b8 100644 --- a/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml +++ b/installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml @@ -130,6 +130,15 @@ spec: status: description: The status of queue. properties: + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocated is allocated resources in queue + type: object completed: description: The number of `Completed` PodGroup in this queue. format: int32 @@ -173,6 +182,8 @@ spec: description: The number of 'Unknown' PodGroup in this queue. format: int32 type: integer + required: + - allocated type: object type: object version: v1beta1 diff --git a/installer/helm/chart/volcano/templates/scheduler.yaml b/installer/helm/chart/volcano/templates/scheduler.yaml index 15b0c2929e..e2b713c670 100644 --- a/installer/helm/chart/volcano/templates/scheduler.yaml +++ b/installer/helm/chart/volcano/templates/scheduler.yaml @@ -63,6 +63,9 @@ rules: - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] resources: ["queues"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["queues/status"] + verbs: ["update"] - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] resources: ["podgroups"] verbs: ["list", "watch", "update"] @@ -150,4 +153,4 @@ spec: selector: app: volcano-scheduler type: ClusterIP -{{- end }} \ No newline at end of file +{{- end }} diff --git a/installer/volcano-development.yaml b/installer/volcano-development.yaml index ce0cf2071b..e658891855 100644 --- a/installer/volcano-development.yaml +++ b/installer/volcano-development.yaml @@ -8690,6 +8690,9 @@ rules: - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] resources: ["queues"] verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] + resources: ["queues/status"] + verbs: ["update"] - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"] resources: ["podgroups"] verbs: ["list", "watch", "update"] @@ -9067,6 +9070,15 @@ spec: status: description: The status of queue. properties: + allocated: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocated is allocated resources in queue + type: object completed: description: The number of `Completed` PodGroup in this queue. format: int32 @@ -9110,6 +9122,8 @@ spec: description: The number of 'Unknown' PodGroup in this queue. format: int32 type: integer + required: + - allocated type: object type: object served: true From 76178526df636282286bcecadb3c62ed357bdcad Mon Sep 17 00:00:00 2001 From: jiangkaihua Date: Tue, 6 Dec 2022 23:08:27 +0800 Subject: [PATCH 3/4] Update queue status process. Signed-off-by: jiangkaihua --- .../queue/queue_controller_action.go | 2 ++ pkg/scheduler/cache/cache.go | 16 +++++++++ pkg/scheduler/cache/interface.go | 21 ++++++----- pkg/scheduler/framework/session.go | 35 +++++++++++++++++++ pkg/scheduler/util/scheduler_helper.go | 20 +++++++++-- 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/pkg/controllers/queue/queue_controller_action.go b/pkg/controllers/queue/queue_controller_action.go index f1b4bb1061..c9ad8ac5d2 100644 --- a/pkg/controllers/queue/queue_controller_action.go +++ b/pkg/controllers/queue/queue_controller_action.go @@ -66,6 +66,8 @@ func (c *queuecontroller) syncQueue(queue *schedulingv1beta1.Queue, updateStateF queueStatus.State = queue.Status.State } + queueStatus.Allocated = queue.Status.Allocated.DeepCopy() + // ignore update when status does not change if reflect.DeepEqual(queueStatus, queue.Status) { return nil diff --git a/pkg/scheduler/cache/cache.go b/pkg/scheduler/cache/cache.go index 594f7d0575..2f8b395c2c 100644 --- a/pkg/scheduler/cache/cache.go +++ b/pkg/scheduler/cache/cache.go @@ -1218,6 +1218,22 @@ func (sc *SchedulerCache) UpdateJobStatus(job *schedulingapi.JobInfo, updatePG b return job, nil } +// UpdateQueueStatus update the status of queue. +func (sc *SchedulerCache) UpdateQueueStatus(queue *schedulingapi.QueueInfo) error { + var newQueue = &vcv1beta1.Queue{} + if err := schedulingscheme.Scheme.Convert(queue.Queue, newQueue, nil); err != nil { + klog.Errorf("error occurred in converting scheduling.Queue to v1beta1.Queue: %s", err.Error()) + return err + } + + _, err := sc.vcClient.SchedulingV1beta1().Queues().UpdateStatus(context.TODO(), newQueue, metav1.UpdateOptions{}) + if err != nil { + klog.Errorf("error occurred in updating Queue <%s>: %s", newQueue.Name, err.Error()) + return err + } + return nil +} + func (sc *SchedulerCache) recordPodGroupEvent(podGroup *schedulingapi.PodGroup, eventType, reason, msg string) { if podGroup == nil { return diff --git a/pkg/scheduler/cache/interface.go b/pkg/scheduler/cache/interface.go index 99bcd0a23c..7b46c48471 100644 --- a/pkg/scheduler/cache/interface.go +++ b/pkg/scheduler/cache/interface.go @@ -22,9 +22,9 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" - scheduling "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/pkg/scheduler/capabilities/volumebinding" ) // Cache collects pods/nodes/queues information @@ -56,17 +56,20 @@ type Cache interface { // UpdateJobStatus puts job in backlog for a while. UpdateJobStatus(job *api.JobInfo, updatePG bool) (*api.JobInfo, error) + // UpdateQueueStatus update queue status. + UpdateQueueStatus(queue *api.QueueInfo) error + // GetPodVolumes get pod volume on the host - GetPodVolumes(task *api.TaskInfo, node *v1.Node) (*scheduling.PodVolumes, error) + GetPodVolumes(task *api.TaskInfo, node *v1.Node) (*volumebinding.PodVolumes, error) // AllocateVolumes allocates volume on the host to the task - AllocateVolumes(task *api.TaskInfo, hostname string, podVolumes *scheduling.PodVolumes) error + AllocateVolumes(task *api.TaskInfo, hostname string, podVolumes *volumebinding.PodVolumes) error // BindVolumes binds volumes to the task - BindVolumes(task *api.TaskInfo, volumes *scheduling.PodVolumes) error + BindVolumes(task *api.TaskInfo, volumes *volumebinding.PodVolumes) error // RevertVolumes clean cache generated by AllocateVolumes - RevertVolumes(task *api.TaskInfo, podVolumes *scheduling.PodVolumes) + RevertVolumes(task *api.TaskInfo, podVolumes *volumebinding.PodVolumes) // Client returns the kubernetes clientSet, which can be used by plugins Client() kubernetes.Interface @@ -88,10 +91,10 @@ type Cache interface { // VolumeBinder interface for allocate and bind volumes type VolumeBinder interface { - GetPodVolumes(task *api.TaskInfo, node *v1.Node) (*scheduling.PodVolumes, error) - RevertVolumes(task *api.TaskInfo, podVolumes *scheduling.PodVolumes) - AllocateVolumes(task *api.TaskInfo, hostname string, podVolumes *scheduling.PodVolumes) error - BindVolumes(task *api.TaskInfo, podVolumes *scheduling.PodVolumes) error + GetPodVolumes(task *api.TaskInfo, node *v1.Node) (*volumebinding.PodVolumes, error) + RevertVolumes(task *api.TaskInfo, podVolumes *volumebinding.PodVolumes) + AllocateVolumes(task *api.TaskInfo, hostname string, podVolumes *volumebinding.PodVolumes) error + BindVolumes(task *api.TaskInfo, podVolumes *volumebinding.PodVolumes) error } // Binder interface for binding task and hostname diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index 965fd64f5f..410e6ade04 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -18,6 +18,7 @@ package framework import ( "fmt" + "reflect" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,6 +29,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" "k8s.io/klog" + "volcano.sh/apis/pkg/apis/scheduling" schedulingscheme "volcano.sh/apis/pkg/apis/scheduling/scheme" vcv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" @@ -185,10 +187,43 @@ func openSession(cache cache.Cache) *Session { return ssn } +// updateQueueStatus updates allocated field in queue status on session close. +func updateQueueStatus(ssn *Session) { + // calculate allocated resources on each queue + var allocatedResources = make(map[api.QueueID]*api.Resource, len(ssn.Queues)) + for queueID := range ssn.Queues { + allocatedResources[queueID] = &api.Resource{} + } + for _, job := range ssn.Jobs { + for _, runningTask := range job.TaskStatusIndex[api.Running] { + allocatedResources[job.Queue].Add(runningTask.Resreq) + } + } + + // update queue status + for queueID := range ssn.Queues { + // convert api.Resource to v1.ResourceList + var queueStatus = util.ConvertRes2ResList(allocatedResources[queueID]).DeepCopy() + if reflect.DeepEqual(ssn.Queues[queueID].Queue.Status.Allocated, queueStatus) { + klog.V(5).Infof("Queue <%s> allocated resource keeps equal, no need to update queue status <%v>.", + queueID, ssn.Queues[queueID].Queue.Status.Allocated) + continue + } + + ssn.Queues[queueID].Queue.Status.Allocated = queueStatus + + if err := ssn.cache.UpdateQueueStatus(ssn.Queues[queueID]); err != nil { + klog.Errorf("failed to update queue <%s> status: %s", ssn.Queues[queueID].Name, err.Error()) + } + } +} + func closeSession(ssn *Session) { ju := newJobUpdater(ssn) ju.UpdateAll() + updateQueueStatus(ssn) + ssn.Jobs = nil ssn.Nodes = nil ssn.RevocableNodes = nil diff --git a/pkg/scheduler/util/scheduler_helper.go b/pkg/scheduler/util/scheduler_helper.go index b63ddcac68..7d72999aa8 100644 --- a/pkg/scheduler/util/scheduler_helper.go +++ b/pkg/scheduler/util/scheduler_helper.go @@ -19,14 +19,17 @@ package util import ( "context" "fmt" - "k8s.io/client-go/util/workqueue" - "k8s.io/klog" - k8sframework "k8s.io/kubernetes/pkg/scheduler/framework" "math" "math/rand" "sort" "sync" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + k8sframework "k8s.io/kubernetes/pkg/scheduler/framework" + "volcano.sh/volcano/cmd/scheduler/app/options" "volcano.sh/volcano/pkg/scheduler/api" ) @@ -199,3 +202,14 @@ func GetMinInt(vals ...int) int { } return min } + +// ConvertRes2ResList convert resource type from api.Resource in scheduler to v1.ResourceList in yaml +func ConvertRes2ResList(res *api.Resource) v1.ResourceList { + var rl = v1.ResourceList{} + rl[v1.ResourceCPU] = *resource.NewMilliQuantity(int64(res.MilliCPU), resource.DecimalSI) + rl[v1.ResourceMemory] = *resource.NewQuantity(int64(res.Memory), resource.BinarySI) + for resourceName, f := range res.ScalarResources { + rl[resourceName] = *resource.NewMilliQuantity(int64(f), resource.DecimalSI) + } + return rl +} From a1cecb424892f3ad53ae1ea7bbbdca8cdbf27f43 Mon Sep 17 00:00:00 2001 From: jiangkaihua Date: Tue, 6 Dec 2022 23:09:27 +0800 Subject: [PATCH 4/4] Update UT & e2e cases. Signed-off-by: jiangkaihua --- .../actions/allocate/allocate_test.go | 50 +++++++++++-------- pkg/scheduler/actions/preempt/preempt_test.go | 26 ++++++---- pkg/scheduler/actions/reclaim/reclaim_test.go | 29 ++++++----- pkg/scheduler/actions/shuffle/shuffle_test.go | 36 +++++++------ pkg/scheduler/plugins/binpack/binpack_test.go | 24 +++++---- pkg/scheduler/plugins/drf/hdrf_test.go | 23 ++++++--- .../plugins/proportion/proportion_test.go | 36 +++++++------ pkg/scheduler/plugins/tdm/tdm_test.go | 48 +++++++++++------- test/e2e/util/queue.go | 6 +++ 9 files changed, 171 insertions(+), 107 deletions(-) diff --git a/pkg/scheduler/actions/allocate/allocate_test.go b/pkg/scheduler/actions/allocate/allocate_test.go index f9e052729b..5c7549a71a 100644 --- a/pkg/scheduler/actions/allocate/allocate_test.go +++ b/pkg/scheduler/actions/allocate/allocate_test.go @@ -24,21 +24,22 @@ import ( "github.com/agiledragon/gomonkey/v2" v1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" "k8s.io/client-go/tools/record" - "volcano.sh/volcano/pkg/scheduler/plugins/gang" - "volcano.sh/volcano/pkg/scheduler/plugins/priority" - storagev1 "k8s.io/api/storage/v1" schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" "volcano.sh/volcano/pkg/scheduler/plugins/drf" + "volcano.sh/volcano/pkg/scheduler/plugins/gang" + "volcano.sh/volcano/pkg/scheduler/plugins/priority" "volcano.sh/volcano/pkg/scheduler/plugins/proportion" "volcano.sh/volcano/pkg/scheduler/util" ) @@ -245,16 +246,19 @@ func TestAllocate(t *testing.T) { Binds: map[string]string{}, Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - - Recorder: record.NewFakeRecorder(100), + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) for _, node := range test.nodes { schedulerCache.AddNode(node) @@ -417,15 +421,21 @@ func TestAllocateWithDynamicPVC(t *testing.T) { Binds: map[string]string{}, Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: fakeVolumeBinder, - Recorder: record.NewFakeRecorder(100), + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = fakeVolumeBinder + schedulerCache.Recorder = record.NewFakeRecorder(100) + schedulerCache.AddQueueV1beta1(queue) schedulerCache.AddPodGroupV1beta1(pg) for i, pod := range test.pods { diff --git a/pkg/scheduler/actions/preempt/preempt_test.go b/pkg/scheduler/actions/preempt/preempt_test.go index bf6ac40883..d213127f7c 100644 --- a/pkg/scheduler/actions/preempt/preempt_test.go +++ b/pkg/scheduler/actions/preempt/preempt_test.go @@ -27,7 +27,7 @@ import ( schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" "volcano.sh/volcano/cmd/scheduler/app/options" - "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" @@ -287,18 +287,22 @@ func TestPreempt(t *testing.T) { evictor := &util.FakeEvictor{ Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - Evictor: evictor, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - PriorityClasses: make(map[string]*schedulingv1.PriorityClass), - Recorder: record.NewFakeRecorder(100), + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.Evictor = evictor + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + schedulerCache.PriorityClasses["high-priority"] = &schedulingv1.PriorityClass{ Value: 100000, } diff --git a/pkg/scheduler/actions/reclaim/reclaim_test.go b/pkg/scheduler/actions/reclaim/reclaim_test.go index 732e1bacef..25f5024397 100644 --- a/pkg/scheduler/actions/reclaim/reclaim_test.go +++ b/pkg/scheduler/actions/reclaim/reclaim_test.go @@ -26,7 +26,8 @@ import ( "k8s.io/client-go/tools/record" schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" - "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" @@ -121,18 +122,22 @@ func TestReclaim(t *testing.T) { evictor := &util.FakeEvictor{ Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - Evictor: evictor, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - PriorityClasses: make(map[string]*schedulingv1.PriorityClass), - - Recorder: record.NewFakeRecorder(100), + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.Evictor = evictor + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + schedulerCache.PriorityClasses["high-priority"] = &schedulingv1.PriorityClass{ Value: 100000, } diff --git a/pkg/scheduler/actions/shuffle/shuffle_test.go b/pkg/scheduler/actions/shuffle/shuffle_test.go index d244ff320d..befdf4f750 100644 --- a/pkg/scheduler/actions/shuffle/shuffle_test.go +++ b/pkg/scheduler/actions/shuffle/shuffle_test.go @@ -17,21 +17,23 @@ package shuffle import ( - "github.com/golang/mock/gomock" "testing" "time" - mock_framework "volcano.sh/volcano/pkg/scheduler/framework/mock_gen" + "github.com/golang/mock/gomock" v1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" + "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" + mock_framework "volcano.sh/volcano/pkg/scheduler/framework/mock_gen" "volcano.sh/volcano/pkg/scheduler/util" ) @@ -44,8 +46,8 @@ func TestShuffle(t *testing.T) { ctl := gomock.NewController(t) fakePlugin := mock_framework.NewMockPlugin(ctl) fakePlugin.EXPECT().Name().AnyTimes().Return("fake") - fakePlugin.EXPECT().OnSessionOpen(gomock.Any()).Return() - fakePlugin.EXPECT().OnSessionClose(gomock.Any()).Return() + fakePlugin.EXPECT().OnSessionOpen(gomock.Any()).AnyTimes().Return() + fakePlugin.EXPECT().OnSessionClose(gomock.Any()).AnyTimes().Return() fakePluginBuilder := func(arguments framework.Arguments) framework.Plugin { return fakePlugin } @@ -135,18 +137,22 @@ func TestShuffle(t *testing.T) { evictor := &util.FakeEvictor{ Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - Evictor: evictor, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - PriorityClasses: make(map[string]*schedulingv1.PriorityClass), - - Recorder: record.NewFakeRecorder(100), + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.Evictor = evictor + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + schedulerCache.PriorityClasses["high-priority"] = &schedulingv1.PriorityClass{ Value: highPriority, } diff --git a/pkg/scheduler/plugins/binpack/binpack_test.go b/pkg/scheduler/plugins/binpack/binpack_test.go index 5aee60f3b0..5e9accb8c1 100644 --- a/pkg/scheduler/plugins/binpack/binpack_test.go +++ b/pkg/scheduler/plugins/binpack/binpack_test.go @@ -27,7 +27,8 @@ import ( "k8s.io/client-go/tools/record" schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" - "volcano.sh/volcano/pkg/scheduler/api" + "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" "volcano.sh/volcano/pkg/scheduler/framework" @@ -243,16 +244,21 @@ func TestNode(t *testing.T) { Binds: map[string]string{}, Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - Recorder: record.NewFakeRecorder(100), + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + for _, node := range test.nodes { schedulerCache.AddNode(node) } diff --git a/pkg/scheduler/plugins/drf/hdrf_test.go b/pkg/scheduler/plugins/drf/hdrf_test.go index 4361beccfa..a826d34433 100644 --- a/pkg/scheduler/plugins/drf/hdrf_test.go +++ b/pkg/scheduler/plugins/drf/hdrf_test.go @@ -14,6 +14,7 @@ import ( schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/actions/allocate" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/cache" @@ -210,15 +211,21 @@ func TestHDRF(t *testing.T) { Binds: map[string]string{}, Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - Recorder: record.NewFakeRecorder(100), + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + for _, node := range test.nodes { schedulerCache.AddNode(node) } diff --git a/pkg/scheduler/plugins/proportion/proportion_test.go b/pkg/scheduler/plugins/proportion/proportion_test.go index 734f29143c..0d5b8c3e75 100644 --- a/pkg/scheduler/plugins/proportion/proportion_test.go +++ b/pkg/scheduler/plugins/proportion/proportion_test.go @@ -14,26 +14,29 @@ limitations under the License. package proportion import ( - "github.com/prometheus/client_golang/prometheus/promhttp" - "io/ioutil" - "k8s.io/client-go/util/workqueue" - "net/http" "reflect" "strconv" "strings" "testing" "time" - "volcano.sh/volcano/pkg/scheduler/actions/allocate" "github.com/agiledragon/gomonkey/v2" + "github.com/prometheus/client_golang/prometheus/promhttp" + "io/ioutil" + apiv1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/workqueue" + + "net/http" schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" + "volcano.sh/volcano/pkg/scheduler/actions/allocate" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" @@ -197,16 +200,21 @@ func TestProportion(t *testing.T) { t.Logf("%s: [Event] %s", test.name, event) } }() - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - PriorityClasses: make(map[string]*schedulingv1.PriorityClass), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - Recorder: recorder, + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = recorder + // deletedJobs to DeletedJobs schedulerCache.DeletedJobs = workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()) diff --git a/pkg/scheduler/plugins/tdm/tdm_test.go b/pkg/scheduler/plugins/tdm/tdm_test.go index 7b48d462ee..9371216c18 100644 --- a/pkg/scheduler/plugins/tdm/tdm_test.go +++ b/pkg/scheduler/plugins/tdm/tdm_test.go @@ -27,6 +27,8 @@ import ( "k8s.io/client-go/tools/record" schedulingv2 "volcano.sh/apis/pkg/apis/scheduling/v1beta1" + "volcano.sh/volcano/cmd/scheduler/app/options" + "volcano.sh/volcano/pkg/kube" "volcano.sh/volcano/pkg/scheduler/api" "volcano.sh/volcano/pkg/scheduler/cache" "volcano.sh/volcano/pkg/scheduler/conf" @@ -232,16 +234,21 @@ func Test_TDM(t *testing.T) { Binds: map[string]string{}, Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - - Recorder: record.NewFakeRecorder(100), + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + for _, node := range test.nodes { schedulerCache.AddNode(node) } @@ -677,16 +684,21 @@ func Test_TDM_victimsFn(t *testing.T) { Binds: map[string]string{}, Channel: make(chan string), } - schedulerCache := &cache.SchedulerCache{ - Nodes: make(map[string]*api.NodeInfo), - Jobs: make(map[api.JobID]*api.JobInfo), - Queues: make(map[api.QueueID]*api.QueueInfo), - Binder: binder, - StatusUpdater: &util.FakeStatusUpdater{}, - VolumeBinder: &util.FakeVolumeBinder{}, - - Recorder: record.NewFakeRecorder(100), + + option := options.NewServerOption() + option.RegisterOptions() + config, err := kube.BuildConfig(option.KubeClientOptions) + if err != nil { + return } + + sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector) + schedulerCache := sc.(*cache.SchedulerCache) + schedulerCache.Binder = binder + schedulerCache.StatusUpdater = &util.FakeStatusUpdater{} + schedulerCache.VolumeBinder = &util.FakeVolumeBinder{} + schedulerCache.Recorder = record.NewFakeRecorder(100) + for _, node := range test.nodes { schedulerCache.AddNode(node) } diff --git a/test/e2e/util/queue.go b/test/e2e/util/queue.go index 8701bbab21..87e955c1cf 100644 --- a/test/e2e/util/queue.go +++ b/test/e2e/util/queue.go @@ -52,6 +52,9 @@ func CreateQueueWithQueueSpec(ctx *TestContext, queueSpec *QueueSpec) { _, err := ctx.Vcclient.SchedulingV1beta1().Queues().Create(context.TODO(), queue, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred(), "failed to create queue %s", queueSpec.Name) } + + // wait for queue state turns to be open + time.Sleep(3 * time.Second) } // CreateQueue creates Queue with the specified name @@ -77,6 +80,9 @@ func CreateQueues(ctx *TestContext) { for _, queue := range ctx.Queues { CreateQueue(ctx, queue) } + + // wait for all queues state open + time.Sleep(3 * time.Second) } // DeleteQueue deletes Queue with the specified name