Add allocated field in queue status to indicate used resources in queue.

config/crd/bases/scheduling.volcano.sh_queues.yaml

@@ -133,6 +133,15 @@ spec:
           status:
             description: The status of queue.
             properties:
+              allocated:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Allocated is allocated resources in queue
+                type: object
               completed:
                 description: The number of `Completed` PodGroup in this queue.
                 format: int32
@@ -176,6 +185,8 @@ spec:
                 description: The number of 'Unknown' PodGroup in this queue.
                 format: int32
                 type: integer
+            required:
+            - allocated
             type: object
         type: object
     served: true

config/crd/v1beta1/scheduling.volcano.sh_queues.yaml

@@ -132,6 +132,15 @@ spec:
         status:
           description: The status of queue.
           properties:
+            allocated:
+              additionalProperties:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              description: Allocated is allocated resources in queue
+              type: object
             completed:
               description: The number of `Completed` PodGroup in this queue.
               format: int32
@@ -175,6 +184,8 @@ spec:
               description: The number of 'Unknown' PodGroup in this queue.
               format: int32
               type: integer
+          required:
+          - allocated
           type: object
       type: object
   version: v1beta1

go.mod

@@ -35,7 +35,7 @@ require (
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 	sigs.k8s.io/yaml v1.3.0
 	stathat.com/c/consistent v1.0.0
-	volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c
+	volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d
 )
 
 require (

go.sum

@@ -827,5 +827,5 @@ sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
 sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
 stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
 stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
-volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c h1:s7K9ouTNVr6OWyH9AX4aG4crc7XvuT8Dcfeaqt6gzkg=
-volcano.sh/apis v1.6.0-alpha.0.0.20221021034835-d3a04f5cfc7c/go.mod h1:xe38GChdXXam/g/FkQXIsR0vhqp4twoZdY2gaGkEP24=
+volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d h1:3ThPUKyWZxJTOHBRHv1VGDzaLGl+Bb/g362xNG749zE=
+volcano.sh/apis v1.6.0-alpha.0.0.20221206071715-99768c621d1d/go.mod h1:xe38GChdXXam/g/FkQXIsR0vhqp4twoZdY2gaGkEP24=

installer/helm/chart/volcano/crd/bases/scheduling.volcano.sh_queues.yaml

@@ -131,6 +131,15 @@ spec:
           status:
             description: The status of queue.
             properties:
+              allocated:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Allocated is allocated resources in queue
+                type: object
               completed:
                 description: The number of `Completed` PodGroup in this queue.
                 format: int32
@@ -174,6 +183,8 @@ spec:
                 description: The number of 'Unknown' PodGroup in this queue.
                 format: int32
                 type: integer
+            required:
+            - allocated
             type: object
         type: object
     served: true

installer/helm/chart/volcano/crd/v1beta1/scheduling.volcano.sh_queues.yaml

@@ -130,6 +130,15 @@ spec:
         status:
           description: The status of queue.
           properties:
+            allocated:
+              additionalProperties:
+                anyOf:
+                - type: integer
+                - type: string
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+              description: Allocated is allocated resources in queue
+              type: object
             completed:
               description: The number of `Completed` PodGroup in this queue.
               format: int32
@@ -173,6 +182,8 @@ spec:
               description: The number of 'Unknown' PodGroup in this queue.
               format: int32
               type: integer
+          required:
+          - allocated
           type: object
       type: object
   version: v1beta1

installer/helm/chart/volcano/templates/scheduler.yaml

@@ -63,6 +63,9 @@ rules:
   - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
     resources: ["queues"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["queues/status"]
+    verbs: ["update"]
   - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
     resources: ["podgroups"]
     verbs: ["list", "watch", "update"]
@@ -150,4 +153,4 @@ spec:
   selector:
     app: volcano-scheduler
   type: ClusterIP
-{{- end }}
+{{- end }}

installer/volcano-development.yaml

@@ -8690,6 +8690,9 @@ rules:
   - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
     resources: ["queues"]
     verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
+    resources: ["queues/status"]
+    verbs: ["update"]
   - apiGroups: ["scheduling.incubator.k8s.io", "scheduling.volcano.sh"]
     resources: ["podgroups"]
     verbs: ["list", "watch", "update"]
@@ -9067,6 +9070,15 @@ spec:
           status:
             description: The status of queue.
             properties:
+              allocated:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Allocated is allocated resources in queue
+                type: object
               completed:
                 description: The number of `Completed` PodGroup in this queue.
                 format: int32
@@ -9110,6 +9122,8 @@ spec:
                 description: The number of 'Unknown' PodGroup in this queue.
                 format: int32
                 type: integer
+            required:
+            - allocated
             type: object
         type: object
     served: true

pkg/controllers/queue/queue_controller_action.go

@@ -66,6 +66,8 @@ func (c *queuecontroller) syncQueue(queue *schedulingv1beta1.Queue, updateStateF
 		queueStatus.State = queue.Status.State
 	}
 
+	queueStatus.Allocated = queue.Status.Allocated.DeepCopy()
+
 	// ignore update when status does not change
 	if reflect.DeepEqual(queueStatus, queue.Status) {
 		return nil

pkg/scheduler/actions/allocate/allocate_test.go

@@ -24,21 +24,22 @@ import (
 
 	"github.com/agiledragon/gomonkey/v2"
 	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/tools/record"
-	"volcano.sh/volcano/pkg/scheduler/plugins/gang"
-	"volcano.sh/volcano/pkg/scheduler/plugins/priority"
 
-	storagev1 "k8s.io/api/storage/v1"
 	schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
 	"volcano.sh/volcano/cmd/scheduler/app/options"
+	"volcano.sh/volcano/pkg/kube"
 	"volcano.sh/volcano/pkg/scheduler/api"
 	"volcano.sh/volcano/pkg/scheduler/cache"
 	"volcano.sh/volcano/pkg/scheduler/conf"
 	"volcano.sh/volcano/pkg/scheduler/framework"
 	"volcano.sh/volcano/pkg/scheduler/plugins/drf"
+	"volcano.sh/volcano/pkg/scheduler/plugins/gang"
+	"volcano.sh/volcano/pkg/scheduler/plugins/priority"
 	"volcano.sh/volcano/pkg/scheduler/plugins/proportion"
 	"volcano.sh/volcano/pkg/scheduler/util"
 )
@@ -245,16 +246,19 @@ func TestAllocate(t *testing.T) {
 				Binds:   map[string]string{},
 				Channel: make(chan string),
 			}
-			schedulerCache := &cache.SchedulerCache{
-				Nodes:         make(map[string]*api.NodeInfo),
-				Jobs:          make(map[api.JobID]*api.JobInfo),
-				Queues:        make(map[api.QueueID]*api.QueueInfo),
-				Binder:        binder,
-				StatusUpdater: &util.FakeStatusUpdater{},
-				VolumeBinder:  &util.FakeVolumeBinder{},
-
-				Recorder: record.NewFakeRecorder(100),
+			option := options.NewServerOption()
+			option.RegisterOptions()
+			config, err := kube.BuildConfig(option.KubeClientOptions)
+			if err != nil {
+				return
 			}
+			sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector)
+			schedulerCache := sc.(*cache.SchedulerCache)
+
+			schedulerCache.Binder = binder
+			schedulerCache.StatusUpdater = &util.FakeStatusUpdater{}
+			schedulerCache.VolumeBinder = &util.FakeVolumeBinder{}
+			schedulerCache.Recorder = record.NewFakeRecorder(100)
 
 			for _, node := range test.nodes {
 				schedulerCache.AddNode(node)
@@ -417,15 +421,21 @@ func TestAllocateWithDynamicPVC(t *testing.T) {
 				Binds:   map[string]string{},
 				Channel: make(chan string),
 			}
-			schedulerCache := &cache.SchedulerCache{
-				Nodes:         make(map[string]*api.NodeInfo),
-				Jobs:          make(map[api.JobID]*api.JobInfo),
-				Queues:        make(map[api.QueueID]*api.QueueInfo),
-				Binder:        binder,
-				StatusUpdater: &util.FakeStatusUpdater{},
-				VolumeBinder:  fakeVolumeBinder,
-				Recorder:      record.NewFakeRecorder(100),
+
+			option := options.NewServerOption()
+			option.RegisterOptions()
+			config, err := kube.BuildConfig(option.KubeClientOptions)
+			if err != nil {
+				return
 			}
+
+			sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector)
+			schedulerCache := sc.(*cache.SchedulerCache)
+			schedulerCache.Binder = binder
+			schedulerCache.StatusUpdater = &util.FakeStatusUpdater{}
+			schedulerCache.VolumeBinder = fakeVolumeBinder
+			schedulerCache.Recorder = record.NewFakeRecorder(100)
+
 			schedulerCache.AddQueueV1beta1(queue)
 			schedulerCache.AddPodGroupV1beta1(pg)
 			for i, pod := range test.pods {

pkg/scheduler/actions/preempt/preempt_test.go

@@ -27,7 +27,7 @@ import (
 
 	schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
 	"volcano.sh/volcano/cmd/scheduler/app/options"
-	"volcano.sh/volcano/pkg/scheduler/api"
+	"volcano.sh/volcano/pkg/kube"
 	"volcano.sh/volcano/pkg/scheduler/cache"
 	"volcano.sh/volcano/pkg/scheduler/conf"
 	"volcano.sh/volcano/pkg/scheduler/framework"
@@ -287,18 +287,22 @@ func TestPreempt(t *testing.T) {
 			evictor := &util.FakeEvictor{
 				Channel: make(chan string),
 			}
-			schedulerCache := &cache.SchedulerCache{
-				Nodes:           make(map[string]*api.NodeInfo),
-				Jobs:            make(map[api.JobID]*api.JobInfo),
-				Queues:          make(map[api.QueueID]*api.QueueInfo),
-				Binder:          binder,
-				Evictor:         evictor,
-				StatusUpdater:   &util.FakeStatusUpdater{},
-				VolumeBinder:    &util.FakeVolumeBinder{},
-				PriorityClasses: make(map[string]*schedulingv1.PriorityClass),
 
-				Recorder: record.NewFakeRecorder(100),
+			option := options.NewServerOption()
+			option.RegisterOptions()
+			config, err := kube.BuildConfig(option.KubeClientOptions)
+			if err != nil {
+				return
 			}
+
+			sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector)
+			schedulerCache := sc.(*cache.SchedulerCache)
+			schedulerCache.Binder = binder
+			schedulerCache.Evictor = evictor
+			schedulerCache.StatusUpdater = &util.FakeStatusUpdater{}
+			schedulerCache.VolumeBinder = &util.FakeVolumeBinder{}
+			schedulerCache.Recorder = record.NewFakeRecorder(100)
+
 			schedulerCache.PriorityClasses["high-priority"] = &schedulingv1.PriorityClass{
 				Value: 100000,
 			}

pkg/scheduler/actions/reclaim/reclaim_test.go

@@ -26,7 +26,8 @@ import (
 	"k8s.io/client-go/tools/record"
 
 	schedulingv1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
-	"volcano.sh/volcano/pkg/scheduler/api"
+	"volcano.sh/volcano/cmd/scheduler/app/options"
+	"volcano.sh/volcano/pkg/kube"
 	"volcano.sh/volcano/pkg/scheduler/cache"
 	"volcano.sh/volcano/pkg/scheduler/conf"
 	"volcano.sh/volcano/pkg/scheduler/framework"
@@ -121,18 +122,22 @@ func TestReclaim(t *testing.T) {
 		evictor := &util.FakeEvictor{
 			Channel: make(chan string),
 		}
-		schedulerCache := &cache.SchedulerCache{
-			Nodes:           make(map[string]*api.NodeInfo),
-			Jobs:            make(map[api.JobID]*api.JobInfo),
-			Queues:          make(map[api.QueueID]*api.QueueInfo),
-			Binder:          binder,
-			Evictor:         evictor,
-			StatusUpdater:   &util.FakeStatusUpdater{},
-			VolumeBinder:    &util.FakeVolumeBinder{},
-			PriorityClasses: make(map[string]*schedulingv1.PriorityClass),
-
-			Recorder: record.NewFakeRecorder(100),
+
+		option := options.NewServerOption()
+		option.RegisterOptions()
+		config, err := kube.BuildConfig(option.KubeClientOptions)
+		if err != nil {
+			return
 		}
+
+		sc := cache.New(config, option.SchedulerNames, option.DefaultQueue, option.NodeSelector)
+		schedulerCache := sc.(*cache.SchedulerCache)
+		schedulerCache.Binder = binder
+		schedulerCache.Evictor = evictor
+		schedulerCache.StatusUpdater = &util.FakeStatusUpdater{}
+		schedulerCache.VolumeBinder = &util.FakeVolumeBinder{}
+		schedulerCache.Recorder = record.NewFakeRecorder(100)
+
 		schedulerCache.PriorityClasses["high-priority"] = &schedulingv1.PriorityClass{
 			Value: 100000,
 		}