Implementation of KEP-3589: uniformly filter manageJobsWithoutQueueNa…

…mes by namespace (#3712) * core implementation of managedJobsNamespaceSelector * add commented out example managedJobsNamespaceSelector to config * validation of managedJobsNamespaceSelector * defaulting of managedJobsNamespaceSelector * update generated files * generate-apiref * kep status is implementable * add ManagedJobsNamespaceSelector to featuregate table * linter fix: context.Context should be first argument * address comments from first round of reviews 1. parse label selector during startup and used cached result 2. guard all uses with feature gate * fixes to get unit tests to pass * linter fix * updating integration tests to pass new jobframework option * Update pkg/controller/jobframework/defaults.go Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com> * integration tests for managedJobNamespaceSelector functionality * linter fix --------- Co-authored-by: Yuki Iwai <yuki.iwai.tz@gmail.com>
kubernetes-sigs · Dec 5, 2024 · 17e0fe8 · 17e0fe8
1 parent eae2b14
commit 17e0fe8
Show file tree

Hide file tree

Showing 48 changed files with 987 additions and 292 deletions.
diff --git a/apis/config/v1beta1/configuration_types.go b/apis/config/v1beta1/configuration_types.go
@@ -48,6 +48,9 @@ type Configuration struct {
 	// unsuspended, they will start immediately.
 	ManageJobsWithoutQueueName bool `json:"manageJobsWithoutQueueName"`
 
+	// ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName
+	ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"`
+
 	// InternalCertManagement is configuration for internalCertManagement
 	InternalCertManagement *InternalCertManagement `json:"internalCertManagement,omitempty"`
 

diff --git a/apis/config/v1beta1/defaults.go b/apis/config/v1beta1/defaults.go
@@ -179,6 +179,20 @@ func SetDefaults_Configuration(cfg *Configuration) {
 		cfg.Integrations.PodOptions.PodSelector = &metav1.LabelSelector{}
 	}
 
+	if cfg.ManagedJobsNamespaceSelector == nil {
+		matchExpressionsValues := []string{"kube-system", *cfg.Namespace}
+
+		cfg.ManagedJobsNamespaceSelector = &metav1.LabelSelector{
+			MatchExpressions: []metav1.LabelSelectorRequirement{
+				{
+					Key:      "kubernetes.io/metadata.name",
+					Operator: metav1.LabelSelectorOpNotIn,
+					Values:   matchExpressionsValues,
+				},
+			},
+		}
+	}
+
 	if cfg.MultiKueue == nil {
 		cfg.MultiKueue = &MultiKueue{}
 	}

diff --git a/apis/config/v1beta1/defaults_test.go b/apis/config/v1beta1/defaults_test.go
@@ -80,6 +80,15 @@ func TestSetDefaults_Configuration(t *testing.T) {
 			MaxCount: 10,
 		},
 	}
+	defaultManagedJobsNamespaceSelector := &metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{
+				Key:      "kubernetes.io/metadata.name",
+				Operator: metav1.LabelSelectorOpNotIn,
+				Values:   []string{"kube-system", "kueue-system"},
+			},
+		},
+	}
 
 	overwriteNamespaceIntegrations := &Integrations{
 		Frameworks: []string{defaultJobFrameworkName},
@@ -97,6 +106,16 @@ func TestSetDefaults_Configuration(t *testing.T) {
 		},
 	}
 
+	overwriteNamespaceSelector := &metav1.LabelSelector{
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{
+				Key:      "kubernetes.io/metadata.name",
+				Operator: metav1.LabelSelectorOpNotIn,
+				Values:   []string{"kube-system", overwriteNamespace},
+			},
+		},
+	}
+
 	defaultMultiKueue := &MultiKueue{
 		GCInterval:        &metav1.Duration{Duration: DefaultMultiKueueGCInterval},
 		Origin:            ptr.To(DefaultMultiKueueOrigin),
@@ -122,10 +141,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"defaulting ControllerManager": {
@@ -163,10 +183,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"should not default ControllerManager": {
@@ -220,10 +241,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"should not set LeaderElectionID": {
@@ -261,10 +283,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"defaulting InternalCertManagement": {
@@ -279,10 +302,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					WebhookServiceName: ptr.To(DefaultWebhookServiceName),
 					WebhookSecretName:  ptr.To(DefaultWebhookSecretName),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     overwriteNamespaceIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 overwriteNamespaceIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: overwriteNamespaceSelector,
 			},
 		},
 		"should not default InternalCertManagement": {
@@ -298,10 +322,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     overwriteNamespaceIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 overwriteNamespaceIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: overwriteNamespaceSelector,
 			},
 		},
 		"should not default values in custom ClientConnection": {
@@ -325,9 +350,10 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					QPS:   ptr.To[float32](123.0),
 					Burst: ptr.To[int32](456),
 				},
-				Integrations:    overwriteNamespaceIntegrations,
-				QueueVisibility: defaultQueueVisibility,
-				MultiKueue:      defaultMultiKueue,
+				Integrations:                 overwriteNamespaceIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: overwriteNamespaceSelector,
 			},
 		},
 		"should default empty custom ClientConnection": {
@@ -344,10 +370,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     overwriteNamespaceIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 overwriteNamespaceIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: overwriteNamespaceSelector,
 			},
 		},
 		"defaulting waitForPodsReady values": {
@@ -375,10 +402,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"set waitForPodsReady.blockAdmission to false when enable is false": {
@@ -406,10 +434,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"respecting provided waitForPodsReady values": {
@@ -443,10 +472,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"integrations": {
@@ -469,8 +499,9 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Frameworks: []string{"a", "b"},
 					PodOptions: defaultIntegrations.PodOptions,
 				},
-				QueueVisibility: defaultQueueVisibility,
-				MultiKueue:      defaultMultiKueue,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"queue visibility": {
@@ -499,7 +530,8 @@ func TestSetDefaults_Configuration(t *testing.T) {
 						MaxCount: 0,
 					},
 				},
-				MultiKueue: defaultMultiKueue,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"multiKueue": {
@@ -527,6 +559,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Origin:            ptr.To("multikueue-manager1"),
 					WorkerLostTimeout: &metav1.Duration{Duration: time.Minute},
 				},
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"multiKueue GCInterval 0": {
@@ -553,6 +586,7 @@ func TestSetDefaults_Configuration(t *testing.T) {
 					Origin:            ptr.To("multikueue-manager1"),
 					WorkerLostTimeout: &metav1.Duration{Duration: 15 * time.Minute},
 				},
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 			},
 		},
 		"add default fair sharing configuration when enabled": {
@@ -570,10 +604,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 				FairSharing: &FairSharing{
 					Enable:               true,
 					PreemptionStrategies: []PreemptionStrategy{LessThanOrEqualToFinalShare, LessThanInitialShare},
@@ -598,10 +633,11 @@ func TestSetDefaults_Configuration(t *testing.T) {
 				InternalCertManagement: &InternalCertManagement{
 					Enable: ptr.To(false),
 				},
-				ClientConnection: defaultClientConnection,
-				Integrations:     defaultIntegrations,
-				QueueVisibility:  defaultQueueVisibility,
-				MultiKueue:       defaultMultiKueue,
+				ClientConnection:             defaultClientConnection,
+				Integrations:                 defaultIntegrations,
+				QueueVisibility:              defaultQueueVisibility,
+				MultiKueue:                   defaultMultiKueue,
+				ManagedJobsNamespaceSelector: defaultManagedJobsNamespaceSelector,
 				Resources: &Resources{
 					Transformations: []ResourceTransformation{
 						{Input: corev1.ResourceCPU, Strategy: ptr.To(DefaultResourceTransformationStrategy)},

diff --git a/apis/config/v1beta1/zz_generated.deepcopy.go b/apis/config/v1beta1/zz_generated.deepcopy.go
diff --git a/cmd/kueue/main.go b/cmd/kueue/main.go
@@ -27,6 +27,7 @@ import (
 	"go.uber.org/zap/zapcore"
 	corev1 "k8s.io/api/core/v1"
 	schedulingv1 "k8s.io/api/scheduling/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -305,6 +306,15 @@ func setupControllers(ctx context.Context, mgr ctrl.Manager, cCache *cache.Cache
 		jobframework.WithCache(cCache),
 		jobframework.WithQueues(queues),
 	}
+	if features.Enabled(features.ManagedJobsNamespaceSelector) {
+		nsSelector, err := metav1.LabelSelectorAsSelector(cfg.ManagedJobsNamespaceSelector)
+		if err != nil {
+			setupLog.Error(err, "Failed to parse managedJobsNamespaceSelector")
+			os.Exit(1)
+		}
+		opts = append(opts, jobframework.WithManagedJobsNamespaceSelector(nsSelector))
+	}
+
 	if err := jobframework.SetupControllers(ctx, mgr, setupLog, opts...); err != nil {
 		setupLog.Error(err, "Unable to create controller or webhook", "kubernetesVersion", serverVersionFetcher.GetServerVersion())
 		os.Exit(1)

diff --git a/config/components/manager/controller_manager_config.yaml b/config/components/manager/controller_manager_config.yaml
@@ -33,6 +33,9 @@ clientConnection:
 #    backoffBaseSeconds: 60
 #    backoffMaxSeconds: 3600
 #manageJobsWithoutQueueName: true
+#managedJobsNamespaceSelector:
+#  matchLabels:
+#    kueue-managed: "true"
 #internalCertManagement:
 #  enable: false
 #  webhookServiceName: ""

diff --git a/keps/3589-manage-jobs-selectively/kep.yaml b/keps/3589-manage-jobs-selectively/kep.yaml
@@ -2,7 +2,7 @@ title: Uniformly filter manageJobsWithoutQueueNames by namespace
 kep-number: 3589
 authors:
   - "@dgrove-oss"
-status: provisional
+status: implementable
 creation-date: 2024-11-19
 reviewers:
   - "@mimowo"
@@ -16,11 +16,11 @@ stage: beta
 # The most recent milestone for which work toward delivery of this KEP has been
 # done. This can be the current (upcoming) milestone, if it is being actively
 # worked on.
-latest-milestone: "v0.11"
+latest-milestone: "v0.10"
 
 # The milestone at which this feature was, or is targeted to be, at each stage.
 milestone:
-  beta: "v0.11"
+  beta: "v0.10"
 
 feature-gates:
   - name: ManagedJobsNamespaceSelector