Merge pull request #1595 from mallow111/issue-1593

Issue #1595 ---Part2 --- Add namespace denied list

docs/cli-arguments.md

@@ -42,6 +42,7 @@ Usage of ./kube-state-metrics:
       --metric-denylist string                Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
       --metric-labels-allowlist string        Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]').
       --namespaces string                     Comma-separated list of namespaces to be enabled. Defaults to ""
+      --namespaces-denylist string            Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.
       --one_output                            If true, only write logs to their native severity level (vs also writing to each lower severity level)
       --pod string                            Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
       --pod-namespace string                  Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.

internal/store/builder.go

@@ -60,6 +60,7 @@ type Builder struct {
 	kubeClient           clientset.Interface
 	vpaClient            vpaclientset.Interface
 	namespaces           options.NamespaceList
+	namespaceFilter      string
 	ctx                  context.Context
 	enabledResources     []string
 	allowDenyList        ksmtypes.AllowDenyLister
@@ -103,8 +104,9 @@ func (b *Builder) WithEnabledResources(r []string) error {
 }
 
 // WithNamespaces sets the namespaces property of a Builder.
-func (b *Builder) WithNamespaces(n options.NamespaceList) {
+func (b *Builder) WithNamespaces(n options.NamespaceList, nsFilter string) {
 	b.namespaces = n
+	b.namespaceFilter = nsFilter
 }
 
 // WithSharding sets the shard and totalShards property of a Builder.
@@ -393,7 +395,7 @@ func (b *Builder) buildStores(
 			familyHeaders,
 			composedMetricGenFuncs,
 		)
-		listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, "")
+		listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.namespaceFilter)
 		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
 		return []cache.Store{store}
 	}
@@ -404,7 +406,7 @@ func (b *Builder) buildStores(
 			familyHeaders,
 			composedMetricGenFuncs,
 		)
-		listWatcher := listWatchFunc(b.kubeClient, ns, "")
+		listWatcher := listWatchFunc(b.kubeClient, ns, b.namespaceFilter)
 		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
 		stores = append(stores, store)
 	}

main.go

@@ -114,17 +114,9 @@ func main() {
 		klog.Fatalf("Failed to set up resources: %v", err)
 	}
 
-	if len(opts.Namespaces) == 0 {
-		klog.Info("Using all namespace")
-		storeBuilder.WithNamespaces(options.DefaultNamespaces)
-	} else {
-		if opts.Namespaces.IsAllNamespaces() {
-			klog.Info("Using all namespace")
-		} else {
-			klog.Infof("Using %s namespaces", opts.Namespaces)
-		}
-		storeBuilder.WithNamespaces(opts.Namespaces)
-	}
+	namespaces := opts.Namespaces.GetNamespaces()
+	nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
+	storeBuilder.WithNamespaces(namespaces, nsFieldSelector)
 
 	allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist)
 	if err != nil {

main_test.go

@@ -66,7 +66,7 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
 	builder.WithKubeClient(kubeClient)
 	builder.WithSharding(0, 1)
 	builder.WithContext(ctx)
-	builder.WithNamespaces(options.DefaultNamespaces)
+	builder.WithNamespaces(options.DefaultNamespaces, "")
 	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc(), false)
 
 	l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
@@ -132,7 +132,7 @@ func TestFullScrapeCycle(t *testing.T) {
 	builder.WithMetrics(reg)
 	builder.WithEnabledResources(options.DefaultResources.AsSlice())
 	builder.WithKubeClient(kubeClient)
-	builder.WithNamespaces(options.DefaultNamespaces)
+	builder.WithNamespaces(options.DefaultNamespaces, "")
 	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc(), false)
 
 	l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
@@ -399,7 +399,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	unshardedBuilder.WithMetrics(reg)
 	unshardedBuilder.WithEnabledResources(options.DefaultResources.AsSlice())
 	unshardedBuilder.WithKubeClient(kubeClient)
-	unshardedBuilder.WithNamespaces(options.DefaultNamespaces)
+	unshardedBuilder.WithNamespaces(options.DefaultNamespaces, "")
 	unshardedBuilder.WithAllowDenyList(l)
 	unshardedBuilder.WithAllowLabels(map[string][]string{})
 	unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc(), false)
@@ -412,7 +412,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	shardedBuilder1.WithMetrics(regShard1)
 	shardedBuilder1.WithEnabledResources(options.DefaultResources.AsSlice())
 	shardedBuilder1.WithKubeClient(kubeClient)
-	shardedBuilder1.WithNamespaces(options.DefaultNamespaces)
+	shardedBuilder1.WithNamespaces(options.DefaultNamespaces, "")
 	shardedBuilder1.WithAllowDenyList(l)
 	shardedBuilder1.WithAllowLabels(map[string][]string{})
 	shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc(), false)
@@ -425,7 +425,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	shardedBuilder2.WithMetrics(regShard2)
 	shardedBuilder2.WithEnabledResources(options.DefaultResources.AsSlice())
 	shardedBuilder2.WithKubeClient(kubeClient)
-	shardedBuilder2.WithNamespaces(options.DefaultNamespaces)
+	shardedBuilder2.WithNamespaces(options.DefaultNamespaces, "")
 	shardedBuilder2.WithAllowDenyList(l)
 	shardedBuilder2.WithAllowLabels(map[string][]string{})
 	shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc(), false)

pkg/builder/builder.go

@@ -55,8 +55,8 @@ func (b *Builder) WithEnabledResources(c []string) error {
 }
 
 // WithNamespaces sets the namespaces property of a Builder.
-func (b *Builder) WithNamespaces(n options.NamespaceList) {
-	b.internal.WithNamespaces(n)
+func (b *Builder) WithNamespaces(n options.NamespaceList, nsFilter string) {
+	b.internal.WithNamespaces(n, nsFilter)
 }
 
 // WithSharding sets the shard and totalShards property of a Builder.

pkg/builder/types/interfaces.go

@@ -34,7 +34,7 @@ import (
 type BuilderInterface interface {
 	WithMetrics(r prometheus.Registerer)
 	WithEnabledResources(c []string) error
-	WithNamespaces(n options.NamespaceList)
+	WithNamespaces(n options.NamespaceList, nsFilter string)
 	WithSharding(shard int32, totalShards int)
 	WithContext(ctx context.Context)
 	WithKubeClient(c clientset.Interface)

pkg/options/options.go

@@ -38,6 +38,7 @@ type Options struct {
 	TLSConfig            string
 	Resources            ResourceSet
 	Namespaces           NamespaceList
+	NamespacesDenylist   NamespaceList
 	Shard                int32
 	TotalShards          int
 	Pod                  string
@@ -93,6 +94,7 @@ func (o *Options) AddFlags() {
 	o.flags.StringVar(&o.TelemetryHost, "telemetry-host", "::", `Host to expose kube-state-metrics self metrics on.`)
 	o.flags.Var(&o.Resources, "resources", fmt.Sprintf("Comma-separated list of Resources to be enabled. Defaults to %q", &DefaultResources))
 	o.flags.Var(&o.Namespaces, "namespaces", fmt.Sprintf("Comma-separated list of namespaces to be enabled. Defaults to %q", &DefaultNamespaces))
+	o.flags.Var(&o.NamespacesDenylist, "namespaces-denylist", "Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.")
 	o.flags.Var(&o.MetricAllowlist, "metric-allowlist", "Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
 	o.flags.Var(&o.MetricDenylist, "metric-denylist", "Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.")
 	o.flags.Var(&o.AnnotationsAllowList, "metric-annotations-allowlist", "Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional annotations provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').")

pkg/options/types.go

@@ -21,6 +21,10 @@ import (
 	"sort"
 	"strings"
 
+	"k8s.io/apimachinery/pkg/fields"
+
+	"k8s.io/klog/v2"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -125,6 +129,37 @@ func (n *NamespaceList) Set(value string) error {
 	return nil
 }
 
+// GetNamespaces is a helper function to get namespaces from opts.Namespaces
+func (n *NamespaceList) GetNamespaces() NamespaceList {
+	ns := *n
+	if len(*n) == 0 {
+		klog.Info("Using all namespace")
+		ns = DefaultNamespaces
+	} else {
+		if n.IsAllNamespaces() {
+			klog.Info("Using all namespace")
+		} else {
+			klog.Infof("Using %s namespaces", ns)
+		}
+	}
+	return ns
+}
+
+// GetExcludeNSFieldSelector will return excluded namespace field selector
+// if nsDenylist = {case1,case2}, the result will be "metadata.namespace!=case1,metadata.namespace!=case2".
+func (n *NamespaceList) GetExcludeNSFieldSelector(nsDenylist []string) string {
+	if len(nsDenylist) == 0 {
+		return ""
+	}
+
+	namespaceExcludeSelectors := make([]fields.Selector, len(nsDenylist))
+	for i, ns := range nsDenylist {
+		selector := fields.OneTermNotEqualSelector("metadata.namespace", ns)
+		namespaceExcludeSelectors[i] = selector
+	}
+	return fields.AndSelectors(namespaceExcludeSelectors...).String()
+}
+
 // Type returns a descriptive string about the NamespaceList type.
 func (n *NamespaceList) Type() string {
 	return "string"

pkg/options/types_test.go

@@ -86,6 +86,75 @@ func TestNamespaceListSet(t *testing.T) {
 	}
 }
 
+func TestNamespaceList_GetNamespaces(t *testing.T) {
+	tests := []struct {
+		Desc       string
+		Namespaces NamespaceList
+		Wanted     NamespaceList
+	}{
+		{
+			Desc:       "empty DeniedNamespaces",
+			Namespaces: NamespaceList{},
+			Wanted:     NamespaceList{""},
+		},
+		{
+			Desc:       "all DeniedNamespaces",
+			Namespaces: DefaultNamespaces,
+			Wanted:     NamespaceList{""},
+		},
+		{
+			Desc:       "general namespaceDenylist",
+			Namespaces: NamespaceList{"default", "kube-system"},
+			Wanted:     NamespaceList{"default", "kube-system"},
+		},
+	}
+
+	for _, test := range tests {
+		ns := &test.Namespaces
+		allowedNamespaces := ns.GetNamespaces()
+		if !reflect.DeepEqual(allowedNamespaces, test.Wanted) {
+			t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, allowedNamespaces)
+		}
+	}
+}
+
+func TestNamespaceList_ExcludeNamespacesFieldSelector(t *testing.T) {
+	tests := []struct {
+		Desc             string
+		Namespaces       NamespaceList
+		DeniedNamespaces NamespaceList
+		Wanted           string
+	}{
+		{
+			Desc:             "empty DeniedNamespaces",
+			Namespaces:       NamespaceList{"default", "kube-system"},
+			DeniedNamespaces: NamespaceList{},
+			Wanted:           "",
+		},
+		{
+			Desc:             "all DeniedNamespaces",
+			Namespaces:       DefaultNamespaces,
+			DeniedNamespaces: NamespaceList{"some-system"},
+			Wanted:           "metadata.namespace!=some-system",
+		},
+		{
+			Desc:             "general case",
+			Namespaces:       DefaultNamespaces,
+			DeniedNamespaces: NamespaceList{"case1-system", "case2-system"},
+			Wanted:           "metadata.namespace!=case1-system,metadata.namespace!=case2-system",
+		},
+	}
+
+	for _, test := range tests {
+		ns := test.Namespaces
+		deniedNS := test.DeniedNamespaces
+		actual := ns.GetExcludeNSFieldSelector(deniedNS)
+		if !reflect.DeepEqual(actual, test.Wanted) {
+			t.Errorf("Test error for Desc: %s. Want: %+v. Got: %+v.", test.Desc, test.Wanted, actual)
+		}
+	}
+}
+
 func TestMetricSetSet(t *testing.T) {
 	tests := []struct {
 		Desc   string