Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CONTP-369] extract kubernetes resource parsers (pod, deployment and metadata) to a separate package #28479

Merged
merged 1 commit into from
Aug 30, 2024
Merged

[CONTP-369] extract kubernetes resource parsers (pod, deployment and metadata) to a separate package #28479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 4 additions & 75 deletions comp/core/workloadmeta/collectors/internal/kubeapiserver/deployments.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,6 @@ package kubeapiserver

import (
"context"
"github.com/DataDog/datadog-agent/pkg/util/log"
"k8s.io/apimachinery/pkg/runtime/schema"
"regexp"
"strings"

appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -23,9 +19,9 @@ import (
"k8s.io/client-go/tools/cache"

"github.com/DataDog/datadog-agent/comp/core/config"
kubernetesresourceparsers "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util/kubernetes_resource_parsers"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
languagedetectionUtil "github.com/DataDog/datadog-agent/pkg/languagedetection/util"
ddkube "github.com/DataDog/datadog-agent/pkg/util/kubernetes"
"github.com/DataDog/datadog-agent/pkg/util/log"
)

// deploymentFilter filters out deployments that can't be used for unified service tagging or process language detection
Expand Down Expand Up @@ -59,10 +55,10 @@ func newDeploymentStore(ctx context.Context, wlm workloadmeta.Component, cfg con

func newDeploymentReflectorStore(wlmetaStore workloadmeta.Component, cfg config.Reader) *reflectorStore {
annotationsExclude := cfg.GetStringSlice("cluster_agent.kubernetes_resources_collection.deployment_annotations_exclude")
parser, err := newdeploymentParser(annotationsExclude)
parser, err := kubernetesresourceparsers.NewDeploymentParser(annotationsExclude)
if err != nil {
_ = log.Errorf("unable to parse all deployment_annotations_exclude: %v, err:", err)
parser, _ = newdeploymentParser(nil)
parser, _ = kubernetesresourceparsers.NewDeploymentParser(nil)
}

store := &reflectorStore{
Expand All @@ -74,70 +70,3 @@ func newDeploymentReflectorStore(wlmetaStore workloadmeta.Component, cfg config.

return store
}

type deploymentParser struct {
annotationsFilter []*regexp.Regexp
gvr *schema.GroupVersionResource
}

func newdeploymentParser(annotationsExclude []string) (objectParser, error) {
filters, err := parseFilters(annotationsExclude)
if err != nil {
return nil, err
}
return deploymentParser{
annotationsFilter: filters,
gvr: &schema.GroupVersionResource{
Group: "apps",
Version: "v1",
Resource: "deployments",
},
}, nil
}

func updateContainerLanguage(cl languagedetectionUtil.ContainersLanguages, container languagedetectionUtil.Container, languages string) {
if _, found := cl[container]; !found {
cl[container] = make(languagedetectionUtil.LanguageSet)
}

for _, lang := range strings.Split(languages, ",") {
cl[container][languagedetectionUtil.Language(strings.TrimSpace(lang))] = struct{}{}
}
}

func (p deploymentParser) Parse(obj interface{}) workloadmeta.Entity {
deployment := obj.(*appsv1.Deployment)
containerLanguages := make(languagedetectionUtil.ContainersLanguages)

for annotation, languages := range deployment.Annotations {

containerName, isInitContainer := languagedetectionUtil.ExtractContainerFromAnnotationKey(annotation)
if containerName != "" && languages != "" {

updateContainerLanguage(
containerLanguages,
languagedetectionUtil.Container{
Name: containerName,
Init: isInitContainer,
},
languages)
}
}

return &workloadmeta.KubernetesDeployment{
EntityID: workloadmeta.EntityID{
Kind: workloadmeta.KindKubernetesDeployment,
ID: deployment.Namespace + "/" + deployment.Name, // we use the namespace/name as id to make it easier for the admission controller to retrieve the corresponding deployment
},
EntityMeta: workloadmeta.EntityMeta{
Name: deployment.Name,
Namespace: deployment.Namespace,
Labels: deployment.Labels,
Annotations: filterMapStringKey(deployment.Annotations, p.annotationsFilter),
},
Env: deployment.Labels[ddkube.EnvTagLabelKey],
Service: deployment.Labels[ddkube.ServiceTagLabelKey],
Version: deployment.Labels[ddkube.VersionTagLabelKey],
InjectableLanguages: containerLanguages,
}
}
181 changes: 2 additions & 179 deletions comp/core/workloadmeta/collectors/internal/kubeapiserver/deployments_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,10 @@ package kubeapiserver

import (
"context"
langUtil "github.com/DataDog/datadog-agent/pkg/languagedetection/util"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
langUtil "github.com/DataDog/datadog-agent/pkg/languagedetection/util"

appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes/fake"
Expand All @@ -22,182 +21,6 @@ import (
"github.com/DataDog/datadog-agent/pkg/languagedetection/languagemodels"
)

func TestDeploymentParser_Parse(t *testing.T) {
excludeAnnotations := []string{"ignore-annotation"}

tests := []struct {
name string
expected *workloadmeta.KubernetesDeployment
deployment *appsv1.Deployment
}{
{
name: "everything",
expected: &workloadmeta.KubernetesDeployment{
EntityID: workloadmeta.EntityID{
Kind: workloadmeta.KindKubernetesDeployment,
ID: "test-namespace/test-deployment",
},
Env: "env",
Service: "service",
Version: "version",
EntityMeta: workloadmeta.EntityMeta{
Name: "test-deployment",
Namespace: "test-namespace",
Labels: map[string]string{
"test-label": "test-value",
"tags.datadoghq.com/env": "env",
"tags.datadoghq.com/service": "service",
"tags.datadoghq.com/version": "version",
},
Annotations: map[string]string{
"internal.dd.datadoghq.com/nginx-cont.detected_langs": "go,java, python ",
"internal.dd.datadoghq.com/init.nginx-cont.detected_langs": "go,java, python ",
},
},
InjectableLanguages: langUtil.ContainersLanguages{
*langUtil.NewInitContainer("nginx-cont"): {
langUtil.Language(languagemodels.Go): {},
langUtil.Language(languagemodels.Java): {},
langUtil.Language(languagemodels.Python): {},
},
*langUtil.NewContainer("nginx-cont"): {
langUtil.Language(languagemodels.Go): {},
langUtil.Language(languagemodels.Java): {},
langUtil.Language(languagemodels.Python): {},
},
},
},

deployment: &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
Namespace: "test-namespace",
Labels: map[string]string{
"test-label": "test-value",
"tags.datadoghq.com/env": "env",
"tags.datadoghq.com/service": "service",
"tags.datadoghq.com/version": "version",
},
Annotations: map[string]string{
"internal.dd.datadoghq.com/nginx-cont.detected_langs": "go,java, python ",
"internal.dd.datadoghq.com/init.nginx-cont.detected_langs": "go,java, python ",
"ignore-annotation": "ignore",
},
},
},
},
{
name: "only usm",
expected: &workloadmeta.KubernetesDeployment{
EntityID: workloadmeta.EntityID{
Kind: workloadmeta.KindKubernetesDeployment,
ID: "test-namespace/test-deployment",
},
EntityMeta: workloadmeta.EntityMeta{
Name: "test-deployment",
Namespace: "test-namespace",
Labels: map[string]string{
"test-label": "test-value",
"tags.datadoghq.com/env": "env",
"tags.datadoghq.com/service": "service",
"tags.datadoghq.com/version": "version",
},
Annotations: map[string]string{},
},
Env: "env",
Service: "service",
Version: "version",
InjectableLanguages: make(langUtil.ContainersLanguages),
},
deployment: &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
Namespace: "test-namespace",
Labels: map[string]string{
"test-label": "test-value",
"tags.datadoghq.com/env": "env",
"tags.datadoghq.com/service": "service",
"tags.datadoghq.com/version": "version",
},
Annotations: map[string]string{
"ignore-annotation": "ignore",
},
},
},
},

{
name: "only languages",
expected: &workloadmeta.KubernetesDeployment{
EntityID: workloadmeta.EntityID{
Kind: workloadmeta.KindKubernetesDeployment,
ID: "test-namespace/test-deployment",
},
EntityMeta: workloadmeta.EntityMeta{
Name: "test-deployment",
Namespace: "test-namespace",
Labels: map[string]string{
"test-label": "test-value",
},
Annotations: map[string]string{
"internal.dd.datadoghq.com/nginx-cont.detected_langs": "go,java, python ",
"internal.dd.datadoghq.com/init.nginx-cont.detected_langs": "go,java, python ",
},
},
InjectableLanguages: langUtil.ContainersLanguages{
*langUtil.NewInitContainer("nginx-cont"): {
langUtil.Language(languagemodels.Go): {},
langUtil.Language(languagemodels.Java): {},
langUtil.Language(languagemodels.Python): {},
},
*langUtil.NewContainer("nginx-cont"): {
langUtil.Language(languagemodels.Go): {},
langUtil.Language(languagemodels.Java): {},
langUtil.Language(languagemodels.Python): {},
},
},
},
deployment: &appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-deployment",
Namespace: "test-namespace",
Labels: map[string]string{
"test-label": "test-value",
},
Annotations: map[string]string{
"ignore-annotation": "ignore",
"internal.dd.datadoghq.com/nginx-cont.detected_langs": "go,java, python ",
"internal.dd.datadoghq.com/init.nginx-cont.detected_langs": "go,java, python ",
},
},
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
parser, err := newdeploymentParser(excludeAnnotations)
require.NoError(t, err)
entity := parser.Parse(tt.deployment)
storedDeployment, ok := entity.(*workloadmeta.KubernetesDeployment)
require.True(t, ok)
assert.Equal(t, tt.expected, storedDeployment)
})
}
}

func Test_DeploymentsFakeKubernetesClient(t *testing.T) {
tests := []struct {
name string
Expand Down
40 changes: 3 additions & 37 deletions comp/core/workloadmeta/collectors/internal/kubeapiserver/metadata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ package kubeapiserver

import (
"context"
"regexp"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -19,7 +18,7 @@ import (
"k8s.io/client-go/tools/cache"

"github.com/DataDog/datadog-agent/comp/core/config"
"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util"
kubernetesresourceparsers "github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors/util/kubernetes_resource_parsers"
workloadmeta "github.com/DataDog/datadog-agent/comp/core/workloadmeta/def"
"github.com/DataDog/datadog-agent/pkg/util/log"
)
Expand All @@ -35,10 +34,10 @@ func newMetadataStore(ctx context.Context, wlmetaStore workloadmeta.Component, c
}

annotationsExclude := config.GetStringSlice("cluster_agent.kube_metadata_collection.resource_annotations_exclude")
parser, err := newMetadataParser(gvr, annotationsExclude)
parser, err := kubernetesresourceparsers.NewMetadataParser(gvr, annotationsExclude)
if err != nil {
_ = log.Errorf("unable to parse all resource_annotations_exclude: %v, err:", err)
parser, _ = newMetadataParser(gvr, nil)
parser, _ = kubernetesresourceparsers.NewMetadataParser(gvr, nil)
}

metadataStore := &reflectorStore{
Expand All @@ -56,36 +55,3 @@ func newMetadataStore(ctx context.Context, wlmetaStore workloadmeta.Component, c
)
return metadataReflector, metadataStore
}

type metadataParser struct {
gvr *schema.GroupVersionResource
annotationsFilter []*regexp.Regexp
}

func newMetadataParser(gvr schema.GroupVersionResource, annotationsExclude []string) (objectParser, error) {
filters, err := parseFilters(annotationsExclude)
if err != nil {
return nil, err
}

return metadataParser{gvr: &gvr, annotationsFilter: filters}, nil
}

func (p metadataParser) Parse(obj interface{}) workloadmeta.Entity {
partialObjectMetadata := obj.(*metav1.PartialObjectMetadata)
id := util.GenerateKubeMetadataEntityID(p.gvr.Group, p.gvr.Resource, partialObjectMetadata.Namespace, partialObjectMetadata.Name)

return &workloadmeta.KubernetesMetadata{
EntityID: workloadmeta.EntityID{
Kind: workloadmeta.KindKubernetesMetadata,
ID: string(id),
},
EntityMeta: workloadmeta.EntityMeta{
Name: partialObjectMetadata.Name,
Namespace: partialObjectMetadata.Namespace,
Labels: partialObjectMetadata.Labels,
Annotations: filterMapStringKey(partialObjectMetadata.Annotations, p.annotationsFilter),
},
GVR: p.gvr,
}
}
Loading
Loading