From b64a1ac035e800eca2d139f95f9d23e5eabd6fc3 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Tue, 17 Dec 2024 12:20:53 +0100 Subject: [PATCH 1/2] start the health manager immediately Signed-off-by: Matthias Bertschy --- main.go | 17 +++++++++++------ pkg/healthmanager/health_manager.go | 15 +++++++++------ pkg/utils/utils.go | 2 +- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 01fd8cb2..575f84f6 100644 --- a/main.go +++ b/main.go @@ -13,6 +13,7 @@ import ( apitypes "github.com/armosec/armoapi-go/armotypes" utilsmetadata "github.com/armosec/utils-k8s-go/armometadata" mapset "github.com/deckarep/golang-set/v2" + "github.com/grafana/pyroscope-go" igconfig "github.com/inspektor-gadget/inspektor-gadget/pkg/config" containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" beUtils "github.com/kubescape/backend/pkg/utils" @@ -56,8 +57,6 @@ import ( "github.com/kubescape/node-agent/pkg/validator" "github.com/kubescape/node-agent/pkg/watcher/dynamicwatcher" "github.com/kubescape/node-agent/pkg/watcher/seccompprofilewatcher" - - pyroscope "github.com/grafana/pyroscope-go" ) func main() { @@ -117,12 +116,14 @@ func main() { if os.Getenv("APPLICATION_NAME") == "" { os.Setenv("APPLICATION_NAME", "node-agent") } + _, err := pyroscope.Start(pyroscope.Config{ ApplicationName: os.Getenv("APPLICATION_NAME"), ServerAddress: pyroscopeServerSvc, Logger: pyroscope.StandardLogger, Tags: map[string]string{"node": cfg.NodeName, "app": "node-agent", "pod": os.Getenv("POD_NAME")}, }) + if err != nil { logger.L().Ctx(ctx).Error("error starting pyroscope", helpers.Error(err)) } @@ -132,6 +133,10 @@ func main() { logger.L().Info("MULTIPLY environment variable is true. Multiplying feature enabled - this is a feature for testing purposes only") } + // Start the health manager + healthManager := healthmanager.NewHealthManager() + healthManager.Start(ctx) + // Create clients k8sClient := k8sinterface.NewKubernetesApi() storageClient, err := storage.CreateStorage(clusterData.Namespace) @@ -202,6 +207,9 @@ func main() { dnsResolver = dnsManager networkManagerClient = networkmanagerv2.CreateNetworkManager(ctx, cfg, clusterData.ClusterName, k8sClient, storageClient, dnsManager, preRunningContainersIDs, k8sObjectCache) } else { + if cfg.EnableRuntimeDetection { + logger.L().Ctx(ctx).Fatal("Network tracing is disabled, but runtime detection is enabled. Network tracing is required for runtime detection.") + } dnsManagerClient = dnsmanager.CreateDNSManagerMock() dnsResolver = dnsmanager.CreateDNSManagerMock() networkManagerClient = networkmanager.CreateNetworkManagerMock() @@ -307,6 +315,7 @@ func main() { if err != nil { logger.L().Ctx(ctx).Fatal("error creating the container watcher", helpers.Error(err)) } + healthManager.SetContainerWatcher(mainHandler) // Start the profileManager profileManager.Start(ctx) @@ -314,10 +323,6 @@ func main() { // Start the prometheusExporter prometheusExporter.Start() - // Start the health manager - healthManager := healthmanager.NewHealthManager(mainHandler) - healthManager.Start(ctx) - // Start the container handler err = mainHandler.Start(ctx) if err != nil { diff --git a/pkg/healthmanager/health_manager.go b/pkg/healthmanager/health_manager.go index f88a6795..35425022 100644 --- a/pkg/healthmanager/health_manager.go +++ b/pkg/healthmanager/health_manager.go @@ -17,13 +17,16 @@ type HealthManager struct { port int } -func NewHealthManager(containerWatcher *containerwatcher.IGContainerWatcher) *HealthManager { +func NewHealthManager() *HealthManager { return &HealthManager{ - containerWatcher: containerWatcher, - port: 7888, + port: 7888, } } +func (h *HealthManager) SetContainerWatcher(containerWatcher *containerwatcher.IGContainerWatcher) { + h.containerWatcher = containerWatcher +} + func (h *HealthManager) Start(ctx context.Context) { go func() { http.HandleFunc("/livez", h.livenessProbe) @@ -45,9 +48,9 @@ func (h *HealthManager) livenessProbe(w http.ResponseWriter, _ *http.Request) { } func (h *HealthManager) readinessProbe(w http.ResponseWriter, _ *http.Request) { - if h.containerWatcher.Ready() { + if h.containerWatcher != nil && h.containerWatcher.Ready() { w.WriteHeader(http.StatusOK) - } else { - w.WriteHeader(http.StatusInternalServerError) + return } + w.WriteHeader(http.StatusInternalServerError) } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 7deb2a5c..a4c6fcdf 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -704,7 +704,7 @@ func ChunkBy[T any](items []T, chunkSize int) [][]T { } // isUnixSocket checks if the given path is a Unix socket. - +// TODO remove this func DetectContainerRuntimeViaK8sAPI(ctx context.Context, k8sClient *k8sinterface.KubernetesApi, nodeName string) (*containerutilsTypes.RuntimeConfig, error) { // Get the current node nodes, err := k8sClient.GetKubernetesClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{ From 99a9a4f961862626e7bd69c1443a92d370b653d4 Mon Sep 17 00:00:00 2001 From: Matthias Bertschy Date: Wed, 18 Dec 2024 10:58:34 +0100 Subject: [PATCH 2/2] use RuntimeConfig detected by IG Signed-off-by: Matthias Bertschy --- go.mod | 6 +- go.sum | 8 +-- main.go | 13 +---- pkg/utils/utils.go | 142 --------------------------------------------- 4 files changed, 10 insertions(+), 159 deletions(-) diff --git a/go.mod b/go.mod index d0fecefe..9942e822 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,6 @@ require ( k8s.io/client-go v0.31.4 k8s.io/cri-api v0.31.4 k8s.io/kubectl v0.31.2 - k8s.io/kubelet v0.31.4 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/yaml v1.4.0 ) @@ -329,7 +328,7 @@ require ( google.golang.org/genproto v0.0.0-20240515191416-fc5f0ca64291 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/protobuf v1.36.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect @@ -343,6 +342,7 @@ require ( k8s.io/component-base v0.31.4 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f // indirect + k8s.io/kubelet v0.31.4 // indirect oras.land/oras-go/v2 v2.4.0 // indirect sigs.k8s.io/controller-runtime v0.19.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect @@ -353,6 +353,6 @@ require ( replace github.com/vishvananda/netns => github.com/inspektor-gadget/netns v0.0.5-0.20230524185006-155d84c555d6 -replace github.com/inspektor-gadget/inspektor-gadget => github.com/amitschendel/inspektor-gadget v0.0.0-20241217115506-b19f6755e077 +replace github.com/inspektor-gadget/inspektor-gadget => github.com/matthyx/inspektor-gadget v0.0.0-20241218095316-52e571ac842a replace github.com/mholt/archiver/v3 v3.5.1 => github.com/anchore/archiver/v3 v3.5.2 diff --git a/go.sum b/go.sum index cbc97db0..e46ea845 100644 --- a/go.sum +++ b/go.sum @@ -109,8 +109,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/amitschendel/inspektor-gadget v0.0.0-20241217115506-b19f6755e077 h1:Ao49WA5l/34GqqbcuOydURjsKS64hLS9bz9NUPe1ffQ= -github.com/amitschendel/inspektor-gadget v0.0.0-20241217115506-b19f6755e077/go.mod h1:4SoaWDOIDcCHyZO9zPubnnprTiVk/vSxyl2JSVwXmlY= github.com/anchore/archiver/v3 v3.5.2 h1:Bjemm2NzuRhmHy3m0lRe5tNoClB9A4zYyDV58PaB6aA= github.com/anchore/archiver/v3 v3.5.2/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= github.com/anchore/clio v0.0.0-20240522144804-d81e109008aa h1:pwlAn4O9SBUnlgfa69YcqIynbUyobLVFYu8HxSoCffA= @@ -720,6 +718,8 @@ github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3v github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matthyx/inspektor-gadget v0.0.0-20241218095316-52e571ac842a h1:AGbpiy9UBZ1I64vL/Y+6w5oPcxmSokU7pN3GxYx4D2I= +github.com/matthyx/inspektor-gadget v0.0.0-20241218095316-52e571ac842a/go.mod h1:NMlrft7MN0ElM4eUpOUMd686Sg/L43NEraBht3yyvww= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -1649,8 +1649,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= +google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 575f84f6..321075b4 100644 --- a/main.go +++ b/main.go @@ -152,14 +152,6 @@ func main() { prometheusExporter = metricsmanager.NewMetricsMock() } - // Detect the container containerRuntime of the node - containerRuntime, err := utils.DetectContainerRuntimeViaK8sAPI(ctx, k8sClient, cfg.NodeName) - if err != nil { - logger.L().Ctx(ctx).Fatal("error detecting the container runtime", helpers.Error(err)) - } - - logger.L().Ctx(ctx).Info("Detected container runtime", helpers.String("containerRuntime", containerRuntime.Name.String())) - // Create watchers dWatcher := dynamicwatcher.NewWatchHandler(k8sClient, storageClient.StorageClient, cfg.SkipNamespace) // create k8sObject cache @@ -298,11 +290,12 @@ func main() { } defer igK8sClient.Close() logger.L().Info("IG Kubernetes client created", helpers.Interface("client", igK8sClient)) + logger.L().Info("Detected container runtime", helpers.String("containerRuntime", igK8sClient.RuntimeConfig.Name.String())) // Create the SBOM manager var sbomManager sbommanager.SbomManagerClient if cfg.EnableSbomGeneration { - sbomManager, err = sbommanagerv1.CreateSbomManager(ctx, cfg, igK8sClient.RuntimeSocketPath, storageClient) + sbomManager, err = sbommanagerv1.CreateSbomManager(ctx, cfg, igK8sClient.RuntimeConfig.SocketPath, storageClient) if err != nil { logger.L().Ctx(ctx).Fatal("error creating SbomManager", helpers.Error(err)) } @@ -311,7 +304,7 @@ func main() { } // Create the container handler - mainHandler, err := containerwatcher.CreateIGContainerWatcher(cfg, applicationProfileManager, k8sClient, igK8sClient, networkManagerClient, dnsManagerClient, prometheusExporter, ruleManager, malwareManager, sbomManager, preRunningContainersIDs, &ruleBindingNotify, containerRuntime, nil, nil, processManager) + mainHandler, err := containerwatcher.CreateIGContainerWatcher(cfg, applicationProfileManager, k8sClient, igK8sClient, networkManagerClient, dnsManagerClient, prometheusExporter, ruleManager, malwareManager, sbomManager, preRunningContainersIDs, &ruleBindingNotify, igK8sClient.RuntimeConfig, nil, nil, processManager) if err != nil { logger.L().Ctx(ctx).Fatal("error creating the container watcher", helpers.Error(err)) } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a4c6fcdf..7d8fe7e8 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,12 +1,10 @@ package utils import ( - "context" "crypto/md5" "crypto/sha1" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "fmt" "hash" @@ -24,23 +22,17 @@ import ( "github.com/armosec/utils-k8s-go/wlid" mapset "github.com/deckarep/golang-set/v2" "github.com/goradd/maps" - runtimeclient "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/runtime-client" - containerutilsTypes "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/types" tracerexectype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/types" traceropentype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/open/types" - igtypes "github.com/inspektor-gadget/inspektor-gadget/pkg/types" "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" "github.com/kubescape/k8s-interface/instanceidhandler" - instanceidhandlerv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1" helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" - "github.com/kubescape/k8s-interface/k8sinterface" "github.com/kubescape/k8s-interface/workloadinterface" "github.com/prometheus/procfs" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" - kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" ) var ( @@ -366,19 +358,6 @@ func SetInMap(newExecMap *maps.SafeMap[string, mapset.Set[string]]) func(k strin } } -func ToInstanceType(c ContainerType) helpersv1.InstanceType { - switch c { - case Container: - return instanceidhandlerv1.Container - case InitContainer: - return instanceidhandlerv1.InitContainer - case EphemeralContainer: - return instanceidhandlerv1.EphemeralContainer - default: - return instanceidhandlerv1.Container - } -} - func GetCmdlineByPid(pid int) (*string, error) { fs, err := procfs.NewFS("/proc") if err != nil { @@ -703,127 +682,6 @@ func ChunkBy[T any](items []T, chunkSize int) [][]T { return append(chunks, items) } -// isUnixSocket checks if the given path is a Unix socket. -// TODO remove this -func DetectContainerRuntimeViaK8sAPI(ctx context.Context, k8sClient *k8sinterface.KubernetesApi, nodeName string) (*containerutilsTypes.RuntimeConfig, error) { - // Get the current node - nodes, err := k8sClient.GetKubernetesClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{ - FieldSelector: fmt.Sprintf("metadata.name=%s", nodeName), - }) - if err != nil { - return nil, fmt.Errorf("failed to list nodes: %v", err) - } - if len(nodes.Items) == 0 { - return nil, fmt.Errorf("no node found with name: %s", nodeName) - } - node := nodes.Items[0] - // parse the runtime info - runtimeConfig := parseRuntimeInfo(node.Status.NodeInfo.ContainerRuntimeVersion) - if runtimeConfig.Name == igtypes.RuntimeNameUnknown { - return nil, fmt.Errorf("unknown container runtime: %s", node.Status.NodeInfo.ContainerRuntimeVersion) - } - // override the socket path - realSocketPath, err := getContainerRuntimeSocketPath(k8sClient, nodeName) - if err != nil { - logger.L().Warning("failed to get container runtime socket path from Kubelet configz", helpers.String("error", err.Error())) - } else { - runtimeConfig.SocketPath = realSocketPath - } - // unset the runtime protocol - runtimeConfig.RuntimeProtocol = "" - return runtimeConfig, nil -} - -func parseRuntimeInfo(version string) *containerutilsTypes.RuntimeConfig { - switch { - case strings.HasPrefix(version, "docker://"): - return &containerutilsTypes.RuntimeConfig{ - Name: igtypes.RuntimeNameDocker, - SocketPath: runtimeclient.DockerDefaultSocketPath, - RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, - } - case strings.HasPrefix(version, "containerd://"): - return &containerutilsTypes.RuntimeConfig{ - Name: igtypes.RuntimeNameContainerd, - SocketPath: runtimeclient.ContainerdDefaultSocketPath, - RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, - } - case strings.HasPrefix(version, "cri-o://"): - return &containerutilsTypes.RuntimeConfig{ - Name: igtypes.RuntimeNameCrio, - SocketPath: runtimeclient.CrioDefaultSocketPath, - RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, - } - case strings.HasPrefix(version, "podman://"): - return &containerutilsTypes.RuntimeConfig{ - Name: igtypes.RuntimeNamePodman, - SocketPath: runtimeclient.PodmanDefaultSocketPath, - RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, - } - default: - return &containerutilsTypes.RuntimeConfig{ - Name: igtypes.RuntimeNameUnknown, - SocketPath: "", - RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, - } - } -} - -func getContainerRuntimeSocketPath(clientset *k8sinterface.KubernetesApi, nodeName string) (string, error) { - kubeletConfig, err := getCurrentKubeletConfig(clientset, nodeName) - if err != nil { - return "", fmt.Errorf("getting /configz: %w", err) - } - - endpoint := kubeletConfig.ContainerRuntimeEndpoint - socketPath := endpoint - - // If it starts with unix://, strip the prefix - if strings.HasPrefix(endpoint, "unix://") { - socketPath = strings.TrimPrefix(endpoint, "unix://") - } - - if socketPath == "" { - return "", fmt.Errorf("container runtime socket path is empty") - } - - logger.L().Info("using the detected container runtime socket path from Kubelet's config", - helpers.String("socketPath", socketPath)) - return socketPath, nil -} - -// The /configz endpoint isn't officially documented. It was introduced in Kubernetes 1.26 and been around for a long time -// as stated in https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/component-base/configz/OWNERS -func getCurrentKubeletConfig(clientset *k8sinterface.KubernetesApi, nodeName string) (*kubeletconfigv1beta1.KubeletConfiguration, error) { - resp, err := clientset.GetKubernetesClient().CoreV1().RESTClient().Get().Resource("nodes"). - Name(nodeName).Suffix("proxy", "configz").DoRaw(context.TODO()) - if err != nil { - return nil, fmt.Errorf("fetching /configz from %q: %w", nodeName, err) - } - kubeCfg, err := decodeConfigz(resp) - if err != nil { - return nil, err - } - return kubeCfg, nil -} - -// Decodes the http response from /configz and returns the kubelet configuration -func decodeConfigz(respBody []byte) (*kubeletconfigv1beta1.KubeletConfiguration, error) { - // This hack because /configz reports the following structure: - // {"kubeletconfig": {the JSON representation of kubeletconfigv1beta1.KubeletConfiguration}} - type configzWrapper struct { - ComponentConfig kubeletconfigv1beta1.KubeletConfiguration `json:"kubeletconfig"` - } - - configz := configzWrapper{} - err := json.Unmarshal(respBody, &configz) - if err != nil { - return nil, err - } - - return &configz.ComponentConfig, nil -} - func DiskUsage(path string) int64 { var s int64 dir, err := os.Open(path)