From edd45fd85ae56189e1225aed6414727b32c3d0f5 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Fri, 9 Aug 2024 09:19:20 +0000 Subject: [PATCH 1/3] Adding enrichment of WithContainerRuntimeEnrichment Signed-off-by: Amit Schendel --- main.go | 11 +- pkg/containerwatcher/v1/container_watcher.go | 8 +- .../v1/container_watcher_private.go | 19 +-- pkg/containerwatcher/v1/open_test.go | 2 +- pkg/rulemanager/v1/rule_manager.go | 2 +- pkg/utils/utils.go | 108 ++++++++++++++++++ 6 files changed, 138 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 94276247..3591b7ab 100644 --- a/main.go +++ b/main.go @@ -123,6 +123,15 @@ func main() { } nodeName := os.Getenv(config.NodeNameEnvVar) + + // Detect the container containerRuntime of the node + containerRuntime, err := utils.DetectContainerRuntimeViaK8sAPI(ctx, k8sClient, nodeName) + if err != nil { + logger.L().Ctx(ctx).Fatal("error detecting the container runtime", helpers.Error(err)) + } + + logger.L().Ctx(ctx).Info("Detected container runtime", helpers.String("containerRuntime", containerRuntime.Name.String())) + // Create watchers dWatcher := dynamicwatcher.NewWatchHandler(k8sClient, cfg.SkipNamespace) // create k8sObject cache @@ -251,7 +260,7 @@ func main() { } // Create the container handler - mainHandler, err := containerwatcher.CreateIGContainerWatcher(cfg, applicationProfileManager, k8sClient, relevancyManager, networkManagerv1Client, networkManagerClient, dnsManagerClient, prometheusExporter, ruleManager, malwareManager, preRunningContainersIDs, &ruleBindingNotify) + mainHandler, err := containerwatcher.CreateIGContainerWatcher(cfg, applicationProfileManager, k8sClient, relevancyManager, networkManagerv1Client, networkManagerClient, dnsManagerClient, prometheusExporter, ruleManager, malwareManager, preRunningContainersIDs, &ruleBindingNotify, containerRuntime) if err != nil { logger.L().Ctx(ctx).Fatal("error creating the container watcher", helpers.Error(err)) } diff --git a/pkg/containerwatcher/v1/container_watcher.go b/pkg/containerwatcher/v1/container_watcher.go index 0adcb016..00683a3d 100644 --- a/pkg/containerwatcher/v1/container_watcher.go +++ b/pkg/containerwatcher/v1/container_watcher.go @@ -7,6 +7,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" containercollection "github.com/inspektor-gadget/inspektor-gadget/pkg/container-collection" + containerutilsTypes "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/types" tracerseccomp "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/advise/seccomp/tracer" tracercapabilities "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/capabilities/tracer" tracercapabilitiestype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/capabilities/types" @@ -127,11 +128,14 @@ type IGContainerWatcher struct { // cache ruleBindingPodNotify *chan rulebinding.RuleBindingNotify + + // container runtime + runtime *containerutilsTypes.RuntimeConfig } var _ containerwatcher.ContainerWatcher = (*IGContainerWatcher)(nil) -func CreateIGContainerWatcher(cfg config.Config, applicationProfileManager applicationprofilemanager.ApplicationProfileManagerClient, k8sClient *k8sinterface.KubernetesApi, relevancyManager relevancymanager.RelevancyManagerClient, networkManagerv1Client networkmanagerv1.NetworkManagerClient, networkManagerClient networkmanager.NetworkManagerClient, dnsManagerClient dnsmanager.DNSManagerClient, metrics metricsmanager.MetricsManager, ruleManager rulemanager.RuleManagerClient, malwareManager malwaremanager.MalwareManagerClient, preRunningContainers mapset.Set[string], ruleBindingPodNotify *chan rulebinding.RuleBindingNotify) (*IGContainerWatcher, error) { +func CreateIGContainerWatcher(cfg config.Config, applicationProfileManager applicationprofilemanager.ApplicationProfileManagerClient, k8sClient *k8sinterface.KubernetesApi, relevancyManager relevancymanager.RelevancyManagerClient, networkManagerv1Client networkmanagerv1.NetworkManagerClient, networkManagerClient networkmanager.NetworkManagerClient, dnsManagerClient dnsmanager.DNSManagerClient, metrics metricsmanager.MetricsManager, ruleManager rulemanager.RuleManagerClient, malwareManager malwaremanager.MalwareManagerClient, preRunningContainers mapset.Set[string], ruleBindingPodNotify *chan rulebinding.RuleBindingNotify, runtime *containerutilsTypes.RuntimeConfig) (*IGContainerWatcher, error) { // Use container collection to get notified for new containers containerCollection := &containercollection.ContainerCollection{} // Create a tracer collection instance @@ -337,6 +341,8 @@ func CreateIGContainerWatcher(cfg config.Config, applicationProfileManager appli timeBasedContainers: mapset.NewSet[string](), ruleManagedPods: mapset.NewSet[string](), + + runtime: runtime, }, nil } diff --git a/pkg/containerwatcher/v1/container_watcher_private.go b/pkg/containerwatcher/v1/container_watcher_private.go index bbcca9d1..b7e44377 100644 --- a/pkg/containerwatcher/v1/container_watcher_private.go +++ b/pkg/containerwatcher/v1/container_watcher_private.go @@ -85,25 +85,28 @@ func (ch *IGContainerWatcher) startContainerCollection(ctx context.Context) erro // Define the different options for the container collection instance opts := []containercollection.ContainerCollectionOption{ - containercollection.WithTracerCollection(ch.tracerCollection), + // Get Notifications from the container collection + containercollection.WithPubSub(containerEventFuncs...), // Enrich events with OCI config information containercollection.WithOCIConfigEnrichment(), - // Get containers created with ebpf (works also if hostPid=false) - containercollection.WithContainerFanotifyEbpf(), - - // Get containers created with docker + // Get containers enriched with cgroup information containercollection.WithCgroupEnrichment(), // Enrich events with Linux namespaces information, it is needed for per container filtering containercollection.WithLinuxNamespaceEnrichment(), + // Get containers created with container runtimes + containercollection.WithContainerRuntimeEnrichment(ch.runtime), + + // Get containers created with ebpf (works also if hostPid=false) + containercollection.WithContainerFanotifyEbpf(), + + containercollection.WithTracerCollection(ch.tracerCollection), + // Enrich those containers with data from the Kubernetes API containercollection.WithKubernetesEnrichment(ch.nodeName, ch.k8sClient.K8SConfig), - - // Get Notifications from the container collection - containercollection.WithPubSub(containerEventFuncs...), } // Initialize the container collection diff --git a/pkg/containerwatcher/v1/open_test.go b/pkg/containerwatcher/v1/open_test.go index 0d3a9f18..7899a63a 100644 --- a/pkg/containerwatcher/v1/open_test.go +++ b/pkg/containerwatcher/v1/open_test.go @@ -23,7 +23,7 @@ func BenchmarkIGContainerWatcher_openEventCallback(b *testing.B) { assert.NoError(b, err) mockExporter := metricsmanager.NewMetricsMock() - mainHandler, err := CreateIGContainerWatcher(cfg, nil, nil, relevancyManager, nil, nil, nil, mockExporter, nil, nil, nil, nil) + mainHandler, err := CreateIGContainerWatcher(cfg, nil, nil, relevancyManager, nil, nil, nil, mockExporter, nil, nil, nil, nil, nil) assert.NoError(b, err) event := &traceropentype.Event{ Event: types.Event{ diff --git a/pkg/rulemanager/v1/rule_manager.go b/pkg/rulemanager/v1/rule_manager.go index 7cd50fae..848bd250 100644 --- a/pkg/rulemanager/v1/rule_manager.go +++ b/pkg/rulemanager/v1/rule_manager.go @@ -114,7 +114,7 @@ func (rm *RuleManager) monitorContainer(ctx context.Context, container *containe select { case <-syscallTicker.C: if rm.syscallPeekFunc == nil { - logger.L().Error("RuleManager - syscallPeekFunc is not set", helpers.String("container ID", watchedContainer.ContainerID)) + logger.L().Debug("RuleManager - syscallPeekFunc is not set", helpers.String("container ID", watchedContainer.ContainerID)) continue } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 383eacb8..b5ee4c91 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,6 +1,7 @@ package utils import ( + "context" "crypto/md5" "crypto/sha1" "crypto/sha256" @@ -15,6 +16,7 @@ import ( "slices" "strconv" "strings" + "syscall" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -28,6 +30,7 @@ import ( "github.com/kubescape/k8s-interface/instanceidhandler" instanceidhandlerv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1" helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/k8s-interface/k8sinterface" "github.com/kubescape/k8s-interface/workloadinterface" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" v1 "k8s.io/api/core/v1" @@ -35,8 +38,11 @@ import ( "github.com/prometheus/procfs" + runtimeclient "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/runtime-client" + containerutilsTypes "github.com/inspektor-gadget/inspektor-gadget/pkg/container-utils/types" tracerexectype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/exec/types" traceropentype "github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/open/types" + igtypes "github.com/inspektor-gadget/inspektor-gadget/pkg/types" apitypes "github.com/armosec/armoapi-go/armotypes" ) @@ -703,3 +709,105 @@ func ChunkBy[T any](items []T, chunkSize int) [][]T { } return append(chunks, items) } + +// isUnixSocket checks if the given path is a Unix socket. +func isUnixSocket(path string) (bool, error) { + fileInfo, err := os.Stat(path) + if err != nil { + return false, err // Could not obtain the file stats + } + + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return false, fmt.Errorf("not a unix file") + } + + // Check if the file is a socket + return (stat.Mode & syscall.S_IFMT) == syscall.S_IFSOCK, nil +} + +func DetectContainerRuntimes(hostMount string) ([]*containerutilsTypes.RuntimeConfig, error) { + runtimes := map[igtypes.RuntimeName]string{ + igtypes.RuntimeNameDocker: runtimeclient.DockerDefaultSocketPath, + igtypes.RuntimeNameCrio: runtimeclient.CrioDefaultSocketPath, + igtypes.RuntimeNameContainerd: runtimeclient.ContainerdDefaultSocketPath, + igtypes.RuntimeNamePodman: runtimeclient.PodmanDefaultSocketPath, + } + + detectedRuntimes := make([]*containerutilsTypes.RuntimeConfig, 0) + + for runtimeName, socketPath := range runtimes { + // Check if the socket is available on the host mount + socketPath = hostMount + socketPath + if isSocket, err := isUnixSocket(socketPath); err == nil && isSocket { + logger.L().Info("Detected container runtime", helpers.String("runtime", runtimeName.String()), helpers.String("socketPath", socketPath)) + detectedRuntimes = append(detectedRuntimes, &containerutilsTypes.RuntimeConfig{ + Name: runtimeName, + SocketPath: socketPath, + }) + } + } + + if len(detectedRuntimes) == 0 { + return nil, fmt.Errorf("no container runtimes detected at the following paths: %v", runtimes) + } + + return detectedRuntimes, nil +} + +func DetectContainerRuntimeViaK8sAPI(ctx context.Context, k8sClient *k8sinterface.KubernetesApi, nodeName string) (*containerutilsTypes.RuntimeConfig, error) { + // Get the current node + nodes, err := k8sClient.GetKubernetesClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{ + FieldSelector: fmt.Sprintf("metadata.name=%s", nodeName), + }) + if err != nil { + return nil, fmt.Errorf("failed to list nodes: %v", err) + } + + if len(nodes.Items) == 0 { + return nil, fmt.Errorf("no node found with name: %s", nodeName) + } + + node := nodes.Items[0] + runtimeConfig := parseRuntimeInfo(node.Status.NodeInfo.ContainerRuntimeVersion) + if runtimeConfig.Name == igtypes.RuntimeNameUnknown { + return nil, fmt.Errorf("unknown container runtime: %s", node.Status.NodeInfo.ContainerRuntimeVersion) + } + + return runtimeConfig, nil +} + +func parseRuntimeInfo(version string) *containerutilsTypes.RuntimeConfig { + switch { + case strings.HasPrefix(version, "docker://"): + return &containerutilsTypes.RuntimeConfig{ + Name: igtypes.RuntimeNameDocker, + SocketPath: runtimeclient.DockerDefaultSocketPath, + RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, + } + case strings.HasPrefix(version, "containerd://"): + return &containerutilsTypes.RuntimeConfig{ + Name: igtypes.RuntimeNameContainerd, + SocketPath: runtimeclient.ContainerdDefaultSocketPath, + RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, + } + case strings.HasPrefix(version, "cri-o://"): + return &containerutilsTypes.RuntimeConfig{ + Name: igtypes.RuntimeNameCrio, + SocketPath: runtimeclient.CrioDefaultSocketPath, + RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, + } + case strings.HasPrefix(version, "podman://"): + return &containerutilsTypes.RuntimeConfig{ + Name: igtypes.RuntimeNamePodman, + SocketPath: runtimeclient.PodmanDefaultSocketPath, + RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, + } + default: + return &containerutilsTypes.RuntimeConfig{ + Name: igtypes.RuntimeNameUnknown, + SocketPath: "", + RuntimeProtocol: containerutilsTypes.RuntimeProtocolCRI, + } + } +} From 44ea61233b8c1028eec012317c1a2fc27a04ef83 Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Fri, 9 Aug 2024 09:43:52 +0000 Subject: [PATCH 2/3] Adding arm64 to build Signed-off-by: Amit Schendel --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8155c923..aca43754 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ binary: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME) docker-build: - docker buildx build --platform linux/amd64 -t $(IMAGE):$(TAG) -f $(DOCKERFILE_PATH) . + docker buildx build --platform linux/amd64,linux/arm64 -t $(IMAGE):$(TAG) -f $(DOCKERFILE_PATH) . + docker-push: docker push $(IMAGE):$(TAG) From b0bbd4f1485a6d8fc089a7f41b4b89d93fe8f38c Mon Sep 17 00:00:00 2001 From: Amit Schendel Date: Fri, 9 Aug 2024 09:47:02 +0000 Subject: [PATCH 3/3] Fixing build Signed-off-by: Amit Schendel --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index aca43754..cdfc6ddd 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ binary: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o $(BINARY_NAME) docker-build: - docker buildx build --platform linux/amd64,linux/arm64 -t $(IMAGE):$(TAG) -f $(DOCKERFILE_PATH) . + docker buildx build --platform linux/amd64 -t $(IMAGE):$(TAG) -f $(DOCKERFILE_PATH) . docker-push: docker push $(IMAGE):$(TAG)