Fix: coredump command for different runtimes (#24)

* feat(kubernetes): split ContainerInfo type to separate types

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* feat(dumper): add draft for coredump working implementation for different runtime

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* feat(flags): extract custom type for dotnet frameworks location

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* chore(dumper): extract method with fs actions

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* fix(dumper): add more logging & fix execution

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* chore(*): inline scripts

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* docs(*): update readme

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

* chore(*): add default values for makefile

Signed-off-by: ArtemTrofimushkin <artemtrofimushkin@gmail.com>

Makefile

@@ -44,20 +44,33 @@ test: test-unit test-integration
 
 .PHONY: test-unit
 test-unit:
-	TEST_RUN_ARGS="$(TEST_RUN_ARGS)" TEST_DIR="$(TEST_DIR)" ./hacks/run-unit-tests.sh
+	go test \
+      -v \
+      -race \
+      -cover \
+      -coverprofile cover.out \
+      -timeout 30s \
+      ./...
 
+NODE_VERSION ?= v1.21.1
 .PHONY: test-integration-setup
 test-integration-setup:
-	kind create cluster --name "kind"
+	kind create cluster --name "kind" --image=kindest/node:$(NODE_VERSION)
 
+FRAMEWORK ?= net6.0
 .PHONY: test-integration-prepare
 test-integration-prepare:
 	./hacks/prepare-integration-tests.sh "$(CURRENT_DIR)" "kind-kind" "$(ARCH)" "$(FRAMEWORK)"
 
 .PHONY: test-integration
-test-integration:
-	./hacks/prepare-integration-tests.sh "$(CURRENT_DIR)" "kind-kind" "$(ARCH)" "$(FRAMEWORK)"
-	./hacks/run-integration-tests.sh "$(FRAMEWORK)"
+test-integration: test-integration-prepare
+	go test \
+      -v \
+      -ldflags="-X github.com/dodopizza/kubectl-shovel/test/integration_test.TargetContainerImage=kubectl-shovel/sample-integration-tests:$(FRAMEWORK)" \
+      -parallel 1 \
+      -timeout 600s \
+      --tags=integration \
+      ./test/integration/...
 
 .PHONY: help
 help:

README.md

@@ -67,8 +67,13 @@ You can find more info and examples in [cli documentation](./cli/docs/kubectl-sh
 
 ## How it works
 
-It runs the job with specified tool on the specified pod's node and mount its `/tmp` folder with dotnet-diagnostic socket.
-So it requires permissions to get pods and create jobs and allowance to mount `/var/lib/docker` path for docker runtime and `/run/containerd` for containerd from a host in read-only mode.
+* For `dotnet-trace`, `dotnet-gcdump`, `dotnet-dump` commands it runs job on pod's node and mounts 
+  * `/tmp` folder with dotnet-diagnostic socket 
+  * `docker` or `containerd` runtime folders (`/var/lib/docker` or `/run/containerd`)
+* For `coredump` command it runs privileged job (with `SYS_PTRACE` capability) on pod's node and mounts
+  * `/tmp` folder with dotnet-diagnostic socket
+  * `docker` or `containerd` runtime folders (`/var/lib/docker` or `/run/containerd`)
+  * `/proc` folder mounted from host to find host process id for container
 
 ## Development
 
@@ -83,6 +88,7 @@ make prepare
 * golang
 * docker
 * kind
+* dotnet sdk
 
 ### Testing
 
@@ -96,7 +102,8 @@ make test-unit
 
 > kind-clusters use containerd as container runtime, so functionality with docker-runtime won't be covered.
 
-* Integration tests require running kind-cluster. You can create it with `make setup`. Also you can specify some version for cluster: `kind create cluster --image=kindest/node:<version>`, e.g v1.19.1 version.
+* Integration tests require running kind-cluster. You can create it with `make test-integration-setup`. 
+Also you can specify some version for cluster: `kind create cluster --image=kindest/node:<version>`, e.g v1.19.1 version.
 * Then run integration tests with `make test-integration`. It will:
   * Build docker image for dumper
   * Upload it to kind-cluster

cli/cmd/launch.go

@@ -119,14 +119,14 @@ func (cb *CommandBuilder) launch() error {
 		return errors.Wrap(err, "Failed to start diagnostics job")
 	}
 
-	jobPodLogs, err := cb.kube.ReadPodLogs(jobPod.Name, globals.PluginName)
+	logs, err := cb.kube.ReadPodLogs(jobPod.Name, globals.PluginName)
 	if err != nil {
 		return errors.Wrap(err, "Failed to read logs from diagnostics job targetPod")
 	}
-	defer utils.Ignore(jobPodLogs.Close)
+	defer utils.Ignore(logs.Close)
 
 	awaiter := events.NewEventAwaiter()
-	output, err := awaiter.AwaitCompletedEvent(jobPodLogs)
+	output, err := awaiter.AwaitCompletedEvent(logs)
 	if err != nil {
 		return errors.Wrap(err, "Failed to complete diagnostics job")
 	}

dumper/cmd/launch.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -17,41 +16,72 @@ import (
 	"github.com/dodopizza/kubectl-shovel/internal/watchdog"
 )
 
-func (cb *CommandBuilder) launch() error {
-	events.NewStatusEvent("Looking for and mapping container fs")
+func (cb *CommandBuilder) prepareFS(container *kubernetes.ContainerConfigInfo) error {
+	events.NewStatusEvent("Looking for and mapping container /tmp")
 
-	container := kubernetes.NewContainerInfoRaw(cb.CommonOptions.ContainerRuntime, cb.CommonOptions.ContainerID)
 	// remove /tmp directory,
-	// because will be mounted either from rootfs or container mounts
+	// because it will be mounted from container /tmp directory
 	if err := os.RemoveAll(globals.PathTmpFolder); err != nil {
 		return err
 	}
 
-	tmpSource, err := container.GetTmpSource()
+	// for dotnet tools, in /tmp folder must exists sockets to running dotnet apps
+	// https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md#diagnostic-ipc-protocol
+	if err := os.Symlink(container.GetTmpSource(), globals.PathTmpFolder); err != nil {
+		return err
+	}
+
+	if !cb.tool.IsPrivileged() {
+		return nil
+	}
+
+	// for privileged commands link framework runtime libs to root
+	events.NewStatusEvent("Looking for and mapping container runtime libs")
+	resolver := flags.NewDotnetToolResolver(container.RootFS)
+	frameworks, err := resolver.LocateFrameworks()
 	if err != nil {
-		events.NewErrorEvent(err, "unable to find mount point for container")
 		return err
 	}
 
-	// for dotnet tools, in /tmp folder must exists sockets to running dotnet apps
-	// https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md#diagnostic-ipc-protocol
-	if err := os.Symlink(tmpSource, globals.PathTmpFolder); err != nil {
-		events.NewErrorEvent(err, "unable to mount tmp folder for container")
+	for _, framework := range frameworks {
+		if framework.Name != flags.DotnetFrameworkApp {
+			continue
+		}
+
+		source := framework.FullPath()
+		destination := filepath.Join(resolver.Path, framework.NameVersion())
+
+		if utils.FileExists(destination) {
+			continue
+		}
+
+		if err := os.Symlink(source, destination); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (cb *CommandBuilder) launch() error {
+	container, err := kubernetes.NewContainerConfigInfo(cb.CommonOptions.ContainerRuntime, cb.CommonOptions.ContainerID)
+	if err != nil {
+		events.NewErrorEvent(err, "unable to locate container configuration")
+		return err
+	}
+
+	if err := cb.prepareFS(container); err != nil {
+		events.NewErrorEvent(err, "unable to prepare job file system for command execution")
 		return err
 	}
 
 	// write output file to /tmp, because it's available in target and worker pods
 	output := fmt.Sprintf("%s/output.%s", globals.PathTmpFolder, cb.tool.ToolName())
 	cb.tool.SetOutput(output)
 
-	// resolve host process id and set for privileged commands
 	if cb.tool.IsPrivileged() {
-		processID, err := container.GetHostProcessID()
-		if err != nil {
-			events.NewErrorEvent(err, "unable to find process id")
-			return err
-		}
-		cb.tool.SetProcessID(processID)
+		// host process required for privileged commands
+		cb.tool.SetProcessID(container.HostProcessID)
 	}
 
 	args := flags.NewArgs()
@@ -70,9 +100,8 @@ func (cb *CommandBuilder) launch() error {
 	}
 	events.NewStatusEvent("Gathering completed")
 
-	_, err = ioutil.ReadFile(output)
-	if err != nil {
-		events.NewErrorEvent(err, "failed to locate output result")
+	if !utils.FileExists(output) {
+		events.NewErrorEvent(fmt.Errorf("failed to locate execution result at: %s", output), "")
 		return err
 	}
 

internal/flags/dotnet_tool_resolver.go

@@ -0,0 +1,70 @@
+package flags
+
+import (
+	"os"
+	"path/filepath"
+)
+
+const (
+	DotnetFrameworkApp = "Microsoft.NETCore.App"
+)
+
+type DotnetToolResolver struct {
+	Root string
+	Path string
+}
+
+type DotnetFramework struct {
+	Root    string
+	Name    string
+	Version string
+}
+
+func NewDotnetToolResolver(root string) *DotnetToolResolver {
+	return &DotnetToolResolver{
+		Root: root,
+		Path: "/usr/share/dotnet/shared",
+	}
+}
+
+func (r *DotnetToolResolver) LocateFrameworks() ([]DotnetFramework, error) {
+	entries, err := os.ReadDir(r.FullPath())
+	if err != nil {
+		return nil, err
+	}
+
+	var frameworks []DotnetFramework
+	for _, entry := range entries {
+		framework := entry.Name()
+		frameworkPath := filepath.Join(r.FullPath(), framework)
+
+		entries, err := os.ReadDir(frameworkPath)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, entry := range entries {
+			frameworks = append(frameworks,
+				DotnetFramework{
+					Name:    framework,
+					Root:    r.FullPath(),
+					Version: entry.Name(),
+				},
+			)
+		}
+	}
+
+	return frameworks, nil
+}
+
+func (r *DotnetToolResolver) FullPath() string {
+	return filepath.Join(r.Root, r.Path)
+}
+
+func (f *DotnetFramework) FullPath() string {
+	return filepath.Join(f.Root, f.Name, f.Version)
+}
+
+func (f *DotnetFramework) NameVersion() string {
+	return filepath.Join(f.Name, f.Version)
+}