From fc9726fe0f494a7362fa6d43cbac5fbc6b9f4930 Mon Sep 17 00:00:00 2001 From: Filip Nikolic Date: Thu, 9 May 2024 12:06:25 +0200 Subject: [PATCH] oci-hook: allow users to set namespace exceptions and define default Prior to this commit `kube-system` was always added as an exception in order for Pods in that namespace to be created in the event that the OCI hook was not able to reach the Tetragon agent. This leads to a deadlock scenario when Tetragon itself is not installed in `kube-system` as the agent won't start and leave the cluster in an unhealthy state. Instead of always adding `kube-syste` this change will always add the namespace Tetragon is deployed in, to guarantee that the agent can always start. In addition to that a user can now define additional namespaces as further exceptions. For example to ensure Pods in business-critical namespaces can still be created even if the OCI hook fails to reach the Tetragon agent. Signed-off-by: Filip Nikolic --- contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go | 5 +++-- contrib/rthooks/tetragon-oci-hook/docs/demo.md | 2 +- contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml | 1 - .../tetragon/templates/_container_oci_hooks_setup.tpl | 1 + install/kubernetes/tetragon/values.yaml | 2 ++ 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go b/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go index 9a6efbf4505..6d26ad03187 100644 --- a/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go +++ b/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go @@ -28,7 +28,8 @@ type Install struct { HostInstallDir string `required help:"Installation dir (in the host). Used for the binary and the hook logfile."` OciHooks struct { - LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"` + LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"` + FailAllowNamespaces string `help:"Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent."` } `embed:"" prefix:"oci-hooks."` } @@ -60,7 +61,7 @@ func (i *Install) ociHooksInstall(log *logrus.Logger) { binFname := filepath.Join(i.HostInstallDir, binBaseName) logFname := filepath.Join(i.HostInstallDir, logBaseName) - hook := ociHooksConfig(binFname, "--log-fname", logFname) + hook := ociHooksConfig(binFname, "--log-fname", logFname, "--fail-allow-namespaces", i.OciHooks.FailAllowNamespaces) data, err := json.MarshalIndent(hook, "", " ") if err != nil { log.WithError(err).Fatal("failed to unmarshall hook info") diff --git a/contrib/rthooks/tetragon-oci-hook/docs/demo.md b/contrib/rthooks/tetragon-oci-hook/docs/demo.md index 3f1d4806f2d..161b71e639d 100644 --- a/contrib/rthooks/tetragon-oci-hook/docs/demo.md +++ b/contrib/rthooks/tetragon-oci-hook/docs/demo.md @@ -29,7 +29,7 @@ helm install --namespace kube-system \ --set tetragon.image.override=localhost/cilium/tetragon:latest \ --set tetragon.grpc.address="unix:///var/run/cilium/tetragon/tetragon.sock" \ --set tetragon.ociHookSetup.enabled=true \ - tetragon ./install/kubernetes + tetragon ./install/kubernetes/tetragon ... kubectl logs -n kube-system tetragon-289tf -c oci-hook-setup time="2023-12-05T09:28:50Z" level=info msg="written binary" hook-dst-path=/hostInstall/tetragon-oci-hook diff --git a/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml b/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml index 491f4d00bd4..0817c778764 100644 --- a/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml +++ b/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: tetragon-oci-hook-uninstall - namespace: kube-system labels: k8s-app: tetragon-oci-hook-setup-test spec: diff --git a/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl b/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl index 895277d9bbe..23f8d47ac94 100644 --- a/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl +++ b/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl @@ -11,6 +11,7 @@ - --local-install-dir={{ include "container.tetragonOCIHookSetup.installPath" . }} - --host-install-dir={{ .Values.tetragon.ociHookSetup.installDir }} - --oci-hooks.local-dir={{ include "container.tetragonOCIHookSetup.hooksPath" . }} + - --oci-hooks.fail-allow-namespaces={{ if .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ printf "%s,%s" .Release.Namespace .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ else }}{{ .Release.Namespace }}{{ end }} volumeMounts: {{- with .Values.tetragon.ociHookSetup.extraVolumeMounts }} {{- toYaml . | nindent 4 }} diff --git a/install/kubernetes/tetragon/values.yaml b/install/kubernetes/tetragon/values.yaml index d8a8a4c5bfb..0bc6292c5d7 100644 --- a/install/kubernetes/tetragon/values.yaml +++ b/install/kubernetes/tetragon/values.yaml @@ -200,6 +200,8 @@ tetragon: # "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). interface: "oci-hooks" installDir: "/opt/tetragon" + # -- Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. + failAllowNamespaces: "" # -- Security context for oci-hook-setup init container securityContext: privileged: true