From 276c5b4008ba296650dffd20048765e3460ef017 Mon Sep 17 00:00:00 2001 From: Filip Nikolic Date: Mon, 13 May 2024 16:02:00 +0200 Subject: [PATCH] oci-hook: allow users to set namespace exceptions and define default Prior to this commit `kube-system` was always added as an exception in order for Pods in that namespace to be created in the event that the OCI hook was not able to reach the Tetragon agent. This leads to a deadlock scenario when Tetragon itself is not installed in `kube-system` as the agent won't start and leave the cluster in an unhealthy state. Instead of always adding `kube-syste` this change will always add the namespace Tetragon is deployed in, to guarantee that the agent can always start. In addition to that a user can now define additional namespaces as further exceptions. For example to ensure Pods in business-critical namespaces can still be created even if the OCI hook fails to reach the Tetragon agent. Signed-off-by: Filip Nikolic --- contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go | 5 +++-- contrib/rthooks/tetragon-oci-hook/docs/demo.md | 2 +- contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml | 1 - docs/content/en/docs/reference/helm-chart.md | 3 ++- install/kubernetes/tetragon/README.md | 3 ++- .../tetragon/templates/_container_oci_hooks_setup.tpl | 1 + install/kubernetes/tetragon/values.yaml | 3 +++ 7 files changed, 12 insertions(+), 6 deletions(-) diff --git a/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go b/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go index 9a6efbf4505..6d26ad03187 100644 --- a/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go +++ b/contrib/rthooks/tetragon-oci-hook/cmd/setup/main.go @@ -28,7 +28,8 @@ type Install struct { HostInstallDir string `required help:"Installation dir (in the host). Used for the binary and the hook logfile."` OciHooks struct { - LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"` + LocalDir string `default:"/hostHooks" help:"oci-hooks drop-in directory (inside the container)"` + FailAllowNamespaces string `help:"Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent."` } `embed:"" prefix:"oci-hooks."` } @@ -60,7 +61,7 @@ func (i *Install) ociHooksInstall(log *logrus.Logger) { binFname := filepath.Join(i.HostInstallDir, binBaseName) logFname := filepath.Join(i.HostInstallDir, logBaseName) - hook := ociHooksConfig(binFname, "--log-fname", logFname) + hook := ociHooksConfig(binFname, "--log-fname", logFname, "--fail-allow-namespaces", i.OciHooks.FailAllowNamespaces) data, err := json.MarshalIndent(hook, "", " ") if err != nil { log.WithError(err).Fatal("failed to unmarshall hook info") diff --git a/contrib/rthooks/tetragon-oci-hook/docs/demo.md b/contrib/rthooks/tetragon-oci-hook/docs/demo.md index 3f1d4806f2d..161b71e639d 100644 --- a/contrib/rthooks/tetragon-oci-hook/docs/demo.md +++ b/contrib/rthooks/tetragon-oci-hook/docs/demo.md @@ -29,7 +29,7 @@ helm install --namespace kube-system \ --set tetragon.image.override=localhost/cilium/tetragon:latest \ --set tetragon.grpc.address="unix:///var/run/cilium/tetragon/tetragon.sock" \ --set tetragon.ociHookSetup.enabled=true \ - tetragon ./install/kubernetes + tetragon ./install/kubernetes/tetragon ... kubectl logs -n kube-system tetragon-289tf -c oci-hook-setup time="2023-12-05T09:28:50Z" level=info msg="written binary" hook-dst-path=/hostInstall/tetragon-oci-hook diff --git a/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml b/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml index 491f4d00bd4..0817c778764 100644 --- a/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml +++ b/contrib/rthooks/tetragon-oci-hook/k8s/ds-uninstall.yaml @@ -2,7 +2,6 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: tetragon-oci-hook-uninstall - namespace: kube-system labels: k8s-app: tetragon-oci-hook-setup-test spec: diff --git a/docs/content/en/docs/reference/helm-chart.md b/docs/content/en/docs/reference/helm-chart.md index f51c3c17879..a8f03a2d93a 100644 --- a/docs/content/en/docs/reference/helm-chart.md +++ b/docs/content/en/docs/reference/helm-chart.md @@ -82,9 +82,10 @@ To use [the values available](#values), with `helm install` or `helm upgrade`, u | tetragon.image.override | string | `nil` | | | tetragon.image.repository | string | `"quay.io/cilium/tetragon"` | | | tetragon.image.tag | string | `"v1.1.0"` | | -| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | +| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"failAllowNamespaces":"","installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | | tetragon.ociHookSetup.enabled | bool | `false` | enable init container to setup tetragon-oci-hook | | tetragon.ociHookSetup.extraVolumeMounts | list | `[]` | Extra volume mounts to add to the oci-hook-setup init container | +| tetragon.ociHookSetup.failAllowNamespaces | string | `""` | Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. The namespace Tetragon is deployed in is always added as an exception and must not be added again. | | tetragon.ociHookSetup.interface | string | `"oci-hooks"` | interface specifices how the hook is configured. There is only one avaialble value for now: "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). | | tetragon.ociHookSetup.resources | object | `{}` | resources for the the oci-hook-setup init container | | tetragon.ociHookSetup.securityContext | object | `{"privileged":true}` | Security context for oci-hook-setup init container | diff --git a/install/kubernetes/tetragon/README.md b/install/kubernetes/tetragon/README.md index 55986abd165..4d9b4cde51e 100644 --- a/install/kubernetes/tetragon/README.md +++ b/install/kubernetes/tetragon/README.md @@ -64,9 +64,10 @@ Helm chart for Tetragon | tetragon.image.override | string | `nil` | | | tetragon.image.repository | string | `"quay.io/cilium/tetragon"` | | | tetragon.image.tag | string | `"v1.1.0"` | | -| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | +| tetragon.ociHookSetup | object | `{"enabled":false,"extraVolumeMounts":[],"failAllowNamespaces":"","installDir":"/opt/tetragon","interface":"oci-hooks","resources":{},"securityContext":{"privileged":true}}` | Configure tetragon's init container for setting up tetragon-oci-hook on the host | | tetragon.ociHookSetup.enabled | bool | `false` | enable init container to setup tetragon-oci-hook | | tetragon.ociHookSetup.extraVolumeMounts | list | `[]` | Extra volume mounts to add to the oci-hook-setup init container | +| tetragon.ociHookSetup.failAllowNamespaces | string | `""` | Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. The namespace Tetragon is deployed in is always added as an exception and must not be added again. | | tetragon.ociHookSetup.interface | string | `"oci-hooks"` | interface specifices how the hook is configured. There is only one avaialble value for now: "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). | | tetragon.ociHookSetup.resources | object | `{}` | resources for the the oci-hook-setup init container | | tetragon.ociHookSetup.securityContext | object | `{"privileged":true}` | Security context for oci-hook-setup init container | diff --git a/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl b/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl index 895277d9bbe..23f8d47ac94 100644 --- a/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl +++ b/install/kubernetes/tetragon/templates/_container_oci_hooks_setup.tpl @@ -11,6 +11,7 @@ - --local-install-dir={{ include "container.tetragonOCIHookSetup.installPath" . }} - --host-install-dir={{ .Values.tetragon.ociHookSetup.installDir }} - --oci-hooks.local-dir={{ include "container.tetragonOCIHookSetup.hooksPath" . }} + - --oci-hooks.fail-allow-namespaces={{ if .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ printf "%s,%s" .Release.Namespace .Values.tetragon.ociHookSetup.failAllowNamespaces }}{{ else }}{{ .Release.Namespace }}{{ end }} volumeMounts: {{- with .Values.tetragon.ociHookSetup.extraVolumeMounts }} {{- toYaml . | nindent 4 }} diff --git a/install/kubernetes/tetragon/values.yaml b/install/kubernetes/tetragon/values.yaml index d8a8a4c5bfb..7f37016f58b 100644 --- a/install/kubernetes/tetragon/values.yaml +++ b/install/kubernetes/tetragon/values.yaml @@ -200,6 +200,9 @@ tetragon: # "oci-hooks" (https://github.com/containers/common/blob/main/pkg/hooks/docs/oci-hooks.5.md). interface: "oci-hooks" installDir: "/opt/tetragon" + # -- Comma-separated list of namespaces to allow Pod creation for, in case tetragon-oci-hook fails to reach Tetragon agent. + # The namespace Tetragon is deployed in is always added as an exception and must not be added again. + failAllowNamespaces: "" # -- Security context for oci-hook-setup init container securityContext: privileged: true