From e2737efa975198efde13a48435cc994daa3ba018 Mon Sep 17 00:00:00 2001 From: Rahul Jadhav <nyrahul@gmail.com> Date: Mon, 14 Mar 2022 15:30:16 +0530 Subject: [PATCH] deploy/core: kubearmor for GKE latest COS images * latest GKE COS images do not have path for /usr/src. * deploygen updated to create kubearmor.yamls accordingly Detailed description: GKE supports multiple images types, viz COS and non-COS(Ubuntu, Debian etc). In case of non-COS images, the `/usr/src` contains the kernel headers. In case of COS, kubearmor internally downloads the kernel headers but still it used to mount `/usr/src` since we used a single yaml for COS and non-COS images. In the latest releases of COS images (for e.g., 1.22.6-gke-1000), the `/usr/src` folder is no longer present. The current changes now mounts /usr to `/media/root/usr` folder for GKE (only). The kubearmor code internally sets `BCC_KERNEL_SOURCE` to `/media/root/usr/src/linux-headers-KERNELVER`. Fixes #579 Signed-off-by: Rahul Jadhav <nyrahul@gmail.com> --- KubeArmor/monitor/systemMonitor.go | 21 ++-- deployments/EKS/kubearmor.yaml | 14 +-- deployments/GKE/kubearmor.yaml | 14 +-- deployments/docker/kubearmor.yaml | 14 +-- deployments/generic/kubearmor.yaml | 14 +-- deployments/get/defaults.go | 166 +++++++++++++--------------- deployments/get/objects.go | 15 +-- deployments/k3s/kubearmor.yaml | 14 +-- deployments/microk8s/kubearmor.yaml | 14 +-- deployments/minikube/kubearmor.yaml | 14 +-- 10 files changed, 139 insertions(+), 161 deletions(-) diff --git a/KubeArmor/monitor/systemMonitor.go b/KubeArmor/monitor/systemMonitor.go index 74237fd374..4ef63c9376 100644 --- a/KubeArmor/monitor/systemMonitor.go +++ b/KubeArmor/monitor/systemMonitor.go @@ -158,9 +158,6 @@ type SystemMonitor struct { // ticker to clean up exited pids Ticker *time.Ticker - - // GKE - IsCOS bool } // NewSystemMonitor Function @@ -193,8 +190,6 @@ func NewSystemMonitor(node tp.Node, logger *fd.Feeder, containers *map[string]tp mon.Ticker = time.NewTicker(time.Second * 10) - mon.IsCOS = false - return mon } @@ -226,8 +221,20 @@ func (mon *SystemMonitor) InitBPF() error { // just for safety time.Sleep(time.Second * 1) - - mon.IsCOS = true + } else { + // In case of GKE COS release >= 1.22, the base OS img does not + // contain /usr/src folder. Thus we now mount /usr folder to + // /media/root/usr folder in kubearmor for GKE. The following code + // checks whether the /media/root/usr/src/kernel-hdrs path exists + // and uses it for BCC kernel source, if present. + lklhdrpath := "/media/root/usr/src/linux-headers-" + mon.KernelVersion + mon.Logger.Printf("checking if kernel headers path (%s) exists", lklhdrpath) + if _, err := os.Stat(lklhdrpath); err == nil { + mon.Logger.Printf("using kernel headers from (%s)", lklhdrpath) + if err := os.Setenv("BCC_KERNEL_SOURCE", lklhdrpath); err != nil { + mon.Logger.Errf("setenv failed for [BCC_KERNEL_SOURCE=%s] Error=%s", lklhdrpath, err.Error()) + } + } } } } diff --git a/deployments/EKS/kubearmor.yaml b/deployments/EKS/kubearmor.yaml index be689564d0..14d050b964 100644 --- a/deployments/EKS/kubearmor.yaml +++ b/deployments/EKS/kubearmor.yaml @@ -101,9 +101,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -116,6 +113,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/run/containerd/containerd.sock @@ -138,10 +138,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -162,6 +158,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate diff --git a/deployments/GKE/kubearmor.yaml b/deployments/GKE/kubearmor.yaml index be689564d0..d576b0b921 100644 --- a/deployments/GKE/kubearmor.yaml +++ b/deployments/GKE/kubearmor.yaml @@ -101,9 +101,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -116,6 +113,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /media/root/usr + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/run/containerd/containerd.sock @@ -138,10 +138,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -162,6 +158,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate diff --git a/deployments/docker/kubearmor.yaml b/deployments/docker/kubearmor.yaml index 4303fb7e30..a2fe32c926 100644 --- a/deployments/docker/kubearmor.yaml +++ b/deployments/docker/kubearmor.yaml @@ -101,9 +101,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -116,6 +113,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/run/docker.sock @@ -135,10 +135,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -159,6 +155,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate diff --git a/deployments/generic/kubearmor.yaml b/deployments/generic/kubearmor.yaml index be689564d0..14d050b964 100644 --- a/deployments/generic/kubearmor.yaml +++ b/deployments/generic/kubearmor.yaml @@ -101,9 +101,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -116,6 +113,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/run/containerd/containerd.sock @@ -138,10 +138,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -162,6 +158,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate diff --git a/deployments/get/defaults.go b/deployments/get/defaults.go index 5a65541d20..49ac41f06d 100644 --- a/deployments/get/defaults.go +++ b/deployments/get/defaults.go @@ -29,6 +29,53 @@ var hostPathDirectoryOrCreate = corev1.HostPathDirectoryOrCreate var hostPathFile = corev1.HostPathFile var hostPathSocket = corev1.HostPathSocket +var gkeHostUsrVolMnt = corev1.VolumeMount{ + Name: "usr-src-path", // /usr -> /media/root/usr (read-only) check issue #579 for details + MountPath: "/media/root/usr", + ReadOnly: true, +} + +var gkeHostUsrVol = corev1.Volume{ // check #579 why GKE is handled separately + Name: "usr-src-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/usr", + Type: &hostPathDirectory, + }, + }, +} + +var hostUsrVolMnt = corev1.VolumeMount{ + Name: "usr-src-path", // /usr/src (read-only) + MountPath: "/usr/src", + ReadOnly: true, +} + +var hostUsrVol = corev1.Volume{ + Name: "usr-src-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/usr/src", + Type: &hostPathDirectory, + }, + }, +} + +var apparmorVolMnt = corev1.VolumeMount{ + Name: "etc-apparmor-d-path", + MountPath: "/etc/apparmor.d", +} + +var apparmorVol = corev1.Volume{ + Name: "etc-apparmor-d-path", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/apparmor.d", + Type: &hostPathDirectoryOrCreate, + }, + }, +} + // Environment Specific Daemonset Configuration var defaultConfigs = map[string]DaemonSetConfig{ "generic": { @@ -36,10 +83,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "-enableKubeArmorHostPolicy", }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + hostUsrVolMnt, + apparmorVolMnt, { Name: "containerd-sock-path", // containerd MountPath: "/var/run/containerd/containerd.sock", @@ -57,15 +102,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + hostUsrVol, + apparmorVol, { Name: "containerd-sock-path", VolumeSource: corev1.VolumeSource{ @@ -100,10 +138,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "-enableKubeArmorHostPolicy", }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + hostUsrVolMnt, + apparmorVolMnt, { Name: "docker-sock-path", // docker MountPath: "/var/run/docker.sock", @@ -116,15 +152,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + hostUsrVol, + apparmorVol, { Name: "docker-sock-path", VolumeSource: corev1.VolumeSource{ @@ -148,10 +177,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "minikube": { Args: []string{}, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + hostUsrVolMnt, + apparmorVolMnt, { Name: "docker-sock-path", // docker MountPath: "/var/run/docker.sock", @@ -164,15 +191,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + hostUsrVol, + apparmorVol, { Name: "docker-sock-path", VolumeSource: corev1.VolumeSource{ @@ -198,10 +218,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "-enableKubeArmorHostPolicy", }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + hostUsrVolMnt, + apparmorVolMnt, { Name: "containerd-sock-path", // containerd MountPath: "/var/snap/microk8s/common/run/containerd.sock", @@ -214,15 +232,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + hostUsrVol, + apparmorVol, { Name: "containerd-sock-path", VolumeSource: corev1.VolumeSource{ @@ -248,10 +259,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "-enableKubeArmorHostPolicy", }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + hostUsrVolMnt, + apparmorVolMnt, { Name: "containerd-sock-path", // containerd MountPath: "/var/run/containerd/containerd.sock", @@ -264,15 +273,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + hostUsrVol, + apparmorVol, { Name: "containerd-sock-path", VolumeSource: corev1.VolumeSource{ @@ -298,10 +300,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "-enableKubeArmorHostPolicy", }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + gkeHostUsrVolMnt, + apparmorVolMnt, { Name: "containerd-sock-path", // containerd MountPath: "/var/run/containerd/containerd.sock", @@ -319,15 +319,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + gkeHostUsrVol, + apparmorVol, { Name: "containerd-sock-path", VolumeSource: corev1.VolumeSource{ @@ -362,10 +355,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ "-enableKubeArmorHostPolicy", }, VolumeMounts: []corev1.VolumeMount{ - { - Name: "etc-apparmor-d-path", - MountPath: "/etc/apparmor.d", - }, + hostUsrVolMnt, + apparmorVolMnt, { Name: "containerd-sock-path", // containerd MountPath: "/var/run/containerd/containerd.sock", @@ -383,15 +374,8 @@ var defaultConfigs = map[string]DaemonSetConfig{ }, }, Volumes: []corev1.Volume{ - { - Name: "etc-apparmor-d-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/apparmor.d", - Type: &hostPathDirectoryOrCreate, - }, - }, - }, + hostUsrVol, + apparmorVol, { Name: "containerd-sock-path", VolumeSource: corev1.VolumeSource{ diff --git a/deployments/get/objects.go b/deployments/get/objects.go index b634fcc915..fa27ac00f8 100644 --- a/deployments/get/objects.go +++ b/deployments/get/objects.go @@ -364,12 +364,8 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet { "-gRPC=" + strconv.Itoa(int(port)), "-logPath=/tmp/kubearmor.log", } + var volumeMounts = []corev1.VolumeMount{ - { - Name: "usr-src-path", //BPF (read-only) - MountPath: "/usr/src", - ReadOnly: true, - }, { Name: "lib-modules-path", //BPF (read-only) MountPath: "/lib/modules", @@ -395,15 +391,6 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet { } var volumes = []corev1.Volume{ - { - Name: "usr-src-path", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/usr/src", - Type: &hostPathDirectory, - }, - }, - }, { Name: "lib-modules-path", VolumeSource: corev1.VolumeSource{ diff --git a/deployments/k3s/kubearmor.yaml b/deployments/k3s/kubearmor.yaml index a97af52cbd..708920211d 100644 --- a/deployments/k3s/kubearmor.yaml +++ b/deployments/k3s/kubearmor.yaml @@ -101,9 +101,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -116,6 +113,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/run/containerd/containerd.sock @@ -135,10 +135,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -159,6 +155,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate diff --git a/deployments/microk8s/kubearmor.yaml b/deployments/microk8s/kubearmor.yaml index 7a5bce27be..4aa510699c 100644 --- a/deployments/microk8s/kubearmor.yaml +++ b/deployments/microk8s/kubearmor.yaml @@ -101,9 +101,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -116,6 +113,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/snap/microk8s/common/run/containerd.sock @@ -135,10 +135,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -159,6 +155,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate diff --git a/deployments/minikube/kubearmor.yaml b/deployments/minikube/kubearmor.yaml index 55036d8efd..94bc91ad20 100644 --- a/deployments/minikube/kubearmor.yaml +++ b/deployments/minikube/kubearmor.yaml @@ -100,9 +100,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - - mountPath: /usr/src - name: usr-src-path - readOnly: true - mountPath: /lib/modules name: lib-modules-path readOnly: true @@ -115,6 +112,9 @@ spec: - mountPath: /media/root/etc/os-release name: os-release-path readOnly: true + - mountPath: /usr/src + name: usr-src-path + readOnly: true - mountPath: /etc/apparmor.d name: etc-apparmor-d-path - mountPath: /var/run/docker.sock @@ -134,10 +134,6 @@ spec: tolerations: - operator: Exists volumes: - - hostPath: - path: /usr/src - type: Directory - name: usr-src-path - hostPath: path: /lib/modules type: Directory @@ -158,6 +154,10 @@ spec: path: /etc/os-release type: File name: os-release-path + - hostPath: + path: /usr/src + type: Directory + name: usr-src-path - hostPath: path: /etc/apparmor.d type: DirectoryOrCreate