deploy/core: kubearmor for GKE latest COS images

* latest GKE COS images do not have path for /usr/src. * deploygen updated to create kubearmor.yamls accordingly Detailed description: GKE supports multiple images types, viz COS and non-COS(Ubuntu, Debian etc). In case of non-COS images, the `/usr/src` contains the kernel headers. In case of COS, kubearmor internally downloads the kernel headers but still it used to mount `/usr/src` since we used a single yaml for COS and non-COS images. In the latest releases of COS images (for e.g., 1.22.6-gke-1000), the `/usr/src` folder is no longer present. The current changes now mounts /usr to /opt/hostusr folder for GKE (only). The kubearmor code internally sets `BCC_KERNEL_SOURCE` to `/opt/hostusr/src/linux-headers-KERNELVER`. Fixes #579 Signed-off-by: Rahul Jadhav <nyrahul@gmail.com>
kubearmor · Mar 14, 2022 · 0932e68 · 0932e68
1 parent 925a26a
commit 0932e68
Show file tree

Hide file tree

Showing 10 changed files with 110 additions and 70 deletions.
diff --git a/KubeArmor/monitor/systemMonitor.go b/KubeArmor/monitor/systemMonitor.go
@@ -158,9 +158,6 @@ type SystemMonitor struct {
 
 	// ticker to clean up exited pids
 	Ticker *time.Ticker
-
-	// GKE
-	IsCOS bool
 }
 
 // NewSystemMonitor Function
@@ -193,8 +190,6 @@ func NewSystemMonitor(node tp.Node, logger *fd.Feeder, containers *map[string]tp
 
 	mon.Ticker = time.NewTicker(time.Second * 10)
 
-	mon.IsCOS = false
-
 	return mon
 }
 
@@ -226,8 +221,20 @@ func (mon *SystemMonitor) InitBPF() error {
 
 				// just for safety
 				time.Sleep(time.Second * 1)
-
-				mon.IsCOS = true
+			} else {
+				// In case of GKE COS release >= 1.22, the base OS img does not
+				// contain /usr/src folder. Thus we now mount /usr folder to
+				// /opt/hostusr folder in kubearmor for GKE. The following code
+				// checks whether the /opt/hostusr/src/kernel-hdrs path exists
+				// and uses it for BCC kernel source, if present.
+				lklhdrpath := "/media/root/usr/src/linux-headers-" + mon.KernelVersion
+				mon.Logger.Printf("checking if kernel headers path (%s) exists", lklhdrpath)
+				if _, err := os.Stat(lklhdrpath); err == nil {
+					mon.Logger.Printf("using kernel headers from (%s)", lklhdrpath)
+					if err := os.Setenv("BCC_KERNEL_SOURCE", lklhdrpath); err != nil {
+						mon.Logger.Errf("setenv failed for [BCC_KERNEL_SOURCE=%s] Error=%s", lklhdrpath, err.Error())
+					}
+				}
 			}
 		}
 	}

diff --git a/deployments/EKS/kubearmor.yaml b/deployments/EKS/kubearmor.yaml
@@ -101,9 +101,6 @@ spec:
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
-        - mountPath: /usr/src
-          name: usr-src-path
-          readOnly: true
         - mountPath: /lib/modules
           name: lib-modules-path
           readOnly: true
@@ -116,6 +113,9 @@ spec:
         - mountPath: /media/root/etc/os-release
           name: os-release-path
           readOnly: true
+        - mountPath: /usr/src
+          name: usr-src-path
+          readOnly: true
         - mountPath: /etc/apparmor.d
           name: etc-apparmor-d-path
         - mountPath: /var/run/containerd/containerd.sock
@@ -138,10 +138,6 @@ spec:
       tolerations:
       - operator: Exists
       volumes:
-      - hostPath:
-          path: /usr/src
-          type: Directory
-        name: usr-src-path
       - hostPath:
           path: /lib/modules
           type: Directory
@@ -162,6 +158,10 @@ spec:
           path: /etc/os-release
           type: File
         name: os-release-path
+      - hostPath:
+          path: /usr/src
+          type: Directory
+        name: usr-src-path
       - hostPath:
           path: /etc/apparmor.d
           type: DirectoryOrCreate

diff --git a/deployments/GKE/kubearmor.yaml b/deployments/GKE/kubearmor.yaml
@@ -101,9 +101,6 @@ spec:
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
-        - mountPath: /usr/src
-          name: usr-src-path
-          readOnly: true
         - mountPath: /lib/modules
           name: lib-modules-path
           readOnly: true
@@ -116,6 +113,9 @@ spec:
         - mountPath: /media/root/etc/os-release
           name: os-release-path
           readOnly: true
+        - mountPath: /media/root/usr
+          name: usr-src-path
+          readOnly: true
         - mountPath: /etc/apparmor.d
           name: etc-apparmor-d-path
         - mountPath: /var/run/containerd/containerd.sock
@@ -138,10 +138,6 @@ spec:
       tolerations:
       - operator: Exists
       volumes:
-      - hostPath:
-          path: /usr/src
-          type: Directory
-        name: usr-src-path
       - hostPath:
           path: /lib/modules
           type: Directory
@@ -162,6 +158,10 @@ spec:
           path: /etc/os-release
           type: File
         name: os-release-path
+      - hostPath:
+          path: /usr
+          type: Directory
+        name: usr-src-path
       - hostPath:
           path: /etc/apparmor.d
           type: DirectoryOrCreate

diff --git a/deployments/docker/kubearmor.yaml b/deployments/docker/kubearmor.yaml
@@ -101,9 +101,6 @@ spec:
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
-        - mountPath: /usr/src
-          name: usr-src-path
-          readOnly: true
         - mountPath: /lib/modules
           name: lib-modules-path
           readOnly: true
@@ -116,6 +113,9 @@ spec:
         - mountPath: /media/root/etc/os-release
           name: os-release-path
           readOnly: true
+        - mountPath: /usr/src
+          name: usr-src-path
+          readOnly: true
         - mountPath: /etc/apparmor.d
           name: etc-apparmor-d-path
         - mountPath: /var/run/docker.sock
@@ -135,10 +135,6 @@ spec:
       tolerations:
       - operator: Exists
       volumes:
-      - hostPath:
-          path: /usr/src
-          type: Directory
-        name: usr-src-path
       - hostPath:
           path: /lib/modules
           type: Directory
@@ -159,6 +155,10 @@ spec:
           path: /etc/os-release
           type: File
         name: os-release-path
+      - hostPath:
+          path: /usr/src
+          type: Directory
+        name: usr-src-path
       - hostPath:
           path: /etc/apparmor.d
           type: DirectoryOrCreate

diff --git a/deployments/generic/kubearmor.yaml b/deployments/generic/kubearmor.yaml
@@ -101,9 +101,6 @@ spec:
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
-        - mountPath: /usr/src
-          name: usr-src-path
-          readOnly: true
         - mountPath: /lib/modules
           name: lib-modules-path
           readOnly: true
@@ -116,6 +113,9 @@ spec:
         - mountPath: /media/root/etc/os-release
           name: os-release-path
           readOnly: true
+        - mountPath: /usr/src
+          name: usr-src-path
+          readOnly: true
         - mountPath: /etc/apparmor.d
           name: etc-apparmor-d-path
         - mountPath: /var/run/containerd/containerd.sock
@@ -138,10 +138,6 @@ spec:
       tolerations:
       - operator: Exists
       volumes:
-      - hostPath:
-          path: /usr/src
-          type: Directory
-        name: usr-src-path
       - hostPath:
           path: /lib/modules
           type: Directory
@@ -162,6 +158,10 @@ spec:
           path: /etc/os-release
           type: File
         name: os-release-path
+      - hostPath:
+          path: /usr/src
+          type: Directory
+        name: usr-src-path
       - hostPath:
           path: /etc/apparmor.d
           type: DirectoryOrCreate

diff --git a/deployments/get/defaults.go b/deployments/get/defaults.go
@@ -29,13 +29,46 @@ var hostPathDirectoryOrCreate = corev1.HostPathDirectoryOrCreate
 var hostPathFile = corev1.HostPathFile
 var hostPathSocket = corev1.HostPathSocket
 
+var gkeHostUsrVolMnt = corev1.VolumeMount{
+	Name:      "usr-src-path", // /usr -> /media/root/usr (read-only) check issue #579 for details
+	MountPath: "/media/root/usr",
+	ReadOnly:  true,
+}
+
+var gkeHostUsrVol = corev1.Volume{ // check #579 why GKE is handled separately
+	Name: "usr-src-path",
+	VolumeSource: corev1.VolumeSource{
+		HostPath: &corev1.HostPathVolumeSource{
+			Path: "/usr",
+			Type: &hostPathDirectory,
+		},
+	},
+}
+
+var hostUsrVolMnt = corev1.VolumeMount{
+	Name:      "usr-src-path", // /usr/src (read-only)
+	MountPath: "/usr/src",
+	ReadOnly:  true,
+}
+
+var hostUsrVol = corev1.Volume{
+	Name: "usr-src-path",
+	VolumeSource: corev1.VolumeSource{
+		HostPath: &corev1.HostPathVolumeSource{
+			Path: "/usr/src",
+			Type: &hostPathDirectory,
+		},
+	},
+}
+
 // Environment Specific Daemonset Configuration
 var defaultConfigs = map[string]DaemonSetConfig{
 	"generic": {
 		Args: []string{
 			"-enableKubeArmorHostPolicy",
 		},
 		VolumeMounts: []corev1.VolumeMount{
+			hostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -57,6 +90,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			hostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{
@@ -100,6 +134,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			"-enableKubeArmorHostPolicy",
 		},
 		VolumeMounts: []corev1.VolumeMount{
+			hostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -116,6 +151,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			hostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{
@@ -148,6 +184,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 	"minikube": {
 		Args: []string{},
 		VolumeMounts: []corev1.VolumeMount{
+			hostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -164,6 +201,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			hostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{
@@ -198,6 +236,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			"-enableKubeArmorHostPolicy",
 		},
 		VolumeMounts: []corev1.VolumeMount{
+			hostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -214,6 +253,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			hostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{
@@ -248,6 +288,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			"-enableKubeArmorHostPolicy",
 		},
 		VolumeMounts: []corev1.VolumeMount{
+			hostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -264,6 +305,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			hostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{
@@ -298,6 +340,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			"-enableKubeArmorHostPolicy",
 		},
 		VolumeMounts: []corev1.VolumeMount{
+			gkeHostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -319,6 +362,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			gkeHostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{
@@ -362,6 +406,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			"-enableKubeArmorHostPolicy",
 		},
 		VolumeMounts: []corev1.VolumeMount{
+			hostUsrVolMnt,
 			{
 				Name:      "etc-apparmor-d-path",
 				MountPath: "/etc/apparmor.d",
@@ -383,6 +428,7 @@ var defaultConfigs = map[string]DaemonSetConfig{
 			},
 		},
 		Volumes: []corev1.Volume{
+			hostUsrVol,
 			{
 				Name: "etc-apparmor-d-path",
 				VolumeSource: corev1.VolumeSource{

diff --git a/deployments/get/objects.go b/deployments/get/objects.go
@@ -364,12 +364,8 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet {
 		"-gRPC=" + strconv.Itoa(int(port)),
 		"-logPath=/tmp/kubearmor.log",
 	}
+
 	var volumeMounts = []corev1.VolumeMount{
-		{
-			Name:      "usr-src-path", //BPF (read-only)
-			MountPath: "/usr/src",
-			ReadOnly:  true,
-		},
 		{
 			Name:      "lib-modules-path", //BPF (read-only)
 			MountPath: "/lib/modules",
@@ -395,15 +391,6 @@ func GenerateDaemonSet(env, namespace string) *appsv1.DaemonSet {
 	}
 
 	var volumes = []corev1.Volume{
-		{
-			Name: "usr-src-path",
-			VolumeSource: corev1.VolumeSource{
-				HostPath: &corev1.HostPathVolumeSource{
-					Path: "/usr/src",
-					Type: &hostPathDirectory,
-				},
-			},
-		},
 		{
 			Name: "lib-modules-path",
 			VolumeSource: corev1.VolumeSource{