Use UnmountMountPoint util to clean up subpaths

pkg/util/mount/BUILD

@@ -9,6 +9,7 @@ go_library(
         "exec_mount_unsupported.go",
         "fake.go",
         "mount.go",
+        "mount_helper.go",
         "mount_linux.go",
         "mount_unsupported.go",
         "mount_windows.go",
@@ -67,6 +68,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "exec_mount_test.go",
+        "mount_helper_test.go",
         "mount_linux_test.go",
         "mount_test.go",
         "mount_windows_test.go",

pkg/util/mount/fake.go

@@ -30,6 +30,8 @@ type FakeMounter struct {
 	MountPoints []MountPoint
 	Log         []FakeAction
 	Filesystem  map[string]FileType
+	// Error to return for a path when calling IsLikelyNotMountPoint
+	MountCheckErrors map[string]error
 	// Some tests run things in parallel, make sure the mounter does not produce
 	// any golang's DATA RACE warnings.
 	mutex sync.Mutex
@@ -119,6 +121,7 @@ func (f *FakeMounter) Unmount(target string) error {
 	}
 	f.MountPoints = newMountpoints
 	f.Log = append(f.Log, FakeAction{Action: FakeActionUnmount, Target: absTarget})
+	delete(f.MountCheckErrors, target)
 	return nil
 }
 
@@ -141,7 +144,12 @@ func (f *FakeMounter) IsLikelyNotMountPoint(file string) (bool, error) {
 	f.mutex.Lock()
 	defer f.mutex.Unlock()
 
-	_, err := os.Stat(file)
+	err := f.MountCheckErrors[file]
+	if err != nil {
+		return false, err
+	}
+
+	_, err = os.Stat(file)
 	if err != nil {
 		return true, err
 	}

pkg/util/mount/mount_helper.go

@@ -0,0 +1,124 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+
+	"k8s.io/klog"
+)
+
+// CleanupMountPoint unmounts the given path and
+// deletes the remaining directory if successful.
+// if extensiveMountPointCheck is true
+// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
+// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs.
+func CleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool) error {
+	// mounter.ExistsPath cannot be used because for containerized kubelet, we need to check
+	// the path in the kubelet container, not on the host.
+	pathExists, pathErr := PathExists(mountPath)
+	if !pathExists {
+		klog.Warningf("Warning: Unmount skipped because path does not exist: %v", mountPath)
+		return nil
+	}
+	corruptedMnt := IsCorruptedMnt(pathErr)
+	if pathErr != nil && !corruptedMnt {
+		return fmt.Errorf("Error checking path: %v", pathErr)
+	}
+	return doCleanupMountPoint(mountPath, mounter, extensiveMountPointCheck, corruptedMnt)
+}
+
+// doCleanupMountPoint unmounts the given path and
+// deletes the remaining directory if successful.
+// if extensiveMountPointCheck is true
+// IsNotMountPoint will be called instead of IsLikelyNotMountPoint.
+// IsNotMountPoint is more expensive but properly handles bind mounts within the same fs.
+// if corruptedMnt is true, it means that the mountPath is a corrupted mountpoint, and the mount point check
+// will be skipped
+func doCleanupMountPoint(mountPath string, mounter Interface, extensiveMountPointCheck bool, corruptedMnt bool) error {
+	if !corruptedMnt {
+		var notMnt bool
+		var err error
+		if extensiveMountPointCheck {
+			notMnt, err = IsNotMountPoint(mounter, mountPath)
+		} else {
+			notMnt, err = mounter.IsLikelyNotMountPoint(mountPath)
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if notMnt {
+			klog.Warningf("Warning: %q is not a mountpoint, deleting", mountPath)
+			return os.Remove(mountPath)
+		}
+	}
+
+	// Unmount the mount path
+	klog.V(4).Infof("%q is a mountpoint, unmounting", mountPath)
+	if err := mounter.Unmount(mountPath); err != nil {
+		return err
+	}
+
+	notMnt, mntErr := mounter.IsLikelyNotMountPoint(mountPath)
+	if mntErr != nil {
+		return mntErr
+	}
+	if notMnt {
+		klog.V(4).Infof("%q is unmounted, deleting the directory", mountPath)
+		return os.Remove(mountPath)
+	}
+	return fmt.Errorf("Failed to unmount path %v", mountPath)
+}
+
+// TODO: clean this up to use pkg/util/file/FileExists
+// PathExists returns true if the specified path exists.
+func PathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	} else if os.IsNotExist(err) {
+		return false, nil
+	} else if IsCorruptedMnt(err) {
+		return true, err
+	} else {
+		return false, err
+	}
+}
+
+// IsCorruptedMnt return true if err is about corrupted mount point
+func IsCorruptedMnt(err error) bool {
+	if err == nil {
+		return false
+	}
+	var underlyingError error
+	switch pe := err.(type) {
+	case nil:
+		return false
+	case *os.PathError:
+		underlyingError = pe.Err
+	case *os.LinkError:
+		underlyingError = pe.Err
+	case *os.SyscallError:
+		underlyingError = pe.Err
+	}
+
+	return underlyingError == syscall.ENOTCONN || underlyingError == syscall.ESTALE || underlyingError == syscall.EIO
+}

pkg/util/mount/mount_helper_test.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mount
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+	"testing"
+)
+
+func TestDoCleanupMountPoint(t *testing.T) {
+	const testMount = "test-mount"
+	const defaultPerm = 0750
+
+	tests := map[string]struct {
+		corruptedMnt bool
+		// Function that prepares the directory structure for the test under
+		// the given base directory.
+		// Returns a fake MountPoint, a fake error for the mount point,
+		// and error if the prepare function encountered a fatal error.
+		prepare   func(base string) (MountPoint, error, error)
+		expectErr bool
+	}{
+		"mount-ok": {
+			prepare: func(base string) (MountPoint, error, error) {
+				path := filepath.Join(base, testMount)
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return MountPoint{}, nil, err
+				}
+				return MountPoint{Device: "/dev/sdb", Path: path}, nil, nil
+			},
+		},
+		"mount-corrupted": {
+			prepare: func(base string) (MountPoint, error, error) {
+				path := filepath.Join(base, testMount)
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return MountPoint{}, nil, err
+				}
+				return MountPoint{Device: "/dev/sdb", Path: path}, os.NewSyscallError("fake", syscall.ESTALE), nil
+			},
+			corruptedMnt: true,
+		},
+		"mount-err-not-corrupted": {
+			prepare: func(base string) (MountPoint, error, error) {
+				path := filepath.Join(base, testMount)
+				if err := os.MkdirAll(path, defaultPerm); err != nil {
+					return MountPoint{}, nil, err
+				}
+				return MountPoint{Device: "/dev/sdb", Path: path}, os.NewSyscallError("fake", syscall.ETIMEDOUT), nil
+			},
+			expectErr: true,
+		},
+	}
+
+	for name, tt := range tests {
+		t.Run(name, func(t *testing.T) {
+
+			tmpDir, err := ioutil.TempDir("", "unmount-mount-point-test")
+			if err != nil {
+				t.Fatalf("failed to create tmpdir: %v", err)
+			}
+			defer os.RemoveAll(tmpDir)
+
+			if tt.prepare == nil {
+				t.Fatalf("prepare function required")
+			}
+
+			mountPoint, mountError, err := tt.prepare(tmpDir)
+			if err != nil {
+				t.Fatalf("failed to prepare test: %v", err)
+			}
+
+			fake := &FakeMounter{
+				MountPoints:      []MountPoint{mountPoint},
+				MountCheckErrors: map[string]error{mountPoint.Path: mountError},
+			}
+
+			err = doCleanupMountPoint(mountPoint.Path, fake, true, tt.corruptedMnt)
+			if tt.expectErr {
+				if err == nil {
+					t.Errorf("test %s failed, expected error, got none", name)
+				}
+				if err := validateDirExists(mountPoint.Path); err != nil {
+					t.Errorf("test %s failed, mount path doesn't exist: %v", name, err)
+				}
+			}
+			if !tt.expectErr {
+				if err != nil {
+					t.Errorf("test %s failed: %v", name, err)
+				}
+				if err := validateDirNotExists(mountPoint.Path); err != nil {
+					t.Errorf("test %s failed, mount path still exists: %v", name, err)
+				}
+			}
+		})
+	}
+}
+
+func validateDirEmpty(dir string) error {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+
+	if len(files) != 0 {
+		return fmt.Errorf("Directory %q is not empty", dir)
+	}
+	return nil
+}
+
+func validateDirExists(dir string) error {
+	_, err := ioutil.ReadDir(dir)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateDirNotExists(dir string) error {
+	_, err := ioutil.ReadDir(dir)
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	return fmt.Errorf("dir %q still exists", dir)
+}
+
+func validateFileExists(file string) error {
+	if _, err := os.Stat(file); err != nil {
+		return err
+	}
+	return nil
+}

pkg/util/mount/mount_linux.go

@@ -891,15 +891,22 @@ func doCleanSubPaths(mounter Interface, podDir string, volumeName string) error
 
 		// scan /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/*
 		fullContainerDirPath := filepath.Join(subPathDir, containerDir.Name())
-		subPaths, err := ioutil.ReadDir(fullContainerDirPath)
-		if err != nil {
-			return fmt.Errorf("error reading %s: %s", fullContainerDirPath, err)
-		}
-		for _, subPath := range subPaths {
-			if err = doCleanSubPath(mounter, fullContainerDirPath, subPath.Name()); err != nil {
+		err = filepath.Walk(fullContainerDirPath, func(path string, info os.FileInfo, err error) error {
+			if path == fullContainerDirPath {
+				// Skip top level directory
+				return nil
+			}
+
+			// pass through errors and let doCleanSubPath handle them
+			if err = doCleanSubPath(mounter, fullContainerDirPath, filepath.Base(path)); err != nil {
 				return err
 			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("error processing %s: %s", fullContainerDirPath, err)
 		}
+
 		// Whole container has been processed, remove its directory.
 		if err := os.Remove(fullContainerDirPath); err != nil {
 			return fmt.Errorf("error deleting %s: %s", fullContainerDirPath, err)
@@ -926,22 +933,12 @@ func doCleanSubPath(mounter Interface, fullContainerDirPath, subPathIndex string
 	// process /var/lib/kubelet/pods/<uid>/volume-subpaths/<volume>/<container name>/<subPathName>
 	klog.V(4).Infof("Cleaning up subpath mounts for subpath %v", subPathIndex)
 	fullSubPath := filepath.Join(fullContainerDirPath, subPathIndex)
-	notMnt, err := IsNotMountPoint(mounter, fullSubPath)
-	if err != nil {
-		return fmt.Errorf("error checking %s for mount: %s", fullSubPath, err)
-	}
-	// Unmount it
-	if !notMnt {
-		if err = mounter.Unmount(fullSubPath); err != nil {
-			return fmt.Errorf("error unmounting %s: %s", fullSubPath, err)
-		}
-		klog.V(5).Infof("Unmounted %s", fullSubPath)
-	}
-	// Remove it *non*-recursively, just in case there were some hiccups.
-	if err = os.Remove(fullSubPath); err != nil {
-		return fmt.Errorf("error deleting %s: %s", fullSubPath, err)
+
+	if err := CleanupMountPoint(fullSubPath, mounter, true); err != nil {
+		return fmt.Errorf("error cleaning subpath mount %s: %s", fullSubPath, err)
 	}
-	klog.V(5).Infof("Removed %s", fullSubPath)
+
+	klog.V(4).Infof("Successfully cleaned subpath directory %s", fullSubPath)
 	return nil
 }