cgroups: set value for cpuset if it's empty

fix issue when echo "$id" > /sys/fs/cgroup/cpu/docker/$container-id/tasks
will get error message: no space left on device, this is because
cpuset.cpus and cpuset.mems in the container's cpu cgroup
directory is empty, the following test can explain this:

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ cat footest/cpuset.cpus

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ cat footest/cpuset.mems

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ echo 1 > footest/cgroup.procs
bash: echo: write error: No space left on device

set value in cpuset.cpus and cpuset.mems can fix the problem

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ echo "0-1" > footest/cpuset.mems

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ echo "0-1" > footest/cpuset.cpus

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ echo 1 > footest/cgroup.procs

[root@r10e19288.sqa.zmf /sys/fs/cgroup/cpu]
$ cat footest/cgroup.procs
1

this patch fix above issue by copy data from cpu cgroup
root directory to container's cpuset.cpus and cpuset.mems

Fixes #642.

Signed-off-by: Ace-Tang <aceapril@126.com>

create.go

@@ -18,8 +18,10 @@ package main
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/Sirupsen/logrus"
 	vc "github.com/containers/virtcontainers"
@@ -134,7 +136,13 @@ func create(containerID, bundlePath, console, pidFilePath string, detach bool,
 		return err
 	}
 
-	if err := createCgroupsFiles(containerID, cgroupsPathList, process.Pid); err != nil {
+	// cgroupsDirPath is CgroupsPath fetch from oci spec
+	var cgroupsDirPath string
+	if ociSpec.Linux != nil {
+		cgroupsDirPath = ociSpec.Linux.CgroupsPath
+	}
+
+	if err := createCgroupsFiles(containerID, cgroupsDirPath, cgroupsPathList, process.Pid); err != nil {
 		return err
 	}
 
@@ -218,7 +226,7 @@ func createContainer(ociSpec oci.CompatOCISpec, containerID, bundlePath,
 	return c.Process(), nil
 }
 
-func createCgroupsFiles(containerID string, cgroupsPathList []string, pid int) error {
+func createCgroupsFiles(containerID string, cgroupsDirPath string, cgroupsPathList []string, pid int) error {
 	if len(cgroupsPathList) == 0 {
 		fields := logrus.Fields{
 			"container": containerID,
@@ -233,6 +241,11 @@ func createCgroupsFiles(containerID string, cgroupsPathList []string, pid int) e
 			return err
 		}
 
+		if strings.Contains(cgroupsPath, "cpu") && cgroupsDirPath != "" {
+			parent := strings.TrimSuffix(cgroupsPath, cgroupsDirPath)
+			copyParentCPUSet(cgroupsPath, parent)
+		}
+
 		tasksFilePath := filepath.Join(cgroupsPath, cgroupsTasksFile)
 		procsFilePath := filepath.Join(cgroupsPath, cgroupsProcsFile)
 
@@ -289,3 +302,47 @@ func createPIDFile(pidFilePath string, pid int) error {
 
 	return nil
 }
+
+// copyParentCPUSet copies the cpuset.cpus and cpuset.mems from the parent
+// directory to the current directory if the file's contents are 0
+func copyParentCPUSet(current, parent string) error {
+	currentCpus, currentMems, err := getCPUSet(current)
+	if err != nil {
+		return err
+	}
+
+	parentCpus, parentMems, err := getCPUSet(parent)
+	if err != nil {
+		return err
+	}
+
+	if len(parentCpus) < 0 || len(parentMems) < 0 {
+		return nil
+	}
+
+	if isEmptyString(currentCpus) {
+		if err := writeFile(filepath.Join(current, "cpuset.cpus"), string(parentCpus)); err != nil {
+			return err
+		}
+	}
+
+	if isEmptyString(currentMems) {
+		if err := writeFile(filepath.Join(current, "cpuset.mems"), string(parentMems)); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func getCPUSet(parent string) (cpus []byte, mems []byte, err error) {
+	if cpus, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.cpus")); err != nil {
+		return
+	}
+
+	if mems, err = ioutil.ReadFile(filepath.Join(parent, "cpuset.mems")); err != nil {
+		return
+	}
+
+	return cpus, mems, nil
+}

create_test.go

@@ -42,14 +42,14 @@ const (
 
 var testStrPID = fmt.Sprintf("%d", testPID)
 
-func testCreateCgroupsFilesSuccessful(t *testing.T, cgroupsPathList []string, pid int) {
-	if err := createCgroupsFiles("foo", cgroupsPathList, pid); err != nil {
+func testCreateCgroupsFilesSuccessful(t *testing.T, cgroupsDirPath string, cgroupsPathList []string, pid int) {
+	if err := createCgroupsFiles("foo", cgroupsDirPath, cgroupsPathList, pid); err != nil {
 		t.Fatalf("This test should succeed (cgroupsPath %q, pid %d): %s", cgroupsPathList, pid, err)
 	}
 }
 
 func TestCgroupsFilesEmptyCgroupsPathSuccessful(t *testing.T) {
-	testCreateCgroupsFilesSuccessful(t, []string{}, testPID)
+	testCreateCgroupsFilesSuccessful(t, "", []string{}, testPID)
 }
 
 func TestCreateCgroupsFilesFailToWriteFile(t *testing.T) {
@@ -71,7 +71,7 @@ func TestCreateCgroupsFilesFailToWriteFile(t *testing.T) {
 
 	files := []string{file}
 
-	err = createCgroupsFiles("foo", files, testPID)
+	err = createCgroupsFiles("foo, ""cgroups-file", files, testPID)
 	assert.Error(err)
 }
 
@@ -81,7 +81,7 @@ func TestCgroupsFilesNonEmptyCgroupsPathSuccessful(t *testing.T) {
 		t.Fatalf("Could not create temporary cgroups directory: %s", err)
 	}
 
-	testCreateCgroupsFilesSuccessful(t, []string{cgroupsPath}, testPID)
+	testCreateCgroupsFilesSuccessful(t, "cgroups-path-", []string{cgroupsPath}, testPID)
 
 	defer os.RemoveAll(cgroupsPath)
 
@@ -1059,3 +1059,60 @@ func TestCreateCreateContainer(t *testing.T) {
 		assert.NoError(err)
 	}
 }
+
+func TestCopyParentCPUSetFail(t *testing.T) {
+	assert := assert.New(t)
+
+	cgroupsPath, err := ioutil.TempDir(testDir, "cgroups-path-")
+	if err != nil {
+		t.Fatalf("Could not create temporary cgroups directory: %s", err)
+	}
+	defer os.RemoveAll(cgroupsPath)
+
+	err = copyParentCPUSet(cgroupsPath, testDir)
+	assert.Error(err)
+}
+
+func TestCopyParentCPUSetSuccessful(t *testing.T) {
+	if os.Geteuid() != 0 {
+		// Create cgroup directory must be root.
+		t.Skip(testDisabledNeedRoot)
+	}
+
+	assert := assert.New(t)
+
+	cgroupsRootPath := findCgroupRoot()
+	if cgroupsRootPath == "" {
+		t.Skip(testDisabledNeedCgroup)
+	}
+
+	cgroupsCPUSetRoot := filepath.Join(cgroupsRootPath, "cpu")
+	if _, err := os.Stat(cgroupsCPUSetRoot); err != nil && os.IsNotExist(err) {
+		t.Skip(testDisabledNeedCgroup)
+	}
+
+	cpusetCpusPath := filepath.Join(cgroupsCPUSetRoot, "cpuset.cpus")
+	if _, err := os.Stat(cpusetCpusPath); err != nil && os.IsNotExist(err) {
+		t.Skip(testDisabledNeedCgroup)
+	}
+
+	cpusetMemsPath := filepath.Join(cgroupsCPUSetRoot, "cpuset.mems")
+	if _, err := os.Stat(cpusetMemsPath); err != nil && os.IsNotExist(err) {
+		t.Skip(testDisabledNeedCgroup)
+	}
+
+	cgroupsPath, err := ioutil.TempDir(cgroupsCPUSetRoot, "cgroups-path-")
+	if err != nil {
+		t.Fatalf("Could not create temporary cgroups directory: %s", err)
+	}
+	defer os.RemoveAll(cgroupsPath)
+
+	err = copyParentCPUSet(cgroupsPath, cgroupsCPUSetRoot)
+	assert.NoError(err)
+
+	currentCpus, currentMems, err := getCPUSet(cgroupsPath)
+	assert.NoError(err)
+
+	assert.False(isEmptyString(currentCpus))
+	assert.False(isEmptyString(currentMems))
+}

main_test.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/json"
 	"errors"
@@ -44,6 +45,7 @@ import (
 const (
 	testDisabledNeedRoot    = "Test disabled as requires root user"
 	testDisabledNeedNonRoot = "Test disabled as requires non-root user"
+	testDisabledNeedCgroup  = "Test disabled as cgroup system not support"
 	testDirMode             = os.FileMode(0750)
 	testFileMode            = os.FileMode(0640)
 	testExeFileMode         = os.FileMode(0750)
@@ -487,6 +489,34 @@ func writeOCIConfigFile(spec oci.CompatOCISpec, configPath string) error {
 	return ioutil.WriteFile(configPath, bytes, testFileMode)
 }
 
+func findCgroupRoot() string {
+	var cgroupRootPath string
+	f, err := os.Open("/proc/mounts")
+	if err != nil {
+		return cgroupRootPath
+	}
+	defer f.Close()
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		text := scanner.Text()
+		fields := strings.Split(text, " ")
+		index := strings.Index(text, " - ")
+		postSeparatorFields := strings.Fields(text[index+3:])
+		numPostFields := len(postSeparatorFields)
+
+		// This is an error as we can't detect if the mount is for "cgroup"
+		if numPostFields == 0 || fields[0] != "cgroup" || numPostFields < 3 {
+			continue
+		}
+
+		cgroupRootPath = filepath.Dir(fields[1])
+		break
+	}
+
+	return cgroupRootPath
+}
+
 func newSingleContainerPodStatusList(podID, containerID string, podState, containerState vc.State, annotations map[string]string) []vc.PodStatus {
 	return []vc.PodStatus{
 		{

utils.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"bytes"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -187,3 +188,21 @@ func runCommandFull(args []string, includeStderr bool) (string, error) {
 func runCommand(args []string) (string, error) {
 	return runCommandFull(args, false)
 }
+
+// writeFile write data into specified file
+func writeFile(filePath string, data string) error {
+	// Normally dir should not be empty, one case is that cgroup subsystem
+	// is not mounted, we will get empty dir, and we want it fail here.
+	if filePath == "" {
+		return fmt.Errorf("no such file for %s", filePath)
+	}
+	if err := ioutil.WriteFile(filePath, []byte(data), 0700); err != nil {
+		return fmt.Errorf("failed to write %v to %v: %v", data, filePath, err)
+	}
+	return nil
+}
+
+// isEmptyString return if string is empty
+func isEmptyString(b []byte) bool {
+	return len(bytes.Trim(b, "\n")) == 0
+}