From 3ae9358054cb381395c384155b3ba6a64ae6e1e7 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Wed, 25 Mar 2020 18:18:06 -0700 Subject: [PATCH 1/8] tests/integration: check_cgroup_value: simplify Consolidate two implementations of check_cgroup_value() into one, putting it into helpers. Remove the first parameter, deducing the variable to get the path from by the parameter name. This should help in future cgroupv2 support. Signed-off-by: Kir Kolyshkin --- tests/integration/cgroups.bats | 19 ++---- tests/integration/helpers.bash | 14 ++++ tests/integration/update.bats | 115 +++++++++++++++------------------ 3 files changed, 72 insertions(+), 76 deletions(-) diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats index 17812abfdcc..f925a0f8252 100644 --- a/tests/integration/cgroups.bats +++ b/tests/integration/cgroups.bats @@ -14,17 +14,6 @@ function setup() { setup_busybox } -function check_cgroup_value() { - cgroup=$1 - source=$2 - expected=$3 - - current=$(cat $cgroup/$source) - echo $cgroup/$source - echo "current" $current "!?" "$expected" - [ "$current" -eq "$expected" ] -} - @test "runc update --kernel-memory (initialized)" { [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup requires cgroups_kmem @@ -45,12 +34,14 @@ EOF runc run -d --console-socket $CONSOLE_SOCKET test_cgroups_kmem [ "$status" -eq 0 ] + check_cgroup_value "memory.kmem.limit_in_bytes" 16777216 + # update kernel memory limit runc update test_cgroups_kmem --kernel-memory 50331648 [ "$status" -eq 0 ] - # check the value - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 + # check the value + check_cgroup_value "memory.kmem.limit_in_bytes" 50331648 } @test "runc update --kernel-memory (uninitialized)" { @@ -71,7 +62,7 @@ EOF [ ! "$status" -eq 0 ] else [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 + check_cgroup_value "memory.kmem.limit_in_bytes" 50331648 fi } diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 8862dcb8b88..161fb84efeb 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -129,6 +129,20 @@ function set_cgroups_path() { sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${cgroups_path}"'",#' "$bundle/config.json" } +# Helper to check a value in cgroups. +function check_cgroup_value() { + source=$1 + expected=$2 + + ctrl=${source%%.*} + eval cgroup=\$CGROUP_${ctrl^^} + + current=$(cat $cgroup/$source) + echo $cgroup/$source + echo "current" $current "!?" "$expected" + [ "$current" -eq "$expected" ] +} + # Helper function to set a resources limit function set_resources_limit() { bundle="${1:-.}" diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 549b2ece831..bdfc169755e 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -38,15 +38,6 @@ EOF sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json } -function check_cgroup_value() { - cgroup=$1 - source=$2 - expected=$3 - - current=$(cat $cgroup/$source) - [ "$current" == "$expected" ] -} - # TODO: test rt cgroup updating @test "update" { # XXX: Also, this test should be split into separate sections so that we @@ -67,52 +58,52 @@ function check_cgroup_value() { CGROUP_SYSTEM_MEMORY=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<'MEMORY'\>/ { print $5; exit }') # check that initial values were properly set - check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 1000000 - check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 500000 - check_cgroup_value $CGROUP_CPU "cpu.shares" 100 - check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 0 - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 16777216 - check_cgroup_value $CGROUP_MEMORY "memory.kmem.tcp.limit_in_bytes" 11534336 - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 33554432 - check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 25165824 - check_cgroup_value $CGROUP_PIDS "pids.max" 20 + check_cgroup_value "cpu.cfs_period_us" 1000000 + check_cgroup_value "cpu.cfs_quota_us" 500000 + check_cgroup_value "cpu.shares" 100 + check_cgroup_value "cpuset.cpus" 0 + check_cgroup_value "memory.kmem.limit_in_bytes" 16777216 + check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 11534336 + check_cgroup_value "memory.limit_in_bytes" 33554432 + check_cgroup_value "memory.soft_limit_in_bytes" 25165824 + check_cgroup_value "pids.max" 20 # update cpu-period runc update test_update --cpu-period 900000 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.cfs_period_us" 900000 + check_cgroup_value "cpu.cfs_period_us" 900000 # update cpu-quota runc update test_update --cpu-quota 600000 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.cfs_quota_us" 600000 + check_cgroup_value "cpu.cfs_quota_us" 600000 # update cpu-shares runc update test_update --cpu-share 200 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPU "cpu.shares" 200 + check_cgroup_value "cpu.shares" 200 # update cpuset if supported (i.e. we're running on a multicore cpu) cpu_count=$(grep '^processor' /proc/cpuinfo | wc -l) if [ $cpu_count -gt 1 ]; then runc update test_update --cpuset-cpus "1" [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_CPUSET "cpuset.cpus" 1 + check_cgroup_value "cpuset.cpus" 1 fi # update memory limit runc update test_update --memory 67108864 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 67108864 + check_cgroup_value "memory.limit_in_bytes" 67108864 runc update test_update --memory 50M [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" 52428800 + check_cgroup_value "memory.limit_in_bytes" 52428800 # update memory soft limit runc update test_update --memory-reservation 33554432 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.soft_limit_in_bytes" 33554432 + check_cgroup_value "memory.soft_limit_in_bytes" 33554432 # Run swap memory tests if swap is available if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then @@ -121,12 +112,12 @@ function check_cgroup_value() { [ "$status" -eq 0 ] # Get System memory swap limit SYSTEM_MEMORY_SW=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.memsw.limit_in_bytes") - check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY_SW} + check_cgroup_value "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY_SW} # update memory swap runc update test_update --memory-swap 96468992 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" 96468992 + check_cgroup_value "memory.memsw.limit_in_bytes" 96468992 fi; # try to remove memory limit @@ -136,27 +127,27 @@ function check_cgroup_value() { # Get System memory limit SYSTEM_MEMORY=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.limit_in_bytes") # check memory limited is gone - check_cgroup_value $CGROUP_MEMORY "memory.limit_in_bytes" ${SYSTEM_MEMORY} + check_cgroup_value "memory.limit_in_bytes" ${SYSTEM_MEMORY} # check swap memory limited is gone if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then - check_cgroup_value $CGROUP_MEMORY "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY} + check_cgroup_value "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY} fi # update kernel memory limit runc update test_update --kernel-memory 50331648 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.kmem.limit_in_bytes" 50331648 + check_cgroup_value "memory.kmem.limit_in_bytes" 50331648 # update kernel memory tcp limit runc update test_update --kernel-memory-tcp 41943040 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_MEMORY "memory.kmem.tcp.limit_in_bytes" 41943040 + check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 41943040 # update pids limit runc update test_update --pids-limit 10 [ "$status" -eq 0 ] - check_cgroup_value $CGROUP_PIDS "pids.max" 10 + check_cgroup_value "pids.max" 10 # Revert to the test initial value via json on stding runc update -r - test_update < Date: Wed, 25 Mar 2020 18:30:44 -0700 Subject: [PATCH 2/8] tests/integration: simplify cgroup paths init 1. Consolidate all the cgroup-related initialization code to a single place, init_cgroup_paths(), so we can see which variables are set. 2. Lazily call init_cgroup_paths() from all places that require it. 3. Don't set globals KMEM and RT_PERIOD. 4. Slightly clarlify variables naming: - use OCI_CGROUPS_PATH for cgroupsPath in config.json - use REL_CGROUPS_PATH for relative cgroups path 5. Do not hardcode the list of cgroup subsystems -- get it from /proc/cgroup. 6. Preliminary support for cgroupv2 unified hierarchy (not yet working). Signed-off-by: Kir Kolyshkin --- tests/integration/helpers.bash | 62 ++++++++++++++++++++++------------ tests/integration/update.bats | 15 ++------ 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 161fb84efeb..0fc78f77ee7 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -36,20 +36,6 @@ ROOT=$(mktemp -d "$BATS_TMPDIR/runc.XXXXXX") # Path to console socket. CONSOLE_SOCKET="$BATS_TMPDIR/console.sock" -# Cgroup paths -CGROUP_MEMORY_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\/ { print $5; exit }') -CGROUP_CPU_BASE_PATH=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\/ { print $5; exit }') -if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then - CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope" -else - CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup" -fi -CGROUP_MEMORY="${CGROUP_MEMORY_BASE_PATH}${CGROUPS_PATH}" - -# CONFIG_MEMCG_KMEM support -KMEM="${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" -RT_PERIOD="${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" - # Check if we're in rootless mode. ROOTLESS=$(id -u) @@ -119,14 +105,44 @@ function runc_rootless_cgroup() { mv "$bundle/config.json"{.tmp,} } -# Helper function to set cgroupsPath to the value of $CGROUPS_PATH +function init_cgroup_paths() { + # init once + test -n "$CGROUP_UNIFIED" && return + + if [ -n "${RUNC_USE_SYSTEMD}" ] ; then + REL_CGROUPS_PATH="/machine.slice/runc-cgroups-integration-test.scope" + OCI_CGROUPS_PATH="machine.slice:runc-cgroups:integration-test" + else + REL_CGROUPS_PATH="/runc-cgroups-integration-test/test-cgroup" + OCI_CGROUPS_PATH=$REL_CGROUPS_PATH + fi + + if stat -f -c %t /sys/fs/cgroup | grep -qFw 63677270; then + CGROUP_UNIFIED=yes + # "pseudo" controllers do not appear in /sys/fs/cgroup/cgroup.controllers. + # - devices (since kernel 4.15) + # - freezer (since kernel 5.2) + # Assume these are always available, as it is hard to detect + CGROUP_SUBSYSTEMS=$(cat /sys/fs/cgroup/cgroup.controllers; echo devices freezer) + CGROUP_BASE_PATH=/sys/fs/cgroup + CGROUP_PATH=${CGROUP_BASE_PATH}${REL_CGROUPS_PATH} + else + CGROUP_UNIFIED=no + CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups) + for g in ${CGROUP_SUBSYSTEMS}; do + base_path=$(gawk '$(NF-2) == "cgroup" && $NF ~ /\<'${g}'\>/ { print $5; exit }' /proc/self/mountinfo) + test -z "$base_path" && continue + eval CGROUP_${g^^}_BASE_PATH="${base_path}" + eval CGROUP_${g^^}="${base_path}${REL_CGROUPS_PATH}" + done + fi +} + +# Helper function to set cgroupsPath to the value of $OCI_CGROUPS_PATH function set_cgroups_path() { bundle="${1:-.}" - cgroups_path="/runc-cgroups-integration-test/test-cgroup" - if [[ -n "${RUNC_USE_SYSTEMD}" ]] ; then - cgroups_path="machine.slice:runc-cgroups:integration-test" - fi - sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${cgroups_path}"'",#' "$bundle/config.json" + init_cgroup_paths + sed -i 's#\("linux": {\)#\1\n "cgroupsPath": "'"${OCI_CGROUPS_PATH}"'",#' "$bundle/config.json" } # Helper to check a value in cgroups. @@ -191,12 +207,14 @@ function requires() { fi ;; cgroups_kmem) - if [ ! -e "$KMEM" ]; then + init_cgroup_paths + if [ ! -e "${CGROUP_MEMORY_BASE_PATH}/memory.kmem.limit_in_bytes" ]; then skip "Test requires ${var}" fi ;; cgroups_rt) - if [ ! -e "$RT_PERIOD" ]; then + init_cgroup_paths + if [ ! -e "${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" ]; then skip "Test requires ${var}" fi ;; diff --git a/tests/integration/update.bats b/tests/integration/update.bats index bdfc169755e..d8cf5761337 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -49,14 +49,6 @@ EOF runc run -d --console-socket $CONSOLE_SOCKET test_update [ "$status" -eq 0 ] - # get the cgroup paths - for g in MEMORY CPUSET CPU BLKIO PIDS; do - base_path=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<'${g}'\>/ { print $5; exit }') - eval CGROUP_${g}="${base_path}${CGROUPS_PATH}" - done - - CGROUP_SYSTEM_MEMORY=$(grep "cgroup" /proc/self/mountinfo | gawk 'toupper($NF) ~ /\<'MEMORY'\>/ { print $5; exit }') - # check that initial values were properly set check_cgroup_value "cpu.cfs_period_us" 1000000 check_cgroup_value "cpu.cfs_quota_us" 500000 @@ -111,7 +103,7 @@ EOF runc update test_update --memory-swap -1 [ "$status" -eq 0 ] # Get System memory swap limit - SYSTEM_MEMORY_SW=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.memsw.limit_in_bytes") + SYSTEM_MEMORY_SW=$(cat "${CGROUP_MEMORY_BASE_PATH}/memory.memsw.limit_in_bytes") check_cgroup_value "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY_SW} # update memory swap @@ -125,7 +117,7 @@ EOF [ "$status" -eq 0 ] # Get System memory limit - SYSTEM_MEMORY=$(cat "${CGROUP_SYSTEM_MEMORY}/memory.limit_in_bytes") + SYSTEM_MEMORY=$(cat "${CGROUP_MEMORY_BASE_PATH}/memory.limit_in_bytes") # check memory limited is gone check_cgroup_value "memory.limit_in_bytes" ${SYSTEM_MEMORY} @@ -239,9 +231,6 @@ EOF runc run -d --console-socket $CONSOLE_SOCKET test_update_rt [ "$status" -eq 0 ] - # get the cgroup paths - eval CGROUP_CPU="${CGROUP_CPU_BASE_PATH}${CGROUPS_PATH}" - runc update -r - test_update_rt < Date: Mon, 30 Mar 2020 13:37:53 -0700 Subject: [PATCH 3/8] tests/integration/update.bats: rm obsoleted comment This comment was added by commit 6cd425be2be2 (Allow update rt_period_us and rt_runtime_us, Nov 4 2016), and the test case was added by commit 51baedf3f3cb0 (Add integration for update rt period and runtime, Nov 28 2016), making the comment obsolete. Remove it. Signed-off-by: Kir Kolyshkin --- tests/integration/update.bats | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/update.bats b/tests/integration/update.bats index d8cf5761337..32c333597ea 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -38,7 +38,6 @@ EOF sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json } -# TODO: test rt cgroup updating @test "update" { # XXX: Also, this test should be split into separate sections so that we # can skip kmem without skipping update tests overall. From b8b46419ce87a65262c137ff8d119a7d63401ae7 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Mon, 30 Mar 2020 14:00:35 -0700 Subject: [PATCH 4/8] tests/integration: rm kmem from upgrade tests ... and add kmem-tcp to cgroups kmem test. First, we already have a separate kmem test in cgroups.bats. Second, making kmem a requirement leads to skipping all the other test cases in the update.bats test. Third, kmem limit is being removed from the kernel, so it makes sense to handle it separately. Signed-off-by: Kir Kolyshkin --- tests/integration/cgroups.bats | 13 ++++++++---- tests/integration/update.bats | 39 ++++++---------------------------- 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats index f925a0f8252..6abb2ed6b02 100644 --- a/tests/integration/cgroups.bats +++ b/tests/integration/cgroups.bats @@ -14,7 +14,7 @@ function setup() { setup_busybox } -@test "runc update --kernel-memory (initialized)" { +@test "runc update --kernel-memory{,-tcp} (initialized)" { [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup requires cgroups_kmem @@ -23,7 +23,8 @@ function setup() { # Set some initial known values DATA=$(cat <<-EOF "memory": { - "kernel": 16777216 + "kernel": 16777216, + "kernelTCP": 11534336 }, EOF ) @@ -35,13 +36,17 @@ EOF [ "$status" -eq 0 ] check_cgroup_value "memory.kmem.limit_in_bytes" 16777216 + check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 11534336 # update kernel memory limit runc update test_cgroups_kmem --kernel-memory 50331648 [ "$status" -eq 0 ] - - # check the value check_cgroup_value "memory.kmem.limit_in_bytes" 50331648 + + # update kernel memory tcp limit + runc update test_cgroups_kmem --kernel-memory-tcp 41943040 + [ "$status" -eq 0 ] + check_cgroup_value "memory.kmem.tcp.limit_in_bytes" 41943040 } @test "runc update --kernel-memory (uninitialized)" { diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 32c333597ea..492d1810dca 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -19,9 +19,7 @@ function setup() { DATA=$(cat < Date: Mon, 30 Mar 2020 15:04:12 -0700 Subject: [PATCH 5/8] tests/integration/update.bats: simplify file creation There's no need for an intermediate variable Signed-off-by: Kir Kolyshkin --- tests/integration/update.bats | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 492d1810dca..8d9f5f8d733 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -164,7 +164,7 @@ EOF check_cgroup_value "pids.max" 10 # reset to initial test value via json file - DATA=$(cat <<"EOF" + cat << EOF > $BATS_TMPDIR/runc-cgroups-integration-test.json { "memory": { "limit": 33554432, @@ -181,8 +181,6 @@ EOF } } EOF -) - echo $DATA > $BATS_TMPDIR/runc-cgroups-integration-test.json runc update -r $BATS_TMPDIR/runc-cgroups-integration-test.json test_update [ "$status" -eq 0 ] From 483f9a0c50d11fd87e066c0b322e4858fad21499 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Tue, 31 Mar 2020 15:27:51 -0700 Subject: [PATCH 6/8] tests/integration: add some cgroup v2 tests 1. Add `cgroups_v1` and `cgroups_v2` options to `requires`. 2. Modify `check_cgroup_value` to be able to work with v2. 3. Split `test "update"` into two: - (1) testing cgroupv1-only cpu shares and cfs - (2) testing limits that are more or less common between v1 and v2: memory/swap, pids, cpusets. Signed-off-by: Kir Kolyshkin --- tests/integration/helpers.bash | 22 ++++- tests/integration/update.bats | 164 ++++++++++++++++++++++----------- 2 files changed, 131 insertions(+), 55 deletions(-) diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index 0fc78f77ee7..de8be4ba53a 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -150,13 +150,17 @@ function check_cgroup_value() { source=$1 expected=$2 - ctrl=${source%%.*} - eval cgroup=\$CGROUP_${ctrl^^} + if [ "x$CGROUP_UNIFIED" = "xyes" ] ; then + cgroup=$CGROUP_PATH + else + ctrl=${source%%.*} + eval cgroup=\$CGROUP_${ctrl^^} + fi current=$(cat $cgroup/$source) echo $cgroup/$source echo "current" $current "!?" "$expected" - [ "$current" -eq "$expected" ] + [ "$current" = "$expected" ] } # Helper function to set a resources limit @@ -218,6 +222,18 @@ function requires() { skip "Test requires ${var}" fi ;; + cgroups_v1) + init_cgroup_paths + if [ "$CGROUP_UNIFIED" != "no" ]; then + skip "Test requires cgroups v1" + fi + ;; + cgroups_v2) + init_cgroup_paths + if [ "$CGROUP_UNIFIED" != "yes" ]; then + skip "Test requires cgroups v2 (unified)" + fi + ;; *) fail "BUG: Invalid requires ${var}." ;; diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 8d9f5f8d733..6c82a065418 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -36,39 +36,43 @@ EOF sed -i "s/\(\"resources\": {\)/\1\n${DATA}/" ${BUSYBOX_BUNDLE}/config.json } -@test "update" { +# Tests whatever limits are (more or less) common between cgroup +# v1 and v2: memory/swap, pids, and cpuset. +@test "update cgroup v1/v2 common limits" { [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup + init_cgroup_paths # run a few busyboxes detached runc run -d --console-socket $CONSOLE_SOCKET test_update [ "$status" -eq 0 ] + # Set a few variables to make the code below work for both v1 and v2 + case $CGROUP_UNIFIED in + no) + MEM_LIMIT="memory.limit_in_bytes" + MEM_RESERVE="memory.soft_limit_in_bytes" + MEM_SWAP="memory.memsw.limit_in_bytes" + SYSTEM_MEM=$(cat "${CGROUP_MEMORY_BASE_PATH}/${MEM_LIMIT}") + SYSTEM_MEM_SWAP=$(cat "${CGROUP_MEMORY_BASE_PATH}/$MEM_SWAP") + ;; + yes) + MEM_LIMIT="memory.max" + MEM_RESERVE="memory.low" + MEM_SWAP="memory.swap.max" + SYSTEM_MEM="max" + SYSTEM_MEM_SWAP="max" + CGROUP_MEMORY=$CGROUP_PATH + ;; + esac + # check that initial values were properly set - check_cgroup_value "cpu.cfs_period_us" 1000000 - check_cgroup_value "cpu.cfs_quota_us" 500000 - check_cgroup_value "cpu.shares" 100 check_cgroup_value "cpuset.cpus" 0 - check_cgroup_value "memory.limit_in_bytes" 33554432 - check_cgroup_value "memory.soft_limit_in_bytes" 25165824 + check_cgroup_value $MEM_LIMIT 33554432 + check_cgroup_value $MEM_RESERVE 25165824 check_cgroup_value "pids.max" 20 - # update cpu-period - runc update test_update --cpu-period 900000 - [ "$status" -eq 0 ] - check_cgroup_value "cpu.cfs_period_us" 900000 - - # update cpu-quota - runc update test_update --cpu-quota 600000 - [ "$status" -eq 0 ] - check_cgroup_value "cpu.cfs_quota_us" 600000 - - # update cpu-shares - runc update test_update --cpu-share 200 - [ "$status" -eq 0 ] - check_cgroup_value "cpu.shares" 200 - # update cpuset if supported (i.e. we're running on a multicore cpu) - cpu_count=$(grep '^processor' /proc/cpuinfo | wc -l) + cpu_count=$(grep -c '^processor' /proc/cpuinfo) if [ $cpu_count -gt 1 ]; then runc update test_update --cpuset-cpus "1" [ "$status" -eq 0 ] @@ -78,44 +82,40 @@ EOF # update memory limit runc update test_update --memory 67108864 [ "$status" -eq 0 ] - check_cgroup_value "memory.limit_in_bytes" 67108864 + check_cgroup_value $MEM_LIMIT 67108864 runc update test_update --memory 50M [ "$status" -eq 0 ] - check_cgroup_value "memory.limit_in_bytes" 52428800 + check_cgroup_value $MEM_LIMIT 52428800 # update memory soft limit runc update test_update --memory-reservation 33554432 [ "$status" -eq 0 ] - check_cgroup_value "memory.soft_limit_in_bytes" 33554432 + check_cgroup_value "$MEM_RESERVE" 33554432 # Run swap memory tests if swap is available - if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then + if [ -f "$CGROUP_MEMORY/$MEM_SWAP" ]; then # try to remove memory swap limit runc update test_update --memory-swap -1 [ "$status" -eq 0 ] - # Get System memory swap limit - SYSTEM_MEMORY_SW=$(cat "${CGROUP_MEMORY_BASE_PATH}/memory.memsw.limit_in_bytes") - check_cgroup_value "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY_SW} + check_cgroup_value "$MEM_SWAP" $SYSTEM_MEM_SWAP # update memory swap runc update test_update --memory-swap 96468992 [ "$status" -eq 0 ] - check_cgroup_value "memory.memsw.limit_in_bytes" 96468992 - fi; + check_cgroup_value "$MEM_SWAP" 96468992 + fi # try to remove memory limit runc update test_update --memory -1 [ "$status" -eq 0 ] - # Get System memory limit - SYSTEM_MEMORY=$(cat "${CGROUP_MEMORY_BASE_PATH}/memory.limit_in_bytes") - # check memory limited is gone - check_cgroup_value "memory.limit_in_bytes" ${SYSTEM_MEMORY} + # check memory limit is gone + check_cgroup_value $MEM_LIMIT $SYSTEM_MEM # check swap memory limited is gone - if [ -f "$CGROUP_MEMORY/memory.memsw.limit_in_bytes" ]; then - check_cgroup_value "memory.memsw.limit_in_bytes" ${SYSTEM_MEMORY} + if [ -f "$CGROUP_MEMORY/$MEM_SWAP" ]; then + check_cgroup_value $MEM_SWAP $SYSTEM_MEM fi # update pids limit @@ -123,7 +123,7 @@ EOF [ "$status" -eq 0 ] check_cgroup_value "pids.max" 10 - # Revert to the test initial value via json on stding + # Revert to the test initial value via json on stdin runc update -r - test_update < $BATS_TMPDIR/runc-cgroups-integration-test.json +{ + "cpu": { + "shares": 100, + "quota": 500000, + "period": 1000000 + } +} EOF runc update -r $BATS_TMPDIR/runc-cgroups-integration-test.json test_update @@ -187,10 +251,6 @@ EOF check_cgroup_value "cpu.cfs_period_us" 1000000 check_cgroup_value "cpu.cfs_quota_us" 500000 check_cgroup_value "cpu.shares" 100 - check_cgroup_value "cpuset.cpus" 0 - check_cgroup_value "memory.limit_in_bytes" 33554432 - check_cgroup_value "memory.soft_limit_in_bytes" 25165824 - check_cgroup_value "pids.max" 20 } @test "update rt period and runtime" { From 0965c970fad484c88a4c8653767fd46a68349d3e Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Fri, 3 Apr 2020 17:15:07 -0700 Subject: [PATCH 7/8] tests/integration: disable swap tests for v2 Swap setting for cgroupv2 is currently broken, so let's temporarily disable this part of test. Signed-off-by: Kir Kolyshkin --- tests/integration/update.bats | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 6c82a065418..61ce69376c8 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -61,7 +61,8 @@ EOF MEM_SWAP="memory.swap.max" SYSTEM_MEM="max" SYSTEM_MEM_SWAP="max" - CGROUP_MEMORY=$CGROUP_PATH + # checking swap is currently disabled for v2 + #CGROUP_MEMORY=$CGROUP_PATH ;; esac From 84583eb1a4a5a53ca79bba32eb67592c26c25704 Mon Sep 17 00:00:00 2001 From: Kir Kolyshkin Date: Fri, 3 Apr 2020 23:12:43 -0700 Subject: [PATCH 8/8] Enable integration tests in cgroupv2 env Those needs to be run on the (Vagrant Fedora 31) host (since we need real systemd running), and so we have to have all the tools needed to compile runc and run the tests. The good news is Fedora packages a decent and recent release of bats-core (1.1.0), which we can use (Debian does not), and we can also use golang (currently 1.13.9) from Fedora. The bad news are 1. Currently cgroups tests are only working with RUNC_USE_SYSTEMD=yes (addressed by #2299, #2305) 2. Tests in events.bats do not work (need cgroupv2 memory.events support) 3. Fedora 31 image is 6 months old (and has broken container-selinux policy) so we need `dnf update`, which adds ~5 min to test time. [v2: add -t to ssh to enforce pty] [v3: disable events tests for cgroupv2] [v4: update fedora packages, use a single dnf transation] Signed-off-by: Kir Kolyshkin --- .travis.yml | 2 ++ Vagrantfile | 8 +++++++- tests/integration/events.bats | 12 ++++++++---- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3e12b820bfd..84213eadb28 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,6 +34,8 @@ matrix: - sudo ssh default sudo podman build -t test /vagrant # Mounting /lib/modules into the container is necessary as CRIU wants to load (via iptables) additional modules - sudo ssh default sudo podman run --privileged --cgroupns=private -v /lib/modules:/lib/modules:ro test make localunittest + # cgroupv2+systemd: test on vagrant host itself as we need systemd + - sudo ssh default -t 'cd /vagrant && sudo make localintegration RUNC_USE_SYSTEMD=yes' allow_failures: - go: tip diff --git a/Vagrantfile b/Vagrantfile index 165b078018d..aba0c0cd717 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -13,6 +13,12 @@ Vagrant.configure("2") do |config| v.cpus = 2 end config.vm.provision "shell", inline: <<-SHELL - dnf install -y podman + cat << EOF | dnf -y shell +config install_weak_deps: False +update +install podman make golang-go libseccomp-devel bats jq +ts run +EOF + dnf clean all SHELL end diff --git a/tests/integration/events.bats b/tests/integration/events.bats index b3e6315b555..8f2f81a23d0 100644 --- a/tests/integration/events.bats +++ b/tests/integration/events.bats @@ -13,7 +13,8 @@ function teardown() { @test "events --stats" { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox @@ -28,7 +29,8 @@ function teardown() { @test "events --interval default " { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox @@ -55,7 +57,8 @@ function teardown() { @test "events --interval 1s " { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox @@ -81,7 +84,8 @@ function teardown() { @test "events --interval 100ms " { # XXX: currently cgroups require root containers. - requires root + # TODO: support cgroup v2 memory.events + requires root cgroups_v1 # run busybox detached runc run -d --console-socket $CONSOLE_SOCKET test_busybox