!2207 [refactor] update possible changed resources for oci spec

From: @duguhaotian Reviewed-by: @xuxuepeng, @taotao-sauce, @wangfengtu, @zh_xiaoyu Signed-off-by: @zh_xiaoyu
openeuler-mirror · Oct 23, 2023 · d473a1d · d473a1d
2 parents e252d7b + 3db983b
commit d473a1d
Show file tree

Hide file tree

Showing 11 changed files with 339 additions and 100 deletions.
diff --git a/CI/test_cases/container_cases/dev_cgroup_rule.sh b/CI/test_cases/container_cases/dev_cgroup_rule.sh
@@ -25,8 +25,10 @@ source ../helpers.sh
 function test_cpu_dev_cgoup_rule_spec()
 {
     local ret=0
+    local runtime=$1
     local image="busybox"
-    local test="container device cgroup rule test => (${FUNCNAME[@]})"
+    local test="container device cgroup rule test with (${runtime}) => (${FUNCNAME[@]})"
+    local test_dev="/dev/testA"
 
     msg_info "${test} starting..."
 
@@ -36,6 +38,47 @@ function test_cpu_dev_cgoup_rule_spec()
     isula images | grep busybox
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - missing list image: ${image}" && ((ret++))
 
+    rm -f $test_dev
+    priv_cid=$(isula run -tid --privileged --runtime $runtime $image /bin/sh)
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run priviledged container failed" && ((ret++))
+    priv_major_88_cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$priv_cid/config.json | grep "major\": 88" | wc -l)
+    priv_minor_88_cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$priv_cid/config.json | grep "minor\": 88" | wc -l)
+
+    mknod $test_dev c 88 88
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mknod failed" && ((ret++))
+    isula restart -t 0 $priv_cid
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart priviledge container failed" && ((ret++))
+    cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$priv_cid/config.json | grep "major\": 88" | wc -l)
+    [[ $? -ne 0 ]]&& [[ $cnt -le $priv_major_88_cnt ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++))
+    cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$priv_cid/config.json | grep "minor\": 88" | wc -l)
+    [[ $? -ne 0 ]] && [[ $cnt -le $priv_minor_88_cnt ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++))
+    isula rm -f $priv_cid
+
+    cid=$(isula run -tid --device "$test_dev:$test_dev" --runtime $runtime $image /bin/sh)
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - run container failed" && ((ret++))
+    cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "major\": 88" | wc -l)
+    [[ $? -ne 0 ]]&& [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++))
+    cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "minor\": 88" | wc -l)
+    [[ $? -ne 0 ]] && [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++))
+    isula exec -it $cid sh -c "cat /sys/fs/cgroup/devices/devices.list" | grep "c 88:88 rwm"
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check c 88:88 rwm: ${image}" && ((ret++))
+    isula stop -t 0 $cid
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop container failed" && ((ret++))
+    rm -f $test_dev
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm device failed" && ((ret++))
+    mknod $test_dev c 99 99
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - mknod failed" && ((ret++))
+    isula start $cid
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start container failed" && ((ret++))
+    cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "major\": 99" | wc -l)
+    [[ $? -ne 0 ]]&& [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device major failed" && ((ret++))
+    cnt=$(cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "minor\": 99" | wc -l)
+    [[ $? -ne 0 ]] && [[ $cnt -ne 2 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check device minor failed" && ((ret++))
+    isula exec -it $cid sh -c "cat /sys/fs/cgroup/devices/devices.list" | grep "c 99:99 rwm"
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - failed to check c 99:99 rwm: ${image}" && ((ret++))
+    isula rm -f $cid
+    rm -f $test_dev
+
     isula run -itd --device-cgroup-rule='b *:*' busybox 2>&1 | grep "Invalid value"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - Invalid value" && ((ret++))
 
@@ -93,6 +136,9 @@ function test_cpu_dev_cgoup_rule_spec()
 
 declare -i ans=0
 
-test_cpu_dev_cgoup_rule_spec || ((ans++))
+for element in ${RUNTIME_LIST[@]};
+do
+    test_cpu_dev_cgoup_rule_spec $element || ((ans++))
+done
 
 show_result ${ans} "${curr_path}/${0}"
diff --git a/CI/test_cases/container_cases/ulimit.sh b/CI/test_cases/container_cases/ulimit.sh
@@ -33,12 +33,25 @@ function test_ulimit()
     local test="ulimit test with (${runtime})=> (${FUNCNAME[@]})"
     msg_info "${test} starting..."
 
+    cid=$(isula run -tid --runtime $runtime $image /bin/sh)
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++))
+    cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "RLIMIT_"
+    [[ $? -eq 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check rlimit failed" && ((ret++))
+
     check_valgrind_log
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - stop isulad failed" && ((ret++))
 
     start_isulad_with_valgrind --default-ulimit nproc=2048:4096 --default-ulimit nproc=2048:8192 --default-ulimit nofile=1024:4096
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - start isulad failed" && ((ret++))
 
+    # if default ulimit of isulad changed, isula start should do update ulimit of oci spec
+    isula restart -t 0 $cid
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - restart failed" && ((ret++))
+    cat ${RUNTIME_ROOT_PATH}/${runtime}/$cid/config.json | grep "RLIMIT_"
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check rlimit failed after restart" && ((ret++))
+    isula rm -f $cid
+    [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - rm container failed" && ((ret++))
+
     isula run --ulimit nproc= $image --runtime $runtime /bin/sh > $ulimitlog 2>&1
     cat $ulimitlog | grep "delimiter '=' can't be the first or the last character"
     [[ $? -ne 0 ]] && msg_err "${FUNCNAME[0]}:${LINENO} - check failed" && ((ret++))

diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
@@ -1313,7 +1313,7 @@ static int ensure_isulad_tmpdir_security()
     if (do_ensure_isulad_tmpdir_security("/tmp") != 0) {
         WARN("Failed to ensure the /tmp directory is a safe directory");
     }
-    
+
     return 0;
 }
 
@@ -1380,12 +1380,17 @@ static int isulad_server_init_common()
     }
 #endif
 
-    if (containers_store_init()) {
+    if (spec_module_init() != 0) {
+        ERROR("Failed to init spec module");
+        goto out;
+    }
+
+    if (containers_store_init() != 0) {
         ERROR("Failed to init containers store");
         goto out;
     }
 
-    if (container_name_index_init()) {
+    if (container_name_index_init() != 0) {
         ERROR("Failed to init name index");
         goto out;
     }
@@ -1791,7 +1796,7 @@ int main(int argc, char **argv)
         ERROR("Failed to init plugin_manager");
         goto failure;
     }
-#endif 
+#endif
 
     clock_gettime(CLOCK_MONOTONIC, &t_end);
     use_time = (double)(t_end.tv_sec - t_start.tv_sec) * (double)1000000000 + (double)(t_end.tv_nsec - t_start.tv_nsec);

diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
@@ -1042,7 +1042,7 @@ static defs_hook *hooks_elem_dup(const defs_hook *src)
     dest = (defs_hook *)util_common_calloc_s(sizeof(defs_hook));
     if (dest == NULL) {
         ERROR("Out of memory");
-        return NULL;        
+        return NULL;
     }
 
     dest->path = util_strdup_s(src->path);
@@ -1090,7 +1090,7 @@ static int hooks_array_dup(const defs_hook **src, const size_t src_len, defs_hoo
         return -1;
     }
 
-    for(i = 0; i < src_len; i++) {
+    for (i = 0; i < src_len; i++) {
         tmp_dst[i] = hooks_elem_dup(src[i]);
         if (tmp_dst[i] == NULL) {
             ERROR("Failed to duplicate hooks element");
@@ -1104,7 +1104,7 @@ static int hooks_array_dup(const defs_hook **src, const size_t src_len, defs_hoo
     return 0;
 
 err_out:
-    for(i = 0; i < tmp_len; i++) {
+    for (i = 0; i < tmp_len; i++) {
         free_defs_hook(tmp_dst[i]);
     }
     free(tmp_dst);

diff --git a/src/daemon/modules/api/specs_api.h b/src/daemon/modules/api/specs_api.h
@@ -30,8 +30,7 @@ int merge_all_specs(host_config *host_spec, const char *real_rootfs, container_c
                     oci_runtime_spec *oci_spec);
 char *merge_container_cgroups_path(const char *id, const host_config *host_spec);
 int merge_global_config(oci_runtime_spec *oci_spec);
-oci_runtime_spec *load_oci_config(const char *rootpath, const char *name);
-oci_runtime_spec *default_spec(bool system_container);
+
 int merge_conf_cgroup(oci_runtime_spec *oci_spec, const host_config *host_spec);
 int save_oci_config(const char *id, const char *rootpath, const oci_runtime_spec *oci_spec);
 
@@ -41,6 +40,15 @@ int parse_security_opt(const host_config *host_spec, bool *no_new_privileges, ch
 int merge_share_namespace(oci_runtime_spec *oci_spec, const host_config *host_spec,
                           const container_config_v2_common_config *v2_spec,
                           const container_network_settings *network_settings);
+
+oci_runtime_spec *load_oci_config(const char *rootpath, const char *name);
+
+oci_runtime_spec *default_spec(bool system_container);
+
+const oci_runtime_spec *get_readonly_default_oci_spec(bool system_container);
+
+int spec_module_init(void);
+
 #ifdef __cplusplus
 }
 #endif

diff --git a/src/daemon/modules/container/container_events_handler.c b/src/daemon/modules/container/container_events_handler.c
@@ -157,7 +157,7 @@ static int container_state_changed(container_t *cont, const struct isulad_events
                 container_wait_stop_cond_broadcast(cont);
 #ifdef ENABLE_PLUGIN
                 plugin_event_container_post_stop(cont);
-#endif 
+#endif
             }
 
             auto_remove = !should_restart && cont->hostconfig != NULL && cont->hostconfig->auto_remove;

diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
@@ -13,20 +13,9 @@
  * Description: provide container supervisor functions
  ******************************************************************************/
 #define _GNU_SOURCE
-#include <sys/stat.h>
 #include <unistd.h>
-#include <sys/mount.h>
-#include <sys/eventfd.h>
-#include <sys/epoll.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <isula_libutils/container_config.h>
-#include <isula_libutils/container_config_v2.h>
-#include <isula_libutils/container_exec_request.h>
-#include <isula_libutils/container_exec_response.h>
-#include <isula_libutils/defs.h>
-#include <isula_libutils/host_config.h>
-#include <isula_libutils/oci_runtime_spec.h>
 #include <limits.h>
 #include <pthread.h>
 #include <signal.h>
@@ -36,15 +25,29 @@
 #include <stdlib.h>
 #include <string.h>
 #include <strings.h>
+#include <sys/stat.h>
+#include <sys/mount.h>
+#include <sys/eventfd.h>
+#include <sys/epoll.h>
+
+#include <isula_libutils/container_config.h>
+#include <isula_libutils/container_config_v2.h>
+#include <isula_libutils/container_exec_request.h>
+#include <isula_libutils/container_exec_response.h>
+#include <isula_libutils/defs.h>
+#include <isula_libutils/host_config.h>
+#include <isula_libutils/oci_runtime_spec.h>
+#include <isula_libutils/log.h>
+#include <isula_libutils/auto_cleanup.h>
 
 #include "service_container_api.h"
-#include "isula_libutils/log.h"
 #include "utils.h"
 #include "err_msg.h"
 #include "events_sender_api.h"
 #include "image_api.h"
 #include "specs_api.h"
 #include "specs_mount.h"
+#include "specs_extend.h"
 #include "isulad_config.h"
 #include "verify.h"
 #include "plugin_api.h"
@@ -688,6 +691,42 @@ static void wait_exit_fifo(const char *id, const int exit_fifo_fd)
     epoll_loop_close(&descr);
 }
 
+static int do_oci_spec_update(const char *id, oci_runtime_spec *oci_spec, host_config *hostconfig)
+{
+    __isula_auto_free char *cgroup_parent = NULL;
+    int ret;
+
+    // If isulad daemon cgroup parent updated, we should update this config into oci spec
+    cgroup_parent = merge_container_cgroups_path(id, hostconfig);
+    if (cgroup_parent == NULL) {
+        return -1;
+    }
+    if (oci_spec->linux->cgroups_path != NULL && strcmp(oci_spec->linux->cgroups_path, cgroup_parent) != 0) {
+        free(oci_spec->linux->cgroups_path);
+        oci_spec->linux->cgroups_path = cgroup_parent;
+        cgroup_parent = NULL;
+    }
+
+    // For Linux.Resources, isula update will save changes into oci spec;
+    // so we just skip it;
+
+    // Remove old devices and update all devices
+    ret = update_devcies_for_oci_spec(oci_spec, hostconfig);
+    if (ret != 0) {
+        ERROR("Failed to do update devices for oci spec");
+        return -1;
+    }
+
+    // If isulad daemon ulimit updated, we should update this config into oci spec.
+    if (merge_global_ulimit(oci_spec) != 0) {
+        return -1;
+    }
+
+    // renew_oci_config() will update process->user and share namespace after.
+
+    return 0;
+}
+
 static int do_start_container(container_t *cont, const char *console_fifos[], bool reset_rm, pid_ppid_info_t *pid_info)
 {
     int ret = 0;
@@ -762,6 +801,14 @@ static int do_start_container(container_t *cont, const char *console_fifos[], bo
         goto close_exit_fd;
     }
 
+    // Update possible changes
+    nret = do_oci_spec_update(id, oci_spec, cont->hostconfig);
+    if (nret != 0) {
+        ERROR("Failed to update possible changes for oci spec");
+        ret = -1;
+        goto close_exit_fd;
+    }
+
     nret = setup_ipc_dirs(cont->hostconfig, cont->common_config);
     if (nret != 0) {
         ERROR("Failed to setup ipc dirs");