From ba68e1a63dab31b77a28efded33a193cc0c6de66 Mon Sep 17 00:00:00 2001 From: Peter Pietzuch Date: Tue, 18 Aug 2020 20:27:47 +0000 Subject: [PATCH 1/6] Make the idle host task execute the shutdown sequence --- arch/lkl/kernel/setup.c | 5 ++++ arch/lkl/kernel/syscalls.c | 57 ++++++++++++++++++++++++++++---------- arch/lkl/kernel/threads.c | 14 ++++++---- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/arch/lkl/kernel/setup.c b/arch/lkl/kernel/setup.c index 4e8524dc659dc3..29106148a29bb2 100644 --- a/arch/lkl/kernel/setup.c +++ b/arch/lkl/kernel/setup.c @@ -163,13 +163,18 @@ long lkl_sys_halt(void) is_running = false; + lkl_printf("lkl_sys_halt(): Shutting down CPU\n"); lkl_cpu_wait_shutdown(); + lkl_printf("lkl_sys_halt(): Syscalls cleanup\n"); syscalls_cleanup(); + + lkl_printf("lkl_sys_halt(): Threads cleanup\n"); threads_cleanup(); /* Shutdown the clockevents source. */ tick_suspend_local(); free_mem(); + lkl_printf("lkl_sys_halt(): thread_join()\n"); lkl_ops->thread_join(current_thread_info()->tid); return 0; diff --git a/arch/lkl/kernel/syscalls.c b/arch/lkl/kernel/syscalls.c index e28e2be79b4769..e9a9b611150135 100644 --- a/arch/lkl/kernel/syscalls.c +++ b/arch/lkl/kernel/syscalls.c @@ -118,6 +118,12 @@ static void del_host_task(void *arg) struct lkl_tls_key *task_key; +/* Record the exit status after shutdown */ +_Atomic(int) lkl_exit_status; + +/* Record received signal after shutdown */ +_Atomic(int) lkl_received_signal; + /* Use this to record an ongoing LKL shutdown */ _Atomic(bool) lkl_shutdown = false; @@ -139,6 +145,23 @@ long lkl_syscall(long no, long *params) no, current->comm, test_tsk_thread_flag(task, TIF_HOST_THREAD), test_tsk_thread_flag(task, TIF_SIGPENDING)); + /** + * If there is an ongoing shutdown, we assume that the caller (the idle + * host task) already holds the CPU lock. + */ + if (lkl_shutdown) { + ret = run_syscall(no, params); + task_work_run(); + do_signal(NULL); + + if (no == __NR_reboot) { + thread_sched_jb(); + } + + LKL_TRACE("Shutdown syscall=%li done\n", no); + return ret; + } + ret = lkl_cpu_get(); if (ret < 0) { @@ -251,13 +274,7 @@ long lkl_syscall(long no, long *params) switch_to_host_task(task); } - /* - * Stop signal handling when LKL is shutting down. We cannot deliver - * signals because we are shutting down the kernel. - */ - if (!lkl_shutdown) { - do_signal(NULL); - } + do_signal(NULL); if (no == __NR_reboot) { thread_sched_jb(); @@ -275,6 +292,10 @@ long lkl_syscall(long no, long *params) static struct task_struct *idle_host_task; +extern _Atomic(int) lkl_exit_status; +extern _Atomic(int) lkl_received_signal; +extern _Atomic(bool) lkl_shutdown; + /* called from idle, don't failed, don't block */ void wakeup_idle_host_task(void) { @@ -295,6 +316,21 @@ static int idle_host_task_loop(void *unused) for (;;) { lkl_cpu_put(); lkl_ops->sem_down(ti->sched_sem); + + if (lkl_shutdown) { + + /** + * Notify the host of the shutdown. + * + * Note that we are not releasing the CPU lock here, which allows + * the termination code to do syscalls directly. + */ + lkl_ops->terminate(lkl_exit_status, lkl_received_signal); + + lkl_ops->thread_exit(); + return 0; + } + if (idle_host_task == NULL) { lkl_ops->thread_exit(); return 0; @@ -368,13 +404,6 @@ int host0_init(void) void syscalls_cleanup(void) { LKL_TRACE("enter\n"); - if (idle_host_task) { - struct thread_info *ti = task_thread_info(idle_host_task); - - idle_host_task = NULL; - lkl_ops->sem_up(ti->sched_sem); - lkl_ops->thread_join(ti->tid); - } if (lkl_ops->tls_free) lkl_ops->tls_free(task_key); diff --git a/arch/lkl/kernel/threads.c b/arch/lkl/kernel/threads.c index 434a80cd01043f..18f413a1c12402 100644 --- a/arch/lkl/kernel/threads.c +++ b/arch/lkl/kernel/threads.c @@ -6,6 +6,8 @@ #include #include +extern _Atomic(int) lkl_exit_status; +extern _Atomic(int) lkl_received_signal; extern _Atomic(bool) lkl_shutdown; static int init_ti(struct thread_info *ti) @@ -89,8 +91,8 @@ static void kill_thread(struct thread_info *ti) * case, we need to notify the host to initiate an LKL shutdown. */ int exit_code = task->exit_code; - int exit_status = exit_code >> 8; - int received_signal = exit_code & 255; + lkl_exit_status = exit_code >> 8; + lkl_received_signal = exit_code & 255; int exit_signal = task->exit_signal; LKL_TRACE( @@ -98,16 +100,16 @@ static void kill_thread(struct thread_info *ti) "received_signal=%i ti->dead=%i task->pid=%i " "task->tgid=%i ti->TIF_SCHED_JB=%i ti->TIF_SIGPENDING=%i)\n", task->exit_state, exit_code, exit_signal, - exit_status, received_signal, ti->dead, + lkl_exit_status, lkl_received_signal, ti->dead, task->pid, task->tgid, test_ti_thread_flag(ti, TIF_SCHED_JB), test_ti_thread_flag(ti, TIF_SIGPENDING)); + /** + * Notify the idle host task to initiate the LKL shutdown + */ lkl_shutdown = true; - /* Notify the LKL host to shut down */ - lkl_ops->terminate(exit_status, received_signal); - ti->dead = true; } lkl_ops->sem_free(ti->sched_sem); From a53bb1160b757f8adea1cb593de516366e5080cf Mon Sep 17 00:00:00 2001 From: Peter Pietzuch Date: Tue, 18 Aug 2020 20:46:59 +0000 Subject: [PATCH 2/6] Avoid LKL CPU/thread shutdown This may block due hanging userspace threads. --- arch/lkl/kernel/setup.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/arch/lkl/kernel/setup.c b/arch/lkl/kernel/setup.c index 29106148a29bb2..6fa22416898bc9 100644 --- a/arch/lkl/kernel/setup.c +++ b/arch/lkl/kernel/setup.c @@ -163,19 +163,15 @@ long lkl_sys_halt(void) is_running = false; - lkl_printf("lkl_sys_halt(): Shutting down CPU\n"); - lkl_cpu_wait_shutdown(); + //lkl_cpu_wait_shutdown(); - lkl_printf("lkl_sys_halt(): Syscalls cleanup\n"); - syscalls_cleanup(); + //syscalls_cleanup(); - lkl_printf("lkl_sys_halt(): Threads cleanup\n"); - threads_cleanup(); + // threads_cleanup(); /* Shutdown the clockevents source. */ - tick_suspend_local(); - free_mem(); - lkl_printf("lkl_sys_halt(): thread_join()\n"); - lkl_ops->thread_join(current_thread_info()->tid); + //tick_suspend_local(); + //free_mem(); + //lkl_ops->thread_join(current_thread_info()->tid); return 0; } From 8552da7975f1db1e178b802a28409ea2c6d12aab Mon Sep 17 00:00:00 2001 From: Peter Pietzuch Date: Wed, 19 Aug 2020 06:41:58 +0000 Subject: [PATCH 3/6] Add comment about shutdown sequence workaround --- arch/lkl/kernel/setup.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/lkl/kernel/setup.c b/arch/lkl/kernel/setup.c index 6fa22416898bc9..b06ee2ef6773ce 100644 --- a/arch/lkl/kernel/setup.c +++ b/arch/lkl/kernel/setup.c @@ -163,6 +163,13 @@ long lkl_sys_halt(void) is_running = false; + /** + * TODO: We may have cloned host threads that did not terminate, + * and prevent us from shutting down the LKL CPU. + * + * As a workaround, we do not shutdown all threads. + */ + //lkl_cpu_wait_shutdown(); //syscalls_cleanup(); From 232d767249a2bb15b99cc0e970800ca4875b04fd Mon Sep 17 00:00:00 2001 From: Peter Pietzuch Date: Wed, 19 Aug 2020 08:33:59 +0000 Subject: [PATCH 4/6] Remove dead code --- arch/lkl/kernel/syscalls.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/arch/lkl/kernel/syscalls.c b/arch/lkl/kernel/syscalls.c index e9a9b611150135..ca116c60bd6731 100644 --- a/arch/lkl/kernel/syscalls.c +++ b/arch/lkl/kernel/syscalls.c @@ -331,10 +331,6 @@ static int idle_host_task_loop(void *unused) return 0; } - if (idle_host_task == NULL) { - lkl_ops->thread_exit(); - return 0; - } schedule_tail(ti->prev_sched); } } From 89513917971fb0ffb1f29a86f6d084037404af16 Mon Sep 17 00:00:00 2001 From: Peter Pietzuch Date: Thu, 20 Aug 2020 16:10:36 +0000 Subject: [PATCH 5/6] Make terminating app thread join --- arch/lkl/kernel/syscalls.c | 17 ++++++++++++++--- arch/lkl/kernel/threads.c | 4 ++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/arch/lkl/kernel/syscalls.c b/arch/lkl/kernel/syscalls.c index ca116c60bd6731..3ffcdb133d08e2 100644 --- a/arch/lkl/kernel/syscalls.c +++ b/arch/lkl/kernel/syscalls.c @@ -127,8 +127,9 @@ _Atomic(int) lkl_received_signal; /* Use this to record an ongoing LKL shutdown */ _Atomic(bool) lkl_shutdown = false; -/* Returns the task_struct associated with the current lthread */ +static struct task_struct *idle_host_task; +/* Returns the task_struct associated with the current lthread */ struct task_struct* lkl_get_current_task_struct(void) { return lkl_ops->tls_get(task_key); @@ -150,6 +151,18 @@ long lkl_syscall(long no, long *params) * host task) already holds the CPU lock. */ if (lkl_shutdown) { + + /** + * Only permit the idle host task to issue system calls now, because + * it is invoking the shutdown sequence. + */ + if (current != idle_host_task) { + LKL_TRACE( + "Ignoring userspace syscall (current->comm=%s allowed=%s)\n", + current->comm, idle_host_task->comm); + return -ENOENT; + } + ret = run_syscall(no, params); task_work_run(); do_signal(NULL); @@ -290,8 +303,6 @@ long lkl_syscall(long no, long *params) return ret; } -static struct task_struct *idle_host_task; - extern _Atomic(int) lkl_exit_status; extern _Atomic(int) lkl_received_signal; extern _Atomic(bool) lkl_shutdown; diff --git a/arch/lkl/kernel/threads.c b/arch/lkl/kernel/threads.c index 18f413a1c12402..989b90bfbc372c 100644 --- a/arch/lkl/kernel/threads.c +++ b/arch/lkl/kernel/threads.c @@ -111,7 +111,11 @@ static void kill_thread(struct thread_info *ti) lkl_shutdown = true; ti->dead = true; + lkl_ops->sem_up(ti->sched_sem); + lkl_ops->thread_join(ti->tid); + ti->tid = NULL; } + lkl_ops->sem_free(ti->sched_sem); ti->sched_sem = NULL; } From 758228ddf0edaa21f7909e8ebbf41404b2b646b4 Mon Sep 17 00:00:00 2001 From: Peter Pietzuch Date: Fri, 21 Aug 2020 18:45:42 +0000 Subject: [PATCH 6/6] Change error when threads perform syscalls during shutdown --- arch/lkl/kernel/syscalls.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/lkl/kernel/syscalls.c b/arch/lkl/kernel/syscalls.c index 3ffcdb133d08e2..d357f94c930aea 100644 --- a/arch/lkl/kernel/syscalls.c +++ b/arch/lkl/kernel/syscalls.c @@ -160,7 +160,7 @@ long lkl_syscall(long no, long *params) LKL_TRACE( "Ignoring userspace syscall (current->comm=%s allowed=%s)\n", current->comm, idle_host_task->comm); - return -ENOENT; + return -ENOTSUPP; } ret = run_syscall(no, params);