lkl: move context switching implementation to host

This patch moves the context switching implementation from kernel to
host. Besides simplifying the kernel - host interface, it also makes it
easier to add support for userspace threads.

Signed-off-by: Octavian Purdila <tavi@cs.pub.ro>

arch/lkl/include/asm/thread_info.h

@@ -17,10 +17,7 @@ struct thread_info {
 	unsigned long flags;
 	int preempt_count;
 	mm_segment_t addr_limit;
-	struct lkl_sem *sched_sem;
-	bool dead;
-	lkl_thread_t tid;
-	struct task_struct *prev_sched;
+	struct lkl_thread *thread;
 	unsigned long stackend;
 };
 

arch/lkl/include/uapi/asm/host_ops.h

@@ -5,7 +5,7 @@
 struct lkl_mutex;
 struct lkl_sem;
 struct lkl_tls_key;
-typedef unsigned long lkl_thread_t;
+struct lkl_thread;
 
 /**
  * lkl_host_operations - host operations used by the Linux kernel
@@ -31,13 +31,13 @@ typedef unsigned long lkl_thread_t;
  * @mutex_lock - acquire the mutex
  * @mutex_unlock - release the mutex
  *
- * @thread_create - create a new thread and run f(arg) in its context; returns a
- * thread handle or 0 if the thread could not be created
- * @thread_detach - on POSIX systems, free up resources held by
- * pthreads. Noop on Win32.
- * @thread_exit - terminates the current thread
- * @thread_join - wait for the given thread to terminate. Returns 0
- * for success, -1 otherwise
+ * @thread_alloc - allocates the resources for a new thread; a newly
+ * created thread is not running until it is scheduled with the
+ * @thread_switch operation
+ * @thread_switch - performs a context switch operation between the
+ * currenttly running thread (prev) and another thread (next)
+ * @thread_free - terminates and frees resource associated with the
+ * given thread
  *
  * @tls_alloc - allocate a thread local storage key; returns 0 if successful; if
  * destructor is not NULL it will be called when a thread terminates with its
@@ -83,12 +83,9 @@ struct lkl_host_operations {
 	void (*mutex_lock)(struct lkl_mutex *mutex);
 	void (*mutex_unlock)(struct lkl_mutex *mutex);
 
-	lkl_thread_t (*thread_create)(void (*f)(void *), void *arg);
-	void (*thread_detach)(void);
-	void (*thread_exit)(void);
-	int (*thread_join)(lkl_thread_t tid);
-	lkl_thread_t (*thread_self)(void);
-	int (*thread_equal)(lkl_thread_t a, lkl_thread_t b);
+	struct lkl_thread *(*thread_alloc)(void (*f)(void *), void *arg);
+	void (*thread_switch)(struct lkl_thread *prev, struct lkl_thread *next);
+	void (*thread_free)(struct lkl_thread *thread);
 
 	struct lkl_tls_key *(*tls_alloc)(void (*destructor)(void *));
 	void (*tls_free)(struct lkl_tls_key *key);

arch/lkl/kernel/setup.c

@@ -37,14 +37,14 @@ void __init setup_arch(char **cl)
 
 static void __init lkl_run_kernel(void *arg)
 {
-	threads_init();
 	start_kernel();
 }
 
 int __init lkl_start_kernel(struct lkl_host_operations *ops,
 			unsigned long _mem_size,
 			const char *fmt, ...)
 {
+	struct thread_info *init_ti = &init_thread_union.thread_info;
 	va_list ap;
 	int ret;
 
@@ -71,11 +71,12 @@ int __init lkl_start_kernel(struct lkl_host_operations *ops,
 		goto out_free_init_sem;
 	}
 
-	ret = lkl_ops->thread_create(lkl_run_kernel, NULL);
-	if (!ret) {
+	init_ti->thread = lkl_ops->thread_alloc(lkl_run_kernel, NULL);
+	if (!init_ti->thread) {
 		ret = -ENOMEM;
 		goto out_free_idle_sem;
 	}
+	lkl_ops->thread_switch(NULL, init_ti->thread);
 
 	lkl_ops->sem_down(init_sem);
 
@@ -149,7 +150,7 @@ void arch_cpu_idle(void)
 
 		is_running = false;
 		lkl_ops->sem_up(halt_sem);
-		lkl_ops->thread_exit();
+		lkl_ops->thread_free(init_thread_union.thread_info.thread);
 	}
 
 	lkl_ops->sem_down(idle_sem);

arch/lkl/kernel/threads.c

@@ -5,19 +5,6 @@
 
 static volatile int threads_counter;
 
-static int init_ti(struct thread_info *ti)
-{
-	ti->sched_sem = lkl_ops->sem_alloc(0);
-	if (!ti->sched_sem)
-		return -ENOMEM;
-
-	ti->dead = false;
-	ti->prev_sched = NULL;
-	ti->tid = 0;
-
-	return 0;
-}
-
 unsigned long *alloc_thread_stack_node(struct task_struct *task, int node)
 {
 	struct thread_info *ti;
@@ -26,13 +13,7 @@ unsigned long *alloc_thread_stack_node(struct task_struct *task, int node)
 	if (!ti)
 		return NULL;
 
-	if (init_ti(ti)) {
-		kfree(ti);
-		return NULL;
-	}
 	ti->task = task;
-
-
 	return (unsigned long *)ti;
 }
 
@@ -52,11 +33,8 @@ void setup_thread_stack(struct task_struct *p, struct task_struct *org)
 
 static void kill_thread(struct thread_info *ti)
 {
-	ti->dead = true;
-	lkl_ops->sem_up(ti->sched_sem);
-	lkl_ops->thread_join(ti->tid);
-	lkl_ops->sem_free(ti->sched_sem);
-
+	lkl_ops->thread_free(ti->thread);
+	__sync_fetch_and_sub(&threads_counter, 1);
 }
 
 void free_thread_stack(struct task_struct *tsk)
@@ -87,18 +65,9 @@ struct task_struct *__switch_to(struct task_struct *prev,
 	 */
 	static struct task_struct *abs_prev = &init_task;
 
-	_current_thread_info = task_thread_info(next);
-	_next->prev_sched = prev;
 	abs_prev = prev;
-
-	lkl_ops->sem_up(_next->sched_sem);
-	lkl_ops->sem_down(_prev->sched_sem);
-
-	if (_prev->dead) {
-		__sync_fetch_and_sub(&threads_counter, 1);
-		lkl_ops->thread_exit();
-	}
-
+	_current_thread_info = task_thread_info(next);
+	lkl_ops->thread_switch(_prev->thread, _next->thread);
 	return abs_prev;
 }
 
@@ -111,15 +80,10 @@ struct thread_bootstrap_arg {
 static void thread_bootstrap(void *_tba)
 {
 	struct thread_bootstrap_arg *tba = (struct thread_bootstrap_arg *)_tba;
-	struct thread_info *ti = tba->ti;
 	int (*f)(void *) = tba->f;
 	void *arg = tba->arg;
 
-	lkl_ops->sem_down(ti->sched_sem);
 	kfree(tba);
-	if (ti->prev_sched)
-		schedule_tail(ti->prev_sched);
-
 	f(arg);
 	do_exit(0);
 }
@@ -138,8 +102,8 @@ int copy_thread(unsigned long clone_flags, unsigned long esp,
 	tba->arg = (void *)unused;
 	tba->ti = ti;
 
-	ti->tid = lkl_ops->thread_create(thread_bootstrap, tba);
-	if (!ti->tid) {
+	ti->thread = lkl_ops->thread_alloc(thread_bootstrap, tba);
+	if (!ti->thread) {
 		kfree(tba);
 		return -ENOMEM;
 	}
@@ -153,22 +117,6 @@ void show_stack(struct task_struct *task, unsigned long *esp)
 {
 }
 
-/**
- * This is called before the kernel initializes, so no kernel calls (including
- * printk) can't be made yet.
- */
-void threads_init(void)
-{
-	int ret;
-	struct thread_info *ti = &init_thread_union.thread_info;
-
-	ret = init_ti(ti);
-	if (ret < 0)
-		lkl_printf("lkl: failed to allocate init schedule semaphore\n");
-
-	ti->tid = lkl_ops->thread_self();
-}
-
 void threads_cleanup(void)
 {
 	struct task_struct *p;
@@ -187,6 +135,4 @@ void threads_cleanup(void)
 
 	while (threads_counter)
 		;
-
-	lkl_ops->sem_free(init_thread_union.thread_info.sched_sem);
 }

tools/lkl/include/lkl_host.h

@@ -151,6 +151,15 @@ struct lkl_dev_net_ops {
 	void (*free)(struct lkl_netdev *nd);
 };
 
+/* native threads */
+typedef unsigned long lkl_thread_t;
+lkl_thread_t thread_create(void (*fn)(void *), void *arg);
+int thread_join(lkl_thread_t tid);
+void thread_detach(void);
+void thread_exit(void);
+lkl_thread_t thread_self(void);
+int thread_equal(lkl_thread_t a, lkl_thread_t b);
+
 #ifdef __cplusplus
 }
 #endif

tools/lkl/lib/Build

@@ -19,3 +19,5 @@ lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_raw.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_macvtap.o
 lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET_DPDK) += virtio_net_dpdk.o
 lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET_VDE) += virtio_net_vde.o
+lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += threads_native.o
+lkl-$(CONFIG_AUTO_LKL_NT_HOST) += threads_native.o

tools/lkl/lib/dbg_handler.c

@@ -1,11 +1,12 @@
 #include <stdio.h>
 #include <lkl_host.h>
+#include "threads.h"
 
 extern void dbg_entrance();
 static int dbg_running = 0;
 
 static void dbg_thread(void* arg) {
-	lkl_host_ops.thread_detach();
+	thread_detach();
 	printf("======Enter Debug======\n");
 	dbg_entrance();
 	printf("======Exit Debug======\n");
@@ -19,7 +20,7 @@ void dbg_handler(int signum) {
 		return;
 	}
 	dbg_running = 1;
-	lkl_host_ops.thread_create(&dbg_thread, NULL);
+	thread_create(&dbg_thread, NULL);
 }
 
 #ifndef __MINGW32__

tools/lkl/lib/nt-host.c

@@ -246,12 +246,9 @@ static void *mem_alloc(unsigned long size)
 
 struct lkl_host_operations lkl_host_ops = {
 	.panic = panic,
-	.thread_create = thread_create,
-	.thread_detach = thread_detach,
-	.thread_exit = thread_exit,
-	.thread_join = thread_join,
-	.thread_self = thread_self,
-	.thread_equal = thread_equal,
+	.thread_alloc = thread_alloc,
+	.thread_switch = thread_switch,
+	.thread_free = thread_free,
 	.sem_alloc = sem_alloc,
 	.sem_free = sem_free,
 	.sem_up = sem_up,

tools/lkl/lib/posix-host.c

@@ -16,6 +16,7 @@
 #include <poll.h>
 #include <lkl_host.h>
 #include "iomem.h"
+#include "threads.h"
 
 /* Let's see if the host has semaphore.h */
 #include <unistd.h>
@@ -178,7 +179,8 @@ static void mutex_free(struct lkl_mutex *_mutex)
 	free(_mutex);
 }
 
-static lkl_thread_t thread_create(void (*fn)(void *), void *arg)
+
+lkl_thread_t thread_create(void (*fn)(void *), void *arg)
 {
 	pthread_t thread;
 	if (WARN_PTHREAD(pthread_create(&thread, NULL, (void* (*)(void *))fn, arg)))
@@ -187,30 +189,30 @@ static lkl_thread_t thread_create(void (*fn)(void *), void *arg)
 		return (lkl_thread_t) thread;
 }
 
-static void thread_detach(void)
+void thread_detach(void)
 {
 	WARN_PTHREAD(pthread_detach(pthread_self()));
 }
 
-static void thread_exit(void)
+void thread_exit(void)
 {
 	pthread_exit(NULL);
 }
 
-static int thread_join(lkl_thread_t tid)
+int thread_join(lkl_thread_t tid)
 {
 	if (WARN_PTHREAD(pthread_join((pthread_t)tid, NULL)))
 		return -1;
 	else
 		return 0;
 }
 
-static lkl_thread_t thread_self(void)
+lkl_thread_t thread_self(void)
 {
 	return (lkl_thread_t)pthread_self();
 }
 
-static int thread_equal(lkl_thread_t a, lkl_thread_t b)
+int thread_equal(lkl_thread_t a, lkl_thread_t b)
 {
 	return pthread_equal(a, b);
 }
@@ -308,12 +310,9 @@ static long _gettid(void)
 
 struct lkl_host_operations lkl_host_ops = {
 	.panic = panic,
-	.thread_create = thread_create,
-	.thread_detach = thread_detach,
-	.thread_exit = thread_exit,
-	.thread_join = thread_join,
-	.thread_self = thread_self,
-	.thread_equal = thread_equal,
+	.thread_alloc = thread_alloc,
+	.thread_switch = thread_switch,
+	.thread_free = thread_free,
 	.sem_alloc = sem_alloc,
 	.sem_free = sem_free,
 	.sem_up = sem_up,

tools/lkl/lib/threads.h

@@ -0,0 +1,10 @@
+#ifndef _LKL_LIB_THREADS_H
+#define _LKL_LIB_THREADS_H
+
+struct lkl_thread;
+
+struct lkl_thread *thread_alloc(void (*fn)(void *), void *arg);
+void thread_switch(struct lkl_thread *prev, struct lkl_thread *next);
+void thread_free(struct lkl_thread *thread);
+
+#endif /* _LKL_LIB_THREADS_H */