lkl: add rump hypercall implementation for frankenlibc build

This commit adds an implementation of new host_ops by using rump
hypercall (wrapped as an LKL host_ops) provided by rumpkernel project.
The rump hypercall implementation is specifically integrated with the
one provided by frankenlibc project, which is an alternative way to use
rump hypercalls.

There are other rump hypercall implementations such as rumprun, NetBSD
kernel tree, but this commit only focuses on the frankenlibc
integration.

The new features covered by this commits ranges in various ways, but
here is some highlights.

- New thread primitives
  frankenlibc implements user-space thread implementation based on
  makecontext(3)&co with non-preemptive (cooperative) scheduler
  implementation.
- LKL specific standard library bind
  frankenlibc offers cross-compiler build chains to build an LKL
  application bound to LKL-versioned standard library (i.e., libc). Our
  musl libc implementation is provided by different repository.
  So, any application can be built with this toolchain to use LKL
  instead of host libc.
- New host environments support
  LKLed application now has potentials to run on the platform which rump
  hypercall implementations support.  frankenlibc supports Liunx,
  FreeBSD, NetBSD, and qemu-arm.  Others supposrts run an LKL
  applications on hypervisors (kvm, xen, etc)

The frankenlibc code is located at the following repository, but it will
be upstreamed to the original one (justincormack/frankenlibc).

https://github.com/libos-nuse/frankenlibc

Signed-off-by: Hajime Tazaki <thehajime@gmail.com>

arch/lkl/Kconfig

@@ -39,6 +39,9 @@ config 64BIT
 config BIG_ENDIAN
        def_bool n
 
+config TTY
+       def_bool n
+
 config GENERIC_CSUM
        def_bool y
 

arch/lkl/Makefile

@@ -21,17 +21,47 @@ NPROC=$(shell nproc)
 endif
 
 LDFLAGS_vmlinux += -r
+
+ifeq ($(buildrump),yes)
+# objcopy trick is taken from rumpkernel
+GCCSYMBOLS=__umoddi3|__udivdi3|__aeabi_idiv|__aeabi_idivmod|__aeabi_llsl
+GCCSYMBOLS:=$(GCCSYMBOLS)|__aeabi_llsr|__aeabi_uidiv|__aeabi_uidivmod|__aeabi_uldivmod
+GCCATOMIC=__sync_synchronize|__sync_fetch_and_sub_4|__sync_fetch_and_add_4
+GCCATOMIC:=$(GCCATOMIC)|__sync_fetch_and_and_4|__sync_fetch_and_or_4
+VMLINUX_SYMS=__start___ex_table|__stop___ex_table|boot_cmdline
+EXP_SYMRENAME=rump|RUMP|bmk_|lib_|nuse_|lkl_|__tls|__initcall_start
+EXP_SYMRENAME:=$(EXP_SYMRENAME)|__initcall_end|__gcov|_end|_GLOBAL_OFFSET_TABLE|__assert13
+EXP_SYMRENAME:=$(EXP_SYMRENAME)|${GCCSYMBOLS}|${GCCATOMIC}'${_SYMQUIRK}'
+EXP_SYMRENAME:=$(EXP_SYMRENAME)${RUMP_SYM_NORENAME:D|${RUMP_SYM_NORENAME}}|${VMLINUX_SYMS}
+
+define make_ns_symbols
+	${Q}echo "  OBJCPY	" $1 $2;	\
+	${NM} -go $1 | awk ' 	\
+	$$NF!~/^'${_PQ}'(${EXP_SYMRENAME})/ \
+	{s=$$NF;sub(/^'${_PQ}'/, "&rumpns_", s); print $$NF, s}'\
+	| sort | uniq  > $2.renametab; \
+	$(OBJCOPY) -R .eh_frame -L __start___ex_table \
+	-L __stop___ex_table --preserve-dates \
+	--redefine-syms $2.renametab $1 $2; \
+	rm -f $2.renametab
+endef
+else
 LKL_ENTRY_POINTS := lkl_start_kernel lkl_sys_halt lkl_syscall lkl_trigger_irq \
 	lkl_get_free_irq lkl_put_irq lkl_create_syscall_thread \
 	lkl_stop_syscall_thread lkl_is_running
+define make_ns_symbols
+	$(OBJCOPY) -R .eh_frame -R .syscall_defs $(foreach sym,$(LKL_ENTRY_POINTS),\
+	-G$(prefix)$(sym)) vmlinux lkl.o
+endef
+endif
 
 core-y += arch/lkl/kernel/
 core-y += arch/lkl/mm/
 
 all: lkl.o
 
 lkl.o: vmlinux
-	$(OBJCOPY) -R .eh_frame -R .syscall_defs $(foreach sym,$(LKL_ENTRY_POINTS),-G$(prefix)$(sym)) vmlinux lkl.o
+	$(call make_ns_symbols,vmlinux,lkl.o)
 
 arch/lkl/include/generated/uapi/asm/syscall_defs.h: vmlinux
 	$(OBJCOPY) -j .syscall_defs -O binary --set-section-flags .syscall_defs=alloc $< $@

arch/lkl/defconfig

@@ -31,6 +31,7 @@ CONFIG_EXT4_FS_POSIX_ACL=y
 CONFIG_EXT4_FS_SECURITY=y
 CONFIG_XFS_FS=y
 CONFIG_XFS_POSIX_ACL=y
+CONFIG_ISO9660_FS=y
 CONFIG_BTRFS_FS=y
 CONFIG_BTRFS_FS_POSIX_ACL=y
 # CONFIG_FILE_LOCKING is not set

arch/lkl/include/asm/Kbuild

@@ -73,5 +73,7 @@ generic-y += topology.h
 generic-y += trace_clock.h
 generic-y += uaccess.h
 generic-y += unaligned.h
+generic-y += vga.h
 generic-y += word-at-a-time.h
 generic-y += xor.h
+header-y += syscalls.h

arch/lkl/kernel/console.c

@@ -1,10 +1,28 @@
 #include <linux/init.h>
 #include <linux/kernel.h>
 #include <linux/console.h>
-#include <asm/host_ops.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/syscalls.h>
+#include <linux/device.h>
+#include <linux/cdev.h>
+#include <linux/major.h>
 
 static void console_write(struct console *con, const char *str, unsigned len)
 {
+	static char buf[256];
+	static int verbose = 0;
+
+	/* when console isn't NULL (not called from file_write() */
+	if (con && !verbose && lkl_ops->getparam &&
+		lkl_ops->getparam("RUMP_VERBOSE", buf, sizeof(buf)) == 0) {
+		if (*buf != 0)
+			verbose = 1;
+	}
+
+	if (con && !verbose)
+		return;
+
 	if (lkl_ops->print)
 		lkl_ops->print(str, len);
 }
@@ -39,3 +57,68 @@ int __init lkl_console_init(void)
 }
 core_initcall(lkl_console_init);
 
+static ssize_t file_write(struct file *fp, const char __user *s,
+			  size_t n, loff_t *off)
+{
+	console_write(NULL, s, n);
+	return n;
+}
+
+static ssize_t file_read(struct file *file, char __user *buf, size_t size,
+			 loff_t *ppos)
+{
+	int err = 0;
+#ifdef TO_BE_IMPLEMENTED
+	/* need to use iovread in host_ops (not directly from rump hypercall) */
+	struct rumpuser_iovec iov;
+	ssize_t ret;
+
+	iov.iov_base = buf;
+	iov.iov_len = size;
+
+	err = rumpuser_iovread(0, &iov, 1, 0, &ret);
+	if (err == 0)
+		return ret;
+
+#endif
+	return -err;
+}
+
+static const struct file_operations lkl_stdio_fops = {
+	.owner		= THIS_MODULE,
+	.write =	file_write,
+	.read =		file_read,
+};
+
+static int __init lkl_stdio_init(void)
+{
+	int err;
+
+	/* prepare /dev/console */
+	err = register_chrdev(TTYAUX_MAJOR, "console", &lkl_stdio_fops);
+	if (err < 0) {
+		pr_err("can't register lkl stdio console.\n");
+		return err;
+	}
+
+	return 0;
+}
+/* should be _before_ default_rootfs creation (noinitramfs.c) */
+fs_initcall(lkl_stdio_init);
+
+static int __init lkl_memdev_init(void)
+{
+	int err;
+
+	/* prepare /dev/null */
+	err = sys_mknod((const char __user __force *) "/dev/null",
+			S_IFCHR | S_IRUSR | S_IWUSR,
+			new_encode_dev(MKDEV(MEM_MAJOR, 3)));
+	if (err < 0) {
+		pr_err("can't register /dev/null.\n");
+		return err;
+	}
+
+	return 0;
+}
+device_initcall(lkl_memdev_init);

tools/lkl/Makefile

@@ -10,6 +10,10 @@ else
   Q = @
 endif
 
+OUTPUT_FORMAT=$(shell $(LD) -r -print-output-format)
+RUMP_PREFIX?=$(abspath ../../../rumpobj/)
+RUMP_LIB=$(RUMP_PREFIX)/rump/lib
+RUMP_INCLUDE?=$(RUMP_PREFIX)/
 # default target
 all:
 
@@ -37,8 +41,8 @@ export srctree
 # Target build configuration
 
 export CFLAGS += -I$(OUTPUT)/include -Iinclude -Wall -g -O2 -Wextra \
-	 -Wno-unused-parameter \
-	 -Wno-missing-field-initializers -fno-strict-aliasing
+	 -Wno-missing-field-initializers -fno-strict-aliasing \
+	 -Wno-unused-parameter -U_FORTIFY_SOURCE -fno-stack-protector
 
 OUTPUT_FORMAT = $(shell $(LD) -r -print-output-format)
 
@@ -49,6 +53,8 @@ ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64 elf32-i386 elf64-x86-64-freebsd e
     LDLIBS   += -lrt -lpthread
   endif
   export CONFIG_AUTO_LKL_POSIX_HOST=y
+  export CONFIG_AUTO_LKL_VIRTIO=y
+  export CONFIG_AUTO_LKL_VIRTIO_NET=y
   CFLAGS += -DCONFIG_AUTO_LKL_POSIX_HOST
 
   # Intel DPDK configuration
@@ -73,6 +79,8 @@ else ifneq (,$(filter $(OUTPUT_FORMAT),pe-i386))
   EXESUF   := .exe
   SOSUF    := .dll
   export CONFIG_AUTO_LKL_NT_HOST=y
+  export CONFIG_AUTO_LKL_VIRTIO=y
+  export CONFIG_AUTO_LKL_VIRTIO_NET=n
   CFLAGS += -DCONFIG_AUTO_LKL_NT_HOST
 else
   $(error Unrecognized platform: $(OUTPUT_FORMAT))
@@ -113,6 +121,19 @@ ifneq (,$(filter $(OUTPUT_FORMAT),elf64-x86-64-freebsd))
   $(OUTPUT)cptofs$(EXESUF): LDLIBS += -largp
 endif
 
+# if there is no rumpuser.h, then skip build
+ifeq (,$(wildcard $(RUMP_INCLUDE)/rump/rumpuser.h))
+else
+  buildrump=yes
+  KOPT+="buildrump="$(buildrump)
+  export CONFIG_AUTO_LKL_RUMP_HOST=y
+  export CONFIG_AUTO_LKL_POSIX_HOST=n
+  export CONFIG_AUTO_LKL_VIRTIO=y
+  export CONFIG_AUTO_LKL_VIRTIO_NET=y
+  CFLAGS += -I$(RUMP_INCLUDE) -DRUMPUSER -DLIBRUMPUSER -D_KERNEL
+  CFLAGS += -UCONFIG_AUTO_LKL_POSIX_HOST
+
+endif
 
 TEST_TARGETS        := test valgrind gdb
 
@@ -146,13 +167,18 @@ $(OUTPUT)cpfromfs$(EXESUF): cptofs$(EXESUF)
 # because of libdl, liblkl-hijack will not compile on windows
 # arm-android neither for the moment
 ifneq (,$(filter $(OUTPUT_FORMAT),pe-i386))
-  all: $(filter-out liblkl-hijack$(SOSUF), $(ALL_LIBRARIES))
-else ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__))
-  all: $(filter-out liblkl-hijack$(SOSUF), $(ALL_LIBRARIES))
-else ifneq (,$(filter $(OUTPUT_FORMAT),elf32-littlearm))
-  all: $(filter-out liblkl-hijack$(SOSUF), $(ALL_LIBRARIES)) $(ALL_PROGRAMS)
+  all: $(ALL_LIBRARIES)
+  install: headers_install libraries_install programs_install
+else ifneq (,$(wildcard $(RUMP_INCLUDE)))
+  all: $(ALL_LIBRARIES)
+  install: headers_install libraries_install
+else ifneq (,$(filter $(OUTPUT_DEF),__ANDROID__ __ARMEL__))
+  all: $(ALL_LIBRARIES)
+  install: headers_install libraries_install programs_install
 else
+  ALL_LIBRARIES := $(ALL_LIBRARIES) $(OUTPUT)liblkl-hijack$(SOSUF)
   all: $(ALL_PROGRAMS) $(ALL_LIBRARIES)
+  install: headers_install libraries_install programs_install
 endif
 
 clean:
@@ -184,9 +210,7 @@ programs_install: $(ALL_PROGRAMS)
 	    install -d $(DESTDIR)$(PREFIX)/bin ; \
 	    install -m 755 $(ALL_PROGRAMS) $(DESTDIR)$(PREFIX)/bin
 
-install: headers_install libraries_install programs_install
-
-
 FORCE: ;
 .PHONY: all clean $(TEST_TARGETS) FORCE
 .PHONY: headers_install libraries_install programs_install install
+.NOTPARALLEL : lib/lkl.o

tools/lkl/lib/Build

@@ -7,15 +7,22 @@ lkl-y += iomem.o
 lkl-y += net.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += posix-host.o
 lkl-$(CONFIG_AUTO_LKL_NT_HOST) += nt-host.o
+lkl-$(CONFIG_AUTO_LKL_RUMP_HOST) += rump.o
+lkl-$(CONFIG_AUTO_LKL_RUMP_HOST) += rump-host.o
 lkl-y += utils.o
-lkl-y += virtio_blk.o
-lkl-y += virtio.o
-lkl-y += dbg.o
-lkl-y += dbg_handler.o
-lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net.o
+lkl-$(CONFIG_AUTO_LKL_VIRTIO) += virtio_blk.o
+lkl-$(CONFIG_AUTO_LKL_VIRTIO) += virtio.o
+lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += dbg.o
+lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += dbg_handler.o
+lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET) += virtio_net.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_fd.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_tap.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_raw.o
 lkl-$(CONFIG_AUTO_LKL_POSIX_HOST) += virtio_net_macvtap.o
 lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET_DPDK) += virtio_net_dpdk.o
 lkl-$(CONFIG_AUTO_LKL_VIRTIO_NET_VDE) += virtio_net_vde.o
+
+# for frankenlibc
+CFLAGS_rump-host.o = -I$(RUMP_PREFIX)/../platform/include \
+		   -I$(RUMP_PREFIX)/../franken/include \
+		   -DCONFIG_AUTO_LKL_POSIX_HOST

tools/lkl/lib/fs.c

@@ -1,10 +1,12 @@
 #include <stdio.h>
 #include <string.h>
-#include <stdlib.h>
 #include <lkl_host.h>
-
 #include "virtio.h"
 
+#ifdef RUMPUSER
+#include "rump.h"
+#endif
+
 #define MAX_FSTYPE_LEN 50
 int lkl_mount_fs(char *fstype)
 {

tools/lkl/lib/net.c

@@ -1,8 +1,12 @@
-#include <string.h>
 #include <stdio.h>
+#include <string.h>
 #include "endian.h"
 #include <lkl_host.h>
 
+#ifdef RUMPUSER
+#include "rump.h"
+#endif
+
 static inline void set_sockaddr(struct lkl_sockaddr_in *sin, unsigned int addr,
 				unsigned short port)
 {