Add a brute-force temporal-safety checking allocator. (#719)

Optionally enable it for lkl_sem and lthread structures.

This allocates each object on a separate page and uses mprotect on the
host to remove all permissions to access the object after it's
deallocated.  In production, this would quickly exhaust enclave memory,
but it can be useful when running tests.

src/include/enclave/enclave_mem.h

@@ -6,6 +6,22 @@
 #include <sys/types.h>
 #include <time.h>
 
+#include "enclave/enclave_util.h"
+#include "enclave/sgxlkl_t.h"
+
+#ifndef PROT_NONE
+#    define PROT_NONE 0x0
+#endif
+#ifndef PROT_READ
+#    define PROT_READ 0x1
+#endif
+#ifndef PROT_WRITE
+#    define PROT_WRITE 0x2
+#endif
+#ifndef PROT_EXEC
+#    define PROT_EXEC 0x4
+#endif
+
 void enclave_mman_init(const void* base, size_t num_pages, int _mmap_files);
 
 void* enclave_mmap(
@@ -56,4 +72,33 @@ long syscall_SYS_mmap(
 
 int enclave_futex_wake(int* uaddr, int val);
 
+/**
+ * Paranoid allocator.  Allocates on a separate page.
+ */
+static inline void* paranoid_alloc(size_t sz)
+{
+    // round up to page size:
+    sz += 4096;
+    sz %= 4096;
+    void* ret =
+        enclave_mmap(NULL, sz, /*fixed*/ 0, PROT_READ | PROT_WRITE, /*zero*/ 1);
+    SGXLKL_ASSERT((intptr_t)ret > 0);
+
+    return ret;
+}
+
+/**
+ * Paranoid deallocate, marks the page as no-access and never reuses it.  This
+ * should not be used in production because it will exhaust enclave memory
+ * quite quickly, but can help tracking use-after-free bugs.
+ */
+static inline void paranoid_dealloc(void* p, size_t sz)
+{
+    // round up to page size:
+    sz += 4096;
+    sz %= 4096;
+    int ret;
+    sgxlkl_host_syscall_mprotect(&ret, p, sz, PROT_NONE);
+}
+
 #endif /* ENCLAVE_MEM_H */

src/lkl/posix-host.c

@@ -178,9 +178,13 @@ static struct lkl_sem* sem_alloc(int count)
 {
     struct lkl_sem* sem;
 
+#ifdef LKL_SEM_UAF_CHECKS
+    sem = paranoid_alloc(sizeof(struct lkl_sem));
+#else
     sem = oe_calloc(1, sizeof(*sem));
     if (!sem)
         return NULL;
+#endif
 
     sem->count = count;
 
@@ -189,7 +193,13 @@ static struct lkl_sem* sem_alloc(int count)
 
 static void sem_free(struct lkl_sem* sem)
 {
+    SGXLKL_VERBOSE("enter: %p\n", sem);
+#if LKL_SEM_UAF_CHECKS
+    paranoid_dealloc(sem, sizeof(struct lkl_sem));
+#else
     oe_free(sem);
+#endif
+    SGXLKL_VERBOSE("exit\n");
 }
 
 static void sem_up(struct lkl_sem* sem)

src/main-oe/serialize_enclave_config.c

@@ -4,7 +4,6 @@
 #include <string.h>
 
 #include <json.h>
-#include "enclave/enclave_mem.h"
 #include "enclave/wireguard.h"
 #include "host/sgxlkl_util.h"
 #include "shared/env.h"
@@ -525,4 +524,4 @@ void serialize_enclave_config(
     VERB("Enclave config: %s\n", *buffer);
 
     free_json(root);
-}
+}

src/main-oe/sgxlkl_evt_chn_cfg.c

@@ -1,4 +1,3 @@
-#include <enclave/enclave_mem.h>
 #include <errno.h>
 #include <host/sgxlkl_util.h>
 #include <host/vio_host_event_channel.h>

src/main-oe/sgxlkl_run_oe.c

@@ -28,7 +28,6 @@
 #include <arpa/inet.h>
 #include <netinet/ip.h>
 
-#include "enclave/enclave_mem.h"
 #include "host/host_state.h"
 #include "host/serialize_enclave_config.h"
 #include "host/sgxlkl_host_config.h"

src/sched/lthread.c

@@ -154,6 +154,24 @@ __asm__("    .text                                  \n"
         "       ret                                              \n");
 #endif
 
+static inline struct lthread* lthread_alloc()
+{
+#ifdef LTHREAD_UAF_CHECKS
+    return paranoid_alloc(sizeof(struct lthread));
+#else
+    return oe_calloc(sizeof(struct lthread), 1);
+#endif
+}
+
+static inline void lthread_dealloc(struct lthread* lt)
+{
+#ifdef LTHREAD_UAF_CHECKS
+    return paranoid_dealloc(lt, sizeof(struct lthread));
+#else
+    return oe_free(lt);
+#endif
+}
+
 static void _exec(void* lt_)
 {
 #if defined(__llvm__) && defined(__x86_64__)
@@ -408,8 +426,7 @@ void _lthread_free(struct lthread* lt)
     }
 #endif /* DEBUG */
 
-    oe_free(lt);
-    lt = 0;
+    lthread_dealloc(lt);
 }
 
 void set_tls_tp(struct lthread* lt)
@@ -601,7 +618,7 @@ int lthread_create_primitive(
         libc.threaded = 1;
     }
 
-    if ((lt = oe_calloc(1, sizeof(struct lthread))) == NULL)
+    if ((lt = lthread_alloc(1, sizeof(struct lthread))) == NULL)
     {
         return -1;
     }
@@ -619,12 +636,12 @@ int lthread_create_primitive(
                  PROT_READ | PROT_WRITE,
                  1 /* zero_pages */)) < 0)
         {
-            oe_free(lt);
+            lthread_dealloc(lt);
             return -1;
         }
         if (__init_utp(__copy_utls(lt, lt->itls, lt->itlssz), 0))
         {
-            oe_free(lt);
+            lthread_dealloc(lt);
             return -1;
         }
     }
@@ -701,7 +718,7 @@ int lthread_create(
 
     stack_size =
         attrp && attrp->stack_size ? attrp->stack_size : sched->stack_size;
-    if ((lt = oe_calloc(1, sizeof(struct lthread))) == NULL)
+    if ((lt = lthread_alloc(1, sizeof(struct lthread))) == NULL)
     {
         return -1;
     }
@@ -713,7 +730,7 @@ int lthread_create(
                                    PROT_READ | PROT_WRITE,
                                    1 /* zero_pages */)) < 0))
     {
-        oe_free(lt);
+        lthread_dealloc(lt);
         return -1;
     }
     lt->attr.stack_size = stack_size;
@@ -729,13 +746,13 @@ int lthread_create(
                  PROT_READ | PROT_WRITE,
                  1 /* zero_pages */)) < 0)
         {
-            oe_free(lt);
+            lthread_dealloc(lt);
             return -1;
         }
         if (__init_utp(__copy_utls(lt, lt->itls, lt->itlssz), 0))
         {
             enclave_munmap(lt->attr.stack, stack_size);
-            oe_free(lt);
+            lthread_dealloc(lt);
             return -1;
         }
     }