Threading without using patched LLVM

Make.inc

@@ -275,12 +275,6 @@ else
 DISABLE_ASSERTIONS := -DNDEBUG
 endif
 
-# Threads
-ifeq ($(JULIA_THREADS), 1)
-LLVM_VER := svn
-LLVM_GIT_URL_LLVM := https://github.com/JuliaLang/llvm.git -b jn/tls37
-endif
-
 # Compiler specific stuff
 
 ifeq ($(USEMSVC), 1)

src/builtins.c

@@ -955,8 +955,6 @@ extern int jl_in_inference;
 extern int jl_boot_file_loaded;
 int jl_eval_with_compiler_p(jl_expr_t *ast, jl_expr_t *expr, int compileloops, jl_module_t *m);
 
-JL_DEFINE_MUTEX_EXT(codegen)
-
 static int jl_eval_inner_with_compiler(jl_expr_t *e, jl_module_t *m)
 {
     int i;

src/cgutils.cpp

@@ -2124,12 +2124,29 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg
     }
 }
 
+static Value *emit_ptls_states()
+{
+#ifndef JULIA_ENABLE_THREADING
+    return prepare_global(jltls_states_var);
+#else
+    return builder.CreateCall(prepare_call(jltls_states_func));
+#endif
+}
+
 static Value *emit_pgcstack()
 {
-    return prepare_global(jlpgcstack_var);
+    Value * addr = emit_nthptr_addr(
+        emit_ptls_states(),
+        (ssize_t)(offsetof(jl_tls_states_t, pgcstack) / sizeof(void*)));
+    return builder.CreateBitCast(addr, PointerType::get(T_ppjlvalue, 0),
+                                 "jl_pgcstack");
 }
 
 static Value *emit_exc_in_transit()
 {
-    return prepare_global(jlexc_var);
+    Value * addr = emit_nthptr_addr(
+        emit_ptls_states(),
+        (ssize_t)(offsetof(jl_tls_states_t,
+                           exception_in_transit) / sizeof(void*)));
+    return builder.CreateBitCast(addr, T_ppjlvalue, "jl_exception_in_transit");
 }

src/codegen.cpp

@@ -447,8 +447,6 @@ static GlobalVariable *jlemptytuple_var;
 #ifdef JL_NEED_FLOATTEMP_VAR
 static GlobalVariable *jlfloattemp_var;
 #endif
-static GlobalVariable *jlpgcstack_var;
-static GlobalVariable *jlexc_var;
 static GlobalVariable *jldiverr_var;
 static GlobalVariable *jlundeferr_var;
 static GlobalVariable *jldomerr_var;
@@ -467,6 +465,12 @@ extern JITMemoryManager* createJITMemoryManagerWin();
 extern RTDyldMemoryManager* createRTDyldMemoryManagerOSX();
 #endif
 
+#ifndef JULIA_ENABLE_THREADING
+static GlobalVariable *jltls_states_var;
+#else
+static Function *jltls_states_func;
+#endif
+
 // important functions
 static Function *jlnew_func;
 static Function *jlthrow_func;
@@ -917,8 +921,6 @@ static void maybe_alloc_arrayvar(jl_sym_t *s, jl_codectx_t *ctx)
     }
 }
 
-JL_DEFINE_MUTEX_EXT(codegen)
-
 // Snooping on which functions are being compiled, and how long it takes
 JL_STREAM *dump_compiles_stream = NULL;
 uint64_t last_time = 0;
@@ -5574,23 +5576,6 @@ static void init_julia_llvm_env(Module *m)
                            "jl_array_t");
     jl_parray_llvmt = PointerType::get(jl_array_llvmt,0);
 
-#ifdef JULIA_ENABLE_THREADING
-#define JL_THREAD_MODEL ,GlobalValue::GeneralDynamicTLSModel
-#else
-#define JL_THREAD_MODEL
-#endif
-    jlpgcstack_var =
-        new GlobalVariable(*m, T_ppjlvalue,
-                           false, GlobalVariable::ExternalLinkage,
-                           NULL, "jl_pgcstack", NULL JL_THREAD_MODEL);
-    add_named_global(jlpgcstack_var, jl_dlsym(jl_dl_handle, "jl_pgcstack"));
-
-    jlexc_var =
-        new GlobalVariable(*m, T_pjlvalue,
-                           false, GlobalVariable::ExternalLinkage,
-                           NULL, "jl_exception_in_transit", NULL JL_THREAD_MODEL);
-    add_named_global(jlexc_var, jl_dlsym(jl_dl_handle, "jl_exception_in_transit"));
-
     global_to_llvm("__stack_chk_guard", (void*)&__stack_chk_guard, m);
     Function *jl__stack_chk_fail =
         Function::Create(FunctionType::get(T_void, false),
@@ -5640,6 +5625,27 @@ static void init_julia_llvm_env(Module *m)
                                      "jl_float_temp"));
 #endif
 
+#ifndef JULIA_ENABLE_THREADING
+    size_t tls_states_size = LLT_ALIGN(sizeof(jl_tls_states_t),
+                                       sizeof(void*)) / sizeof(void*);
+    jltls_states_var =
+        new GlobalVariable(*m, ArrayType::get(T_pint8, tls_states_size),
+                           false, GlobalVariable::ExternalLinkage,
+                           NULL, "jl_tls_states");
+    add_named_global(jltls_states_var, (void*)&jl_tls_states);
+#else
+    jltls_states_func = Function::Create(FunctionType::get(T_ppint8, false),
+                                         Function::ExternalLinkage,
+                                         "jl_get_ptls_states", m);
+    jltls_states_func->setAttributes(
+        jltls_states_func->getAttributes()
+        .addAttribute(jltls_states_func->getContext(),
+                      AttributeSet::FunctionIndex, Attribute::ReadNone)
+        .addAttribute(jltls_states_func->getContext(),
+                      AttributeSet::FunctionIndex, Attribute::NoUnwind));
+    add_named_global(jltls_states_func, (void*)jl_get_ptls_states_getter());
+#endif
+
     std::vector<Type*> args1(0);
     args1.push_back(T_pint8);
     jlerror_func =

src/gc.c

@@ -251,7 +251,7 @@ typedef struct {
 // In the single-threaded version, they are essentially noops, but nonetheless
 // serve to check that the thread context macros are being used.
 #ifdef JULIA_ENABLE_THREADING
-static JL_THREAD jl_thread_heap_t *jl_thread_heap;
+#define jl_thread_heap (jl_get_ptls_states()->heap)
 #define FOR_EACH_HEAP()                                                 \
     for (jl_each_heap_index_t __current_heap_idx = {jl_n_threads, NULL}; \
          --current_heap_index >= 0 &&                                   \

src/gf.c

@@ -420,7 +420,6 @@ jl_function_t *jl_method_cache_insert(jl_methtable_t *mt, jl_tupletype_t *type,
   can be equal to "li" if not applicable.
 */
 int jl_in_inference = 0;
-JL_DEFINE_MUTEX_EXT(codegen)
 void jl_type_infer(jl_lambda_info_t *li, jl_tupletype_t *argtypes, jl_lambda_info_t *def)
 {
     JL_LOCK(codegen);

src/init.c

@@ -463,6 +463,10 @@ static void jl_resolve_sysimg_location(JL_IMAGE_SEARCH rel)
 
 void _julia_init(JL_IMAGE_SEARCH rel)
 {
+#ifdef JULIA_ENABLE_THREADING
+    // Make sure we finalize the tls callback before starting any threads.
+    jl_get_ptls_states_getter();
+#endif
     libsupport_init();
     jl_io_loop = uv_default_loop(); // this loop will internal events (spawning process etc.),
                                     // best to call this first, since it also initializes libuv

src/julia.h

@@ -57,30 +57,19 @@ extern "C" {
 
 // threading ------------------------------------------------------------------
 
-// WARNING: Threading support is incomplete and experimental (and only works with llvm-svn)
-// Nonetheless, we define JL_THREAD and use it to give advanced notice to maintainers
-// of what eventual threading support will change.
+// WARNING: Threading support is incomplete and experimental
+// Nonetheless, we define JL_THREAD and use it to give advanced notice to
+// maintainers of what eventual threading support will change.
 
 // JULIA_ENABLE_THREADING is switched on in Make.inc if JULIA_THREADS is
 // set (in Make.user)
 
-#ifndef JULIA_ENABLE_THREADING
-// Definition for compiling non-thread-safe Julia.
-#  define JL_THREAD
-#elif !defined(_OS_WINDOWS_)
-// Definition for compiling Julia on platforms with GCC __thread.
-#  define JL_THREAD __thread
-#else
-// Definition for compiling Julia on Windows
-#  define JL_THREAD __declspec(thread)
-#endif
-
 DLLEXPORT int16_t jl_threadid(void);
 DLLEXPORT void *jl_threadgroup(void);
 DLLEXPORT void jl_cpu_pause(void);
 DLLEXPORT void jl_threading_profile(void);
 
-#if __GNUC__
+#if defined(__GNUC__)
 #  define JL_ATOMIC_FETCH_AND_ADD(a,b)                                    \
        __sync_fetch_and_add(&(a), (b))
 #  define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c)                               \
@@ -93,7 +82,7 @@ DLLEXPORT void jl_threading_profile(void);
 #  define JL_ATOMIC_FETCH_AND_ADD(a,b)                                    \
        _InterlockedExchangeAdd((volatile LONG *)&(a), (b))
 #  define JL_ATOMIC_COMPARE_AND_SWAP(a,b,c)                               \
-       _InterlockedCompareExchange64(&(a), (c), (b))
+       _InterlockedCompareExchange64((volatile LONG64 *)&(a), (c), (b))
 #  define JL_ATOMIC_TEST_AND_SET(a)                                       \
        _InterlockedExchange64(&(a), 1)
 #  define JL_ATOMIC_RELEASE(a)                                            \
@@ -585,7 +574,7 @@ typedef struct _jl_gcframe_t {
 // jl_value_t *x=NULL, *y=NULL; JL_GC_PUSH2(&x, &y);
 // x = f(); y = g(); foo(x, y)
 
-extern DLLEXPORT JL_THREAD jl_gcframe_t *jl_pgcstack;
+#define jl_pgcstack (jl_get_ptls_states()->pgcstack)
 
 #define JL_GC_PUSH1(arg1)                                                 \
   void *__gc_stkf[] = {(void*)3, jl_pgcstack, arg1};                      \
@@ -1485,17 +1474,38 @@ typedef struct {
     jl_value_t * volatile *ptask_arg_in_transit;
 } jl_thread_task_state_t;
 
-extern DLLEXPORT JL_THREAD jl_task_t * volatile jl_current_task;
-extern DLLEXPORT JL_THREAD jl_task_t *jl_root_task;
-extern DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit;
-extern DLLEXPORT JL_THREAD jl_value_t * volatile jl_task_arg_in_transit;
+#define jl_current_task (jl_get_ptls_states()->current_task)
+#define jl_root_task (jl_get_ptls_states()->root_task)
+#define jl_exception_in_transit (jl_get_ptls_states()->exception_in_transit)
+#define jl_task_arg_in_transit (jl_get_ptls_states()->task_arg_in_transit)
 
 DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize);
 DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg);
 DLLEXPORT void NORETURN jl_throw(jl_value_t *e);
 DLLEXPORT void NORETURN jl_rethrow(void);
 DLLEXPORT void NORETURN jl_rethrow_other(jl_value_t *e);
 
+typedef struct _jl_tls_states_t {
+    jl_gcframe_t *pgcstack;
+    jl_value_t *exception_in_transit;
+    struct _jl_thread_heap_t *heap;
+    jl_task_t *volatile current_task;
+    jl_task_t *root_task;
+    jl_value_t *volatile task_arg_in_transit;
+    void *stackbase;
+    jl_jmp_buf *volatile jmp_target;
+    jl_jmp_buf base_ctx; // base context of stack
+    int16_t tid;
+} jl_tls_states_t;
+DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void);
+#ifndef JULIA_ENABLE_THREADING
+extern DLLEXPORT jl_tls_states_t jl_tls_states;
+#define jl_get_ptls_states() (&jl_tls_states)
+#else
+typedef jl_tls_states_t *(*jl_get_ptls_states_func)(void);
+DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f);
+#endif
+
 STATIC_INLINE void jl_eh_restore_state(jl_handler_t *eh)
 {
     JL_SIGATOMIC_BEGIN();

src/julia_internal.h

@@ -146,7 +146,7 @@ void jl_init_serializer(void);
 
 void _julia_init(JL_IMAGE_SEARCH rel);
 #ifdef COPY_STACKS
-extern JL_THREAD void *jl_stackbase;
+#define jl_stackbase (jl_get_ptls_states()->stackbase)
 #endif
 
 void jl_set_stackbase(char *__stk);
@@ -155,6 +155,9 @@ void jl_set_base_ctx(char *__stk);
 void jl_init_threading(void);
 void jl_start_threads(void);
 void jl_shutdown_threading(void);
+#ifdef JULIA_ENABLE_THREADING
+jl_get_ptls_states_func jl_get_ptls_states_getter(void);
+#endif
 
 void jl_dump_bitcode(char *fname, const char *sysimg_data, size_t sysimg_len);
 void jl_dump_objfile(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len);
@@ -335,6 +338,8 @@ DLLEXPORT jl_value_t *jl_flipsign_int(jl_value_t *a, jl_value_t *b);
 DLLEXPORT jl_value_t *jl_select_value(jl_value_t *isfalse, jl_value_t *a, jl_value_t *b);
 DLLEXPORT jl_value_t *jl_arraylen(jl_value_t *a);
 
+JL_DEFINE_MUTEX_EXT(codegen)
+
 #ifdef __cplusplus
 }
 #endif

src/task.c

@@ -138,21 +138,17 @@ static jl_sym_t *runnable_sym;
 
 extern size_t jl_page_size;
 jl_datatype_t *jl_task_type;
-DLLEXPORT JL_THREAD jl_task_t * volatile jl_current_task;
-JL_THREAD jl_task_t *jl_root_task;
-DLLEXPORT JL_THREAD jl_value_t *jl_exception_in_transit;
-DLLEXPORT JL_THREAD jl_gcframe_t *jl_pgcstack = NULL;
+#define jl_root_task (jl_get_ptls_states()->root_task)
 
 #ifdef COPY_STACKS
-static JL_THREAD jl_jmp_buf * volatile jl_jmp_target;
+#define jl_jmp_target (jl_get_ptls_states()->jmp_target)
 
 #if (defined(_CPU_X86_64_) || defined(_CPU_X86_)) && !defined(_COMPILER_MICROSOFT_)
 #define ASM_COPY_STACKS
 #endif
-JL_THREAD void *jl_stackbase;
 
 #ifndef ASM_COPY_STACKS
-static JL_THREAD jl_jmp_buf jl_base_ctx; // base context of stack
+#define jl_base_ctx (jl_get_ptls_states()->base_ctx)
 #endif
 
 static void NOINLINE save_stack(jl_task_t *t)
@@ -359,7 +355,6 @@ static void ctx_switch(jl_task_t *t, jl_jmp_buf *where)
     //JL_SIGATOMIC_END();
 }
 
-JL_THREAD jl_value_t * volatile jl_task_arg_in_transit;
 extern int jl_in_gc;
 DLLEXPORT jl_value_t *jl_switchto(jl_task_t *t, jl_value_t *arg)
 {

src/threading.c

@@ -21,6 +21,8 @@
 #ifndef _MSC_VER
 #include <unistd.h>
 #include <sched.h>
+#else
+#define sleep(x) Sleep(1000*x)
 #endif
 
 #include "julia.h"
@@ -34,8 +36,55 @@ extern "C" {
 #include "threadgroup.h"
 #include "threading.h"
 
+#ifdef JULIA_ENABLE_THREADING
+// fallback provided for embedding
+static JL_CONST_FUNC jl_tls_states_t *jl_get_ptls_states_fallback(void)
+{
+#  if !defined(_COMPILER_MICROSOFT_)
+    static __thread jl_tls_states_t tls_states;
+#  else
+    static __declspec(thread) jl_tls_states_t tls_states;
+#  endif
+    return &tls_states;
+}
+static jl_tls_states_t *jl_get_ptls_states_init(void);
+static jl_get_ptls_states_func jl_tls_states_cb = jl_get_ptls_states_init;
+static jl_tls_states_t *jl_get_ptls_states_init(void)
+{
+    // This is clearly not thread safe but should be fine since we
+    // make sure the tls states callback is finalized before adding
+    // multiple threads
+    jl_tls_states_cb = jl_get_ptls_states_fallback;
+    return jl_get_ptls_states_fallback();
+}
+DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void)
+{
+    return (*jl_tls_states_cb)();
+}
+DLLEXPORT void jl_set_ptls_states_getter(jl_get_ptls_states_func f)
+{
+    // only allow setting this once
+    if (f && f != jl_get_ptls_states_init &&
+        jl_tls_states_cb == jl_get_ptls_states_init) {
+        jl_tls_states_cb = f;
+    }
+}
+jl_get_ptls_states_func jl_get_ptls_states_getter(void)
+{
+    if (jl_tls_states_cb == jl_get_ptls_states_init)
+        jl_get_ptls_states_init();
+    // for codegen
+    return jl_tls_states_cb;
+}
+#else
+DLLEXPORT jl_tls_states_t jl_tls_states;
+DLLEXPORT JL_CONST_FUNC jl_tls_states_t *(jl_get_ptls_states)(void)
+{
+    return &jl_tls_states;
+}
+#endif
+
 // thread ID
-JL_THREAD int16_t ti_tid = 0;
 DLLEXPORT int jl_n_threads;     // # threads we're actually using
 DLLEXPORT int jl_max_threads;   // # threads possible
 jl_thread_task_state_t *jl_all_task_states;