diff --git a/base/boot.jl b/base/boot.jl index d2cf29d60ebba..ba95bdf70a32a 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -248,7 +248,8 @@ end type WeakRef value WeakRef() = WeakRef(nothing) - WeakRef(v::ANY) = ccall(:jl_gc_new_weakref, Ref{WeakRef}, (Any,), v) + WeakRef(v::ANY) = ccall(:jl_gc_new_weakref_th, Ref{WeakRef}, + (Ptr{Void}, Any), getptls(), v) end TypeVar(n::Symbol) = diff --git a/src/alloc.c b/src/alloc.c index a7861bcb655f5..44cc65928609a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -113,6 +113,7 @@ typedef struct { // Note that this function updates len static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_datatype(dt)); jl_datatype_t *bt = (jl_datatype_t*)dt; size_t nb = jl_datatype_size(bt); @@ -127,7 +128,7 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); - jl_value_t *v = (jl_value_t*)newobj((jl_value_t*)bt, NWORDS(nb)); + jl_value_t *v = jl_gc_alloc(ptls, nb, bt); switch (nb) { case 1: *(int8_t*) jl_data_ptr(v) = *(int8_t*)data; break; case 2: *(int16_t*) jl_data_ptr(v) = *(int16_t*)data; break; @@ -224,11 +225,12 @@ JL_DLLEXPORT int jl_field_isdefined(jl_value_t *v, size_t i) JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) { + jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; va_list args; size_t nf = jl_datatype_nfields(type); va_start(args, type); - jl_value_t *jv = newstruct(type); + jl_value_t *jv = newstruct(ptls, type); for(size_t i=0; i < nf; i++) { jl_set_nth_field(jv, i, va_arg(args, jl_value_t*)); } @@ -239,9 +241,10 @@ JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uint32_t na) { + jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; size_t nf = jl_datatype_nfields(type); - jl_value_t *jv = newstruct(type); + jl_value_t *jv = newstruct(ptls, type); for(size_t i=0; i < na; i++) { jl_set_nth_field(jv, i, args[i]); } @@ -255,8 +258,9 @@ JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) { + jl_ptls_t ptls = jl_get_ptls_states(); if (type->instance != NULL) return type->instance; - jl_value_t *jv = newstruct(type); + jl_value_t *jv = newstruct(ptls, type); if (type->size > 0) memset(jl_data_ptr(jv), 0, type->size); return jv; @@ -385,9 +389,10 @@ static void jl_lambda_info_set_ast(jl_lambda_info_t *li, jl_expr_t *ast) JL_DLLEXPORT jl_lambda_info_t *jl_new_lambda_info_uninit(void) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_lambda_info_t *li = - (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - NWORDS(sizeof(jl_lambda_info_t))); + (jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t), + jl_lambda_info_type); li->code = NULL; li->slotnames = li->slotflags = NULL; li->slottypes = li->ssavaluetypes = NULL; @@ -578,8 +583,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(void) { jl_ptls_t ptls = jl_get_ptls_states(); jl_method_t *m = - (jl_method_t*)newobj((jl_value_t*)jl_method_type, - NWORDS(sizeof(jl_method_t))); + (jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t), jl_method_type); m->specializations.unknown = jl_nothing; m->sig = NULL; m->tvars = NULL; @@ -809,8 +813,10 @@ jl_sym_t *jl_demangle_typename(jl_sym_t *s) JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module) { - jl_methtable_t *mt = (jl_methtable_t*)jl_gc_allocobj(sizeof(jl_methtable_t)); - jl_set_typeof(mt, jl_methtable_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_methtable_t *mt = + (jl_methtable_t*)jl_gc_alloc(ptls, sizeof(jl_methtable_t), + jl_methtable_type); mt->name = jl_demangle_typename(name); mt->module = module; mt->defs.unknown = jl_nothing; @@ -825,7 +831,10 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *module) { - jl_typename_t *tn=(jl_typename_t*)newobj((jl_value_t*)jl_typename_type, NWORDS(sizeof(jl_typename_t))); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_typename_t *tn = + (jl_typename_t*)jl_gc_alloc(ptls, sizeof(jl_typename_t), + jl_typename_type); tn->name = name; tn->module = module; tn->primary = NULL; @@ -856,12 +865,14 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_datatype_t *super, JL_DLLEXPORT jl_datatype_t *jl_new_uninitialized_datatype(size_t nfields, int8_t fielddesc_type) { + jl_ptls_t ptls = jl_get_ptls_states(); // fielddesc_type is specified manually for builtin types // and is (will be) calculated automatically for user defined types. uint32_t fielddesc_size = jl_fielddesc_size(fielddesc_type); - jl_datatype_t *t = (jl_datatype_t*) - newobj((jl_value_t*)jl_datatype_type, - NWORDS(sizeof(jl_datatype_t) + nfields * fielddesc_size)); + jl_datatype_t *t = + (jl_datatype_t*)jl_gc_alloc(ptls, (sizeof(jl_datatype_t) + + nfields * fielddesc_size), + jl_datatype_type); // fielddesc_type should only be assigned here. It can cause data // corruption otherwise. t->fielddesc_type = fielddesc_type; @@ -1080,6 +1091,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_bitstype(jl_value_t *name, jl_datatype_t *sup JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body) { + jl_ptls_t ptls = jl_get_ptls_states(); #ifndef NDEBUG size_t i, np = jl_svec_len(p); for (i = 0; i < np; i++) { @@ -1087,7 +1099,9 @@ JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body) assert(jl_is_typevar(tv) && !tv->bound); } #endif - jl_typector_t *tc = (jl_typector_t*)newobj((jl_value_t*)jl_typector_type, NWORDS(sizeof(jl_typector_t))); + jl_typector_t *tc = + (jl_typector_t*)jl_gc_alloc(ptls, sizeof(jl_typector_t), + jl_typector_type); tc->parameters = p; tc->body = body; return (jl_value_t*)tc; @@ -1099,10 +1113,10 @@ JL_DLLEXPORT jl_value_t *jl_new_type_constructor(jl_svec_t *p, jl_value_t *body) #define BOXN_FUNC(nb,nw) \ JL_DLLEXPORT jl_value_t *jl_box##nb(jl_datatype_t *t, int##nb##_t x) \ { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ assert(jl_isbits(t)); \ assert(jl_datatype_size(t) == sizeof(x)); \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, t); \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), t); \ *(int##nb##_t*)jl_data_ptr(v) = x; \ return v; \ } @@ -1135,13 +1149,14 @@ UNBOX_FUNC(float32, float) UNBOX_FUNC(float64, double) UNBOX_FUNC(voidpointer, void*) -#define BOX_FUNC(typ,c_type,pfx,nw) \ - JL_DLLEXPORT jl_value_t *pfx##_##typ(c_type x) \ - { \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, jl_##typ##_type); \ - *(c_type*)jl_data_ptr(v) = x; \ - return v; \ +#define BOX_FUNC(typ,c_type,pfx,nw) \ + JL_DLLEXPORT jl_value_t *pfx##_##typ(c_type x) \ + { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), \ + jl_##typ##_type); \ + *(c_type*)jl_data_ptr(v) = x; \ + return v; \ } BOX_FUNC(float32, float, jl_box, 1) BOX_FUNC(voidpointer, void*, jl_box, 1) @@ -1153,26 +1168,28 @@ BOX_FUNC(float64, double, jl_box, 2) #define NBOX_C 1024 -#define SIBOX_FUNC(typ,c_type,nw) \ - static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ - JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ - { \ - c_type idx = x+NBOX_C/2; \ - if ((u##c_type)idx < (u##c_type)NBOX_C) \ - return boxed_##typ##_cache[idx]; \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, jl_##typ##_type); \ - *(c_type*)jl_data_ptr(v) = x; \ - return v; \ +#define SIBOX_FUNC(typ,c_type,nw)\ + static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ + JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ + { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ + c_type idx = x+NBOX_C/2; \ + if ((u##c_type)idx < (u##c_type)NBOX_C) \ + return boxed_##typ##_cache[idx]; \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), \ + jl_##typ##_type); \ + *(c_type*)jl_data_ptr(v) = x; \ + return v; \ } #define UIBOX_FUNC(typ,c_type,nw) \ static jl_value_t *boxed_##typ##_cache[NBOX_C]; \ JL_DLLEXPORT jl_value_t *jl_box_##typ(c_type x) \ { \ + jl_ptls_t ptls = jl_get_ptls_states(); \ if (x < NBOX_C) \ return boxed_##typ##_cache[x]; \ - jl_value_t *v = (jl_value_t*)jl_gc_alloc_##nw##w(); \ - jl_set_typeof(v, jl_##typ##_type); \ + jl_value_t *v = jl_gc_alloc(ptls, nw * sizeof(void*), \ + jl_##typ##_type); \ *(c_type*)jl_data_ptr(v) = x; \ return v; \ } @@ -1236,23 +1253,23 @@ void jl_init_box_caches(void) } } -void jl_mark_box_caches(void) +void jl_mark_box_caches(jl_ptls_t ptls) { int64_t i; for(i=0; i < 256; i++) { - jl_gc_setmark(boxed_int8_cache[i]); - jl_gc_setmark(boxed_uint8_cache[i]); + jl_gc_setmark(ptls, boxed_int8_cache[i]); + jl_gc_setmark(ptls, boxed_uint8_cache[i]); } for(i=0; i < NBOX_C; i++) { - jl_gc_setmark(boxed_int16_cache[i]); - jl_gc_setmark(boxed_int32_cache[i]); - jl_gc_setmark(boxed_int64_cache[i]); - jl_gc_setmark(boxed_uint16_cache[i]); - jl_gc_setmark(boxed_uint32_cache[i]); - jl_gc_setmark(boxed_char_cache[i]); - jl_gc_setmark(boxed_uint64_cache[i]); - jl_gc_setmark(boxed_ssavalue_cache[i]); - jl_gc_setmark(boxed_slotnumber_cache[i]); + jl_gc_setmark(ptls, boxed_int16_cache[i]); + jl_gc_setmark(ptls, boxed_int32_cache[i]); + jl_gc_setmark(ptls, boxed_int64_cache[i]); + jl_gc_setmark(ptls, boxed_uint16_cache[i]); + jl_gc_setmark(ptls, boxed_uint32_cache[i]); + jl_gc_setmark(ptls, boxed_char_cache[i]); + jl_gc_setmark(ptls, boxed_uint64_cache[i]); + jl_gc_setmark(ptls, boxed_ssavalue_cache[i]); + jl_gc_setmark(ptls, boxed_slotnumber_cache[i]); } } @@ -1267,10 +1284,11 @@ JL_DLLEXPORT jl_value_t *jl_box_bool(int8_t x) jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *ar = n==0 ? (jl_array_t*)jl_an_empty_vec_any : jl_alloc_vec_any(n); JL_GC_PUSH1(&ar); - jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); - jl_set_typeof(ex, jl_expr_type); + jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc(ptls, sizeof(jl_expr_t), + jl_expr_type); ex->head = head; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; @@ -1280,14 +1298,15 @@ jl_expr_t *jl_exprn(jl_sym_t *head, size_t n) JL_CALLABLE(jl_f__expr) { + jl_ptls_t ptls = jl_get_ptls_states(); JL_NARGSV(Expr, 1); JL_TYPECHK(Expr, symbol, args[0]); jl_array_t *ar = jl_alloc_vec_any(nargs-1); JL_GC_PUSH1(&ar); for(size_t i=0; i < nargs-1; i++) jl_array_ptr_set(ar, i, args[i+1]); - jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc_3w(); assert(NWORDS(sizeof(jl_expr_t))==3); - jl_set_typeof(ex, jl_expr_type); + jl_expr_t *ex = (jl_expr_t*)jl_gc_alloc(ptls, sizeof(jl_expr_t), + jl_expr_type); ex->head = (jl_sym_t*)args[0]; ex->args = ar; ex->etype = (jl_value_t*)jl_any_type; diff --git a/src/array.c b/src/array.c index 9580a3eb74d55..40c19ec0da478 100644 --- a/src/array.c +++ b/src/array.c @@ -54,6 +54,7 @@ size_t jl_arr_xtralloc_limit = 0; static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, int isunboxed, int elsz) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t i, tot, nel=1; void *data; jl_array_t *a; @@ -90,9 +91,8 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, size_t doffs = tsz; tsz += tot; tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); // align whole object - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, (jl_datatype_t*)atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.how = 0; data = (char*)a + doffs; if (tot > 0 && !isunboxed) { @@ -104,11 +104,10 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, data = jl_gc_managed_malloc(tot); // Allocate the Array **after** allocating the data // to make sure the array is still young - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, (jl_datatype_t*)atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.how = 2; - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); if (!isunboxed) memset(data, 0, tot); } @@ -173,6 +172,7 @@ static inline int is_ntuple_long(jl_value_t *v) JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_value_t *_dims) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_array_t *a; size_t ndims = jl_nfields(_dims); assert(is_ntuple_long(_dims)); @@ -180,9 +180,8 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*), JL_SMALL_BYTE_ALIGNMENT); - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, (jl_datatype_t*)atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->flags.ndims = ndims; a->offset = 0; @@ -239,6 +238,7 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz; jl_array_t *a; jl_value_t *el_type = jl_tparam0(atype); @@ -251,9 +251,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, int ndimwords = jl_array_ndimwords(1); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, (jl_datatype_t*)atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN @@ -266,7 +265,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers if (own_buffer) { a->flags.how = 2; - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { @@ -282,6 +281,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *_dims, int own_buffer) { + jl_ptls_t ptls = jl_get_ptls_states(); size_t elsz, nel = 1; jl_array_t *a; size_t ndims = jl_nfields(_dims); @@ -306,9 +306,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, int ndimwords = jl_array_ndimwords(ndims); int tsz = JL_ARRAY_ALIGN(sizeof(jl_array_t) + ndimwords*sizeof(size_t), JL_CACHE_BYTE_ALIGNMENT); - a = (jl_array_t*)jl_gc_allocobj(tsz); + a = (jl_array_t*)jl_gc_alloc(ptls, tsz, (jl_datatype_t*)atype); // No allocation or safepoint allowed after this - jl_set_typeof(a, atype); a->flags.pooled = tsz <= GC_MAX_SZCLASS; a->data = data; #ifdef STORE_ARRAY_LEN @@ -322,7 +321,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, a->flags.isaligned = 0; if (own_buffer) { a->flags.how = 2; - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); } else { @@ -369,10 +368,10 @@ JL_DLLEXPORT jl_array_t *jl_pchar_to_array(const char *str, size_t len) JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a) { + jl_ptls_t ptls = jl_get_ptls_states(); if (!jl_typeis(a, jl_array_uint8_type)) jl_type_error("jl_array_to_string", (jl_value_t*)jl_array_uint8_type, (jl_value_t*)a); - jl_value_t *s = (jl_value_t*)jl_gc_alloc_1w(); - jl_set_typeof(s, jl_string_type); + jl_value_t *s = jl_gc_alloc(ptls, sizeof(void*), jl_string_type); jl_set_nth_field(s, 0, (jl_value_t*)a); return s; } @@ -565,6 +564,7 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i) // the **beginning** of the new buffer. static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(!a->flags.isshared || a->flags.how == 3); size_t elsz = a->elsize; size_t nbytes = newlen * elsz; @@ -593,12 +593,12 @@ static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) #endif ) { a->data = jl_gc_managed_malloc(nbytes); - jl_gc_track_malloced_array(a); + jl_gc_track_malloced_array(ptls, a); a->flags.how = 2; a->flags.isaligned = 1; } else { - a->data = allocb(nbytes); + a->data = jl_gc_alloc_buf(ptls, nbytes); a->flags.how = 1; jl_gc_wb_buf(a, a->data, nbytes); } diff --git a/src/ccall.cpp b/src/ccall.cpp index 071a4542bcfda..9f5eb0c5ff3d2 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -854,7 +854,7 @@ static jl_cgval_t mark_or_box_ccall_result(Value *result, bool isboxed, jl_value int nb = sizeof(void*); // TODO: can this be tighter than tbaa_value? return mark_julia_type( - init_bits_value(emit_allocobj(ctx, nb), runtime_bt, result, tbaa_value), + init_bits_value(emit_allocobj(ctx, nb, runtime_bt), result, tbaa_value), true, (jl_value_t*)jl_pointer_type, ctx); } return mark_julia_type(result, isboxed, rt, ctx); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 528645da22634..e7c5e13b0dc62 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1333,28 +1333,25 @@ static Value *emit_array_nd_index(const jl_cgval_t &ainfo, jl_value_t *ex, size_ // --- boxing --- -static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size); -static void init_tag(Value *v, Value *jt) +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt); +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, + const jl_cgval_t &v); +static Value *init_bits_value(Value *newv, Value *v, MDNode *tbaa) { - tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); -} -static Value *init_bits_value(Value *newv, Value *jt, Value *v, MDNode *tbaa) -{ - init_tag(newv, jt); + // newv should already be tagged tbaa_decorate(tbaa, builder.CreateAlignedStore(v, builder.CreateBitCast(newv, PointerType::get(v->getType(),0)), sizeof(void*))); // min alignment in julia's gc is pointer-aligned return newv; } static Value *as_value(Type *t, const jl_cgval_t&); static Value *init_bits_cgval(Value *newv, const jl_cgval_t& v, MDNode *tbaa, jl_codectx_t *ctx) { - Value *jt = literal_pointer_val(v.typ); + // newv should already be tagged if (v.ispointer()) { - init_tag(newv, jt); builder.CreateMemCpy(newv, data_pointer(v, ctx, T_pint8), jl_datatype_size(v.typ), sizeof(void*)); return newv; } else { - return init_bits_value(newv, jt, v.V, tbaa); + return init_bits_value(newv, v.V, tbaa); } } @@ -1521,7 +1518,8 @@ static Value *boxed(const jl_cgval_t &vinfo, jl_codectx_t *ctx, bool gcrooted) return literal_pointer_val(jb->instance); } else { - box = init_bits_cgval(emit_allocobj(ctx, jl_datatype_size(jt)), vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); + box = init_bits_cgval(emit_allocobj(ctx, jl_datatype_size(jt), vinfo), + vinfo, jb->mutabl ? tbaa_mutab : tbaa_immut, ctx); } if (gcrooted) { @@ -1555,23 +1553,33 @@ static void emit_cpointercheck(const jl_cgval_t &x, const std::string &msg, jl_c } // allocation for known size object -static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size) +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, Value *jt) { int osize; int end_offset; int offset = jl_gc_classify_pools(static_size, &osize, &end_offset); Value *ptls_ptr = builder.CreateBitCast(ctx->ptlsStates, T_pint8); + Value *v; if (offset < 0) { Value *args[] = {ptls_ptr, ConstantInt::get(T_size, static_size + sizeof(void*))}; - return builder.CreateCall(prepare_call(jlalloc_big_func), - ArrayRef(args, 2)); + v = builder.CreateCall(prepare_call(jlalloc_big_func), + ArrayRef(args, 2)); + } + else { + Value *pool_ptr = builder.CreateConstGEP1_32(ptls_ptr, offset); + Value *args[] = {ptls_ptr, pool_ptr, ConstantInt::get(T_int32, osize), + ConstantInt::get(T_int32, end_offset)}; + v = builder.CreateCall(prepare_call(jlalloc_pool_func), + ArrayRef(args, 4)); } - Value *pool_ptr = builder.CreateConstGEP1_32(ptls_ptr, offset); - Value *args[] = {ptls_ptr, pool_ptr, ConstantInt::get(T_int32, osize), - ConstantInt::get(T_int32, end_offset)}; - return builder.CreateCall(prepare_call(jlalloc_pool_func), - ArrayRef(args, 4)); + tbaa_decorate(tbaa_tag, builder.CreateStore(jt, emit_typeptr_addr(v))); + return v; +} +static Value *emit_allocobj(jl_codectx_t *ctx, size_t static_size, + const jl_cgval_t &v) +{ + return emit_allocobj(ctx, static_size, literal_pointer_val(v.typ)); } // if ptr is NULL this emits a write barrier _back_ @@ -1720,10 +1728,9 @@ static jl_cgval_t emit_new_struct(jl_value_t *ty, size_t nargs, jl_value_t **arg f1 = boxed(fval_info, ctx); j++; } - Value *strct = emit_allocobj(ctx, sty->size); + Value *strct = emit_allocobj(ctx, sty->size, + literal_pointer_val((jl_value_t*)ty)); jl_cgval_t strctinfo = mark_julia_type(strct, true, ty, ctx); - tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ty), - emit_typeptr_addr(strct))); if (f1) { jl_cgval_t f1info = mark_julia_type(f1, true, jl_any_type, ctx); if (!jl_subtype(expr_type(args[1],ctx), jl_field_type(sty,0), 0)) diff --git a/src/codegen.cpp b/src/codegen.cpp index 907f8a69a74b9..1047d050fd919 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3704,9 +3704,8 @@ static Function *gen_cfun_wrapper(jl_function_t *ff, jl_value_t *jlrettype, jl_t (void)julia_type_to_llvm(jargty, &isboxed); if (isboxed) { // passed an unboxed T, but want something boxed - Value *mem = emit_allocobj(&ctx, jl_datatype_size(jargty)); - tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)jargty), - emit_typeptr_addr(mem))); + Value *mem = emit_allocobj(&ctx, jl_datatype_size(jargty), + literal_pointer_val((jl_value_t*)jargty)); tbaa_decorate(jl_is_mutable(jargty) ? tbaa_mutab : tbaa_immut, builder.CreateAlignedStore(val, builder.CreateBitCast(mem, val->getType()->getPointerTo()), diff --git a/src/dump.c b/src/dump.c index 4eb9295189a4f..10a9ecc66e212 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1311,6 +1311,7 @@ static jl_value_t *jl_deserialize_value(ios_t *s, jl_value_t **loc) static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t **loc) { + jl_ptls_t ptls = jl_get_ptls_states(); int usetable = (mode != MODE_AST); size_t i; @@ -1422,7 +1423,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t return (jl_value_t*)e; } else if (vtag == (jl_value_t*)jl_tvar_type) { - jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, NWORDS(sizeof(jl_tvar_t))); + jl_tvar_t *tv = (jl_tvar_t*)jl_gc_alloc(ptls, sizeof(jl_tvar_t), + jl_tvar_type); if (usetable) arraylist_push(&backref_list, tv); tv->name = (jl_sym_t*)jl_deserialize_value(s, NULL); @@ -1436,8 +1438,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_method_type) { jl_method_t *m = - (jl_method_t*)newobj((jl_value_t*)jl_method_type, - NWORDS(sizeof(jl_method_t))); + (jl_method_t*)jl_gc_alloc(ptls, sizeof(jl_method_t), + jl_method_type); if (usetable) arraylist_push(&backref_list, m); m->specializations.unknown = jl_deserialize_value(s, (jl_value_t**)&m->specializations); @@ -1473,8 +1475,8 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_lambda_info_type) { jl_lambda_info_t *li = - (jl_lambda_info_t*)newobj((jl_value_t*)jl_lambda_info_type, - NWORDS(sizeof(jl_lambda_info_t))); + (jl_lambda_info_t*)jl_gc_alloc(ptls, sizeof(jl_lambda_info_t), + jl_lambda_info_type); if (usetable) arraylist_push(&backref_list, li); li->code = jl_deserialize_value(s, &li->code); jl_gc_wb(li, li->code); @@ -1606,7 +1608,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t } else if (vtag == (jl_value_t*)jl_datatype_type || vtag == (jl_value_t*)SmallDataType_tag) { int32_t sz = (vtag == (jl_value_t*)SmallDataType_tag ? read_uint8(s) : read_int32(s)); - jl_value_t *v = jl_gc_allocobj(sz); + jl_value_t *v = jl_gc_alloc(ptls, sz, NULL); int pos = backref_list.len; if (usetable) arraylist_push(&backref_list, v); @@ -1666,7 +1668,7 @@ static jl_value_t *jl_deserialize_value_(ios_t *s, jl_value_t *vtag, jl_value_t backref_list.items[pos] = dt->instance; return dt->instance; } - jl_value_t *v = (jl_value_t*)jl_gc_alloc_0w(); + jl_value_t *v = (jl_value_t*)jl_gc_alloc(ptls, 0, NULL); if (usetable) { uintptr_t pos = backref_list.len; arraylist_push(&backref_list, (void*)v); diff --git a/src/gc.c b/src/gc.c index 557c9195645e1..e52185e0701c7 100644 --- a/src/gc.c +++ b/src/gc.c @@ -100,9 +100,8 @@ static void schedule_finalization(void *o, void *f) arraylist_push(&to_finalize, f); } -static void run_finalizer(jl_value_t *o, jl_value_t *ff) +static void run_finalizer(jl_ptls_t ptls, jl_value_t *o, jl_value_t *ff) { - jl_ptls_t ptls = jl_get_ptls_states(); assert(!jl_typeis(ff, jl_voidpointer_type)); jl_value_t *args[2] = {ff,o}; JL_TRY { @@ -179,9 +178,8 @@ static void jl_gc_run_finalizers_in_list(jl_ptls_t ptls, arraylist_t *list) jl_value_t **items = (jl_value_t**)list->items; jl_gc_push_arraylist(ptls, list); JL_UNLOCK_NOGC(&finalizers_lock); - for (size_t i = 2;i < len;i += 2) { - run_finalizer(items[i], items[i + 1]); - } + for (size_t i = 2;i < len;i += 2) + run_finalizer(ptls, items[i], items[i + 1]); JL_GC_POP(); } @@ -262,14 +260,13 @@ static void gc_add_finalizer_(jl_ptls_t ptls, void *v, void *f) jl_gc_unsafe_leave(ptls, gc_state); } -STATIC_INLINE void gc_add_ptr_finalizer(jl_ptls_t ptls, - jl_value_t *v, void *f) +STATIC_INLINE void gc_add_ptr_finalizer(jl_ptls_t ptls, jl_value_t *v, void *f) { gc_add_finalizer_(ptls, (void*)(((uintptr_t)v) | 1), f); } -JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_ptls_t ptls, - jl_value_t *v, jl_function_t *f) +JL_DLLEXPORT void jl_gc_add_finalizer_th(jl_ptls_t ptls, jl_value_t *v, + jl_function_t *f) { if (__unlikely(jl_typeis(f, jl_voidpointer_type))) { gc_add_ptr_finalizer(ptls, v, jl_unbox_voidpointer(f)); @@ -311,8 +308,6 @@ JL_DLLEXPORT void jl_finalize_th(jl_ptls_t ptls, jl_value_t *o) arraylist_free(&copied_list); } -#define GC_POOL_END_OFS(osize) ((((GC_PAGE_SZ - GC_PAGE_OFFSET)/(osize)) - 1)*(osize) + GC_PAGE_OFFSET) - // GC knobs and self-measurement variables static int64_t last_gc_total_bytes = 0; @@ -550,11 +545,11 @@ static inline int maybe_collect(jl_ptls_t ptls) // weak references -JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) +JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref_th(jl_ptls_t ptls, + jl_value_t *value) { - jl_ptls_t ptls = jl_get_ptls_states(); - jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc_1w(); - jl_set_typeof(wr, jl_weakref_type); + jl_weakref_t *wr = (jl_weakref_t*)jl_gc_alloc(ptls, sizeof(void*), + jl_weakref_type); wr->value = value; // NOTE: wb not needed here arraylist_push(&ptls->heap.weak_refs, wr); return wr; @@ -593,7 +588,8 @@ static void sweep_weak_refs(void) // big value list -static NOINLINE jl_taggedvalue_t *alloc_big(jl_ptls_t ptls, size_t sz) +// Size includes the tag and the tag is not cleared!! +JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz) { maybe_collect(ptls); size_t offs = offsetof(bigval_t, header); @@ -609,10 +605,9 @@ static NOINLINE jl_taggedvalue_t *alloc_big(jl_ptls_t ptls, size_t sz) memset(v, 0xee, allocsz); #endif v->sz = allocsz; - v->header = 0; v->age = 0; gc_big_object_link(v, &ptls->heap.big_objects); - return (jl_taggedvalue_t*)&v->header; + return jl_valueof(&v->header); } // Sweep list rooted at *pv, removing and freeing any unmarked objects. @@ -677,10 +672,9 @@ static void sweep_big(jl_ptls_t ptls, int sweep_full) // tracking Arrays with malloc'd storage -void jl_gc_track_malloced_array(jl_array_t *a) +void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) { // This is **NOT** a GC safe point. - jl_ptls_t ptls = jl_get_ptls_states(); mallocarray_t *ma; if (ptls->heap.mafreelist == NULL) { ma = (mallocarray_t*)malloc(sizeof(mallocarray_t)); @@ -773,7 +767,7 @@ static inline jl_taggedvalue_t *reset_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg return beg; } -static NOINLINE void add_page(jl_gc_pool_t *p) +static NOINLINE jl_taggedvalue_t *add_page(jl_gc_pool_t *p) { // Do not pass in `ptls` as argument. This slows down the fast path // in pool_alloc significantly @@ -788,15 +782,17 @@ static NOINLINE void add_page(jl_gc_pool_t *p) pg->thread_n = ptls->tid; jl_taggedvalue_t *fl = reset_page(p, pg, p->newpages); p->newpages = fl; + return fl; } -static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, - int osize, int end_offset) +// Size includes the tag and the tag is not cleared!! +JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, + int osize, int end_offset) { + assert(ptls->gc_state == 0); #ifdef MEMDEBUG - return alloc_big(ptls, osize); + return jl_gc_big_alloc(ptls, osize); #endif - jl_taggedvalue_t *v, *end; // FIXME - need JL_ATOMIC_FETCH_AND_ADD here if (__unlikely((gc_num.allocd += osize) >= 0) || gc_debug_check_pool()) { //gc_num.allocd -= osize; @@ -808,11 +804,9 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, } gc_num.poolalloc++; // first try to use the freelist - v = p->freelist; + jl_taggedvalue_t *v = p->freelist; if (v) { jl_taggedvalue_t *next = v->next; - v->header = 0; - p->nfree--; p->freelist = next; if (__unlikely(gc_page_data(v) != gc_page_data(next))) { // we only update pg's fields when the freelist changes page @@ -821,18 +815,14 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, assert(pg->osize == p->osize); pg->nfree = 0; pg->has_young = 1; - if (next) - p->nfree = page_metadata(next)->nfree; } - return v; + return jl_valueof(v); } // if the freelist is empty we reuse empty but not freed pages v = p->newpages; - if (__unlikely(!v)) { - add_page(p); - v = p->newpages; - } - end = (jl_taggedvalue_t*)&(gc_page_data(v)[end_offset]); + if (__unlikely(!v)) + v = add_page(p); + jl_taggedvalue_t *end = (jl_taggedvalue_t*)&(gc_page_data(v)[end_offset]); if (__likely(v != end)) { p->newpages = (jl_taggedvalue_t*)((char*)v + osize); } @@ -844,88 +834,7 @@ static inline jl_taggedvalue_t *__pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, pg->has_young = 1; p->newpages = v->next; } - v->header = 0; - return v; -} - -// use this variant when osize is statically known -// and is definitely in sizeclasses -// GC_POOL_END_OFS uses an integer division -static inline jl_taggedvalue_t *_pool_alloc(jl_ptls_t ptls, - jl_gc_pool_t *p, int osize) -{ - return __pool_alloc(ptls, p, osize, GC_POOL_END_OFS(osize)); -} - -static inline jl_taggedvalue_t *pool_alloc(jl_ptls_t ptls, - jl_gc_pool_t *p) -{ - return __pool_alloc(ptls, p, p->osize, p->end_offset); -} - -// Size includes the tag!! -JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, - int osize, int end_offset) -{ - return jl_valueof(__pool_alloc(ptls, p, osize, end_offset)); -} - -// Size includes the tag!! -JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz) -{ - return jl_valueof(alloc_big(ptls, allocsz)); -} - -// pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) -static const int sizeclasses[JL_GC_N_POOLS] = { -#ifdef _P64 - 8, -#else - 4, 8, 12, -#endif - - // 16 pools at 16-byte spacing - 16, 32, 48, 64, 80, 96, 112, 128, - 144, 160, 176, 192, 208, 224, 240, 256, - - // the following tables are computed for maximum packing efficiency via the formula: - // sz=(div(2^14-8,rng)÷16)*16; hcat(sz, (2^14-8)÷sz, 2^14-(2^14-8)÷sz.*sz)' - - // rng = 60:-4:32 (8 pools) - 272, 288, 304, 336, 368, 400, 448, 496, -// 60, 56, 53, 48, 44, 40, 36, 33, /pool -// 64, 256, 272, 256, 192, 384, 256, 16, bytes lost - - // rng = 30:-2:16 (8 pools) - 544, 576, 624, 672, 736, 816, 896, 1008, -// 30, 28, 26, 24, 22, 20, 18, 16, /pool -// 64, 256, 160, 256, 192, 64, 256, 256, bytes lost - - // rng = 15:-1:8 (8 pools) - 1088, 1168, 1248, 1360, 1488, 1632, 1808, 2032 -// 15, 14, 13, 12, 11, 10, 9, 8, /pool -// 64, 32, 160, 64, 16, 64, 112, 128, bytes lost -}; - -static inline int szclass(size_t sz) -{ -#ifdef _P64 - if (sz <= 8) - return 0; - const int N = 0; -#else - if (sz <= 12) - return (sz + 3) / 4 - 1; - const int N = 2; -#endif - if (sz <= 256) - return (sz + 15) / 16 + N; - if (sz <= 496) - return 16 - 16376 / 4 / LLT_ALIGN(sz, 16 * 4) + 16 + N; - if (sz <= 1008) - return 16 - 16376 / 2 / LLT_ALIGN(sz, 16 * 2) + 24 + N; - assert(sz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t) && sizeclasses[JL_GC_N_POOLS-1] == GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)); - return 16 - 16376 / 1 / LLT_ALIGN(sz, 16 * 1) + 32 + N; + return jl_valueof(v); } int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) @@ -933,8 +842,8 @@ int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset) if (sz > GC_MAX_SZCLASS) return -1; size_t allocsz = sz + sizeof(jl_taggedvalue_t); - int klass = szclass(allocsz); - *osize = sizeclasses[klass]; + int klass = jl_gc_szclass(allocsz); + *osize = jl_gc_sizeclasses[klass]; *end_offset = GC_POOL_END_OFS(*osize); return (int)(intptr_t)(&((jl_ptls_t)0)->heap.norm_pools[klass]); } @@ -1093,6 +1002,22 @@ static void gc_sweep_other(jl_ptls_t ptls, int sweep_full) sweep_big(ptls, sweep_full); } +static void gc_pool_sync_nfree(jl_gc_pagemeta_t *pg, jl_taggedvalue_t *last) +{ + assert(pg->fl_begin_offset != -1); + char *cur_pg = gc_page_data(last); + // Fast path for page that has no allocation + jl_taggedvalue_t *fl_beg = (jl_taggedvalue_t*)(cur_pg + pg->fl_begin_offset); + if (last == fl_beg) + return; + int nfree = 0; + do { + nfree++; + last = last->next; + } while (gc_page_data(last) == cur_pg); + pg->nfree = nfree; +} + static void gc_sweep_pool(int sweep_full) { gc_time_pool_start(); @@ -1109,8 +1034,8 @@ static void gc_sweep_pool(int sweep_full) jl_taggedvalue_t *last = p->freelist; if (last) { jl_gc_pagemeta_t *pg = page_metadata(last); + gc_pool_sync_nfree(pg, last); pg->has_young = 1; - pg->nfree = p->nfree; } p->freelist = NULL; pfl[t_i * JL_GC_N_POOLS + i] = &p->freelist; @@ -1132,15 +1057,10 @@ static void gc_sweep_pool(int sweep_full) } - // null out terminal pointers of free lists and cache back pg->nfree in the jl_gc_pool_t + // null out terminal pointers of free lists for (int t_i = 0;t_i < jl_n_threads;t_i++) { - jl_ptls_t ptls2 = jl_all_tls_states[t_i]; for (int i = 0; i < JL_GC_N_POOLS; i++) { - jl_gc_pool_t *p = &ptls2->heap.norm_pools[i]; *pfl[t_i * JL_GC_N_POOLS + i] = NULL; - if (p->freelist) { - p->nfree = page_metadata(p->freelist)->nfree; - } } } @@ -1232,9 +1152,9 @@ static inline int gc_push_root(jl_ptls_t ptls, void *v, int d) // v isa jl_value return !gc_old(bits); } -void jl_gc_setmark(jl_value_t *v) // TODO rename this as it is misleading now +// TODO rename this as it is misleading now +void jl_gc_setmark(jl_ptls_t ptls, jl_value_t *v) { - jl_ptls_t ptls = jl_get_ptls_states(); jl_taggedvalue_t *o = jl_astaggedvalue(v); if (!gc_marked(o->bits.gc)) { gc_setmark_pool(ptls, o, GC_MARKED); @@ -1555,8 +1475,6 @@ void visit_mark_stack(jl_ptls_t ptls) assert(!mark_sp); } -void jl_mark_box_caches(void); - extern jl_array_t *jl_module_init_order; extern jl_typemap_entry_t *call_cache[N_CALL_CACHE]; @@ -1592,7 +1510,7 @@ void pre_mark(jl_ptls_t ptls) if (call_cache[i]) gc_push_root(ptls, call_cache[i], 0); - jl_mark_box_caches(); + jl_mark_box_caches(ptls); //gc_push_root(ptls, jl_unprotect_stack_func, 0); gc_push_root(ptls, jl_typetype_type, 0); @@ -1934,84 +1852,25 @@ JL_DLLEXPORT void jl_gc_collect(int full) // allocator entry points -void *allocb(size_t sz) +JL_DLLEXPORT jl_value_t *(jl_gc_alloc)(jl_ptls_t ptls, size_t sz, + jl_datatype_t *ty) { - jl_ptls_t ptls = jl_get_ptls_states(); - jl_taggedvalue_t *b = NULL; - size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (allocsz < sz) // overflow in adding offs, size was "negative" - jl_throw(jl_memory_exception); - if (allocsz > GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { - b = alloc_big(ptls, allocsz); - b->header = jl_buff_tag; - } - else { - b = pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)]); - b->header = jl_buff_tag; - } - return jl_valueof(b); -} - -JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (allocsz < sz) // overflow in adding offs, size was "negative" - jl_throw(jl_memory_exception); - if (allocsz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { - return jl_valueof(pool_alloc(ptls, &ptls->heap.norm_pools[szclass(allocsz)])); - } - return jl_valueof(alloc_big(ptls, allocsz)); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = sizeof(jl_taggedvalue_t); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*), - JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 2, - JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); -} - -JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) -{ - jl_ptls_t ptls = jl_get_ptls_states(); - const int sz = LLT_ALIGN(sizeof(jl_taggedvalue_t) + sizeof(void*) * 3, - JL_SMALL_BYTE_ALIGNMENT); - void *tag = _pool_alloc(ptls, &ptls->heap.norm_pools[szclass(sz)], sz); - return jl_valueof(tag); + return jl_gc_alloc_(ptls, sz, ty); } // Per-thread initialization (when threading is fully implemented) void jl_mk_thread_heap(jl_ptls_t ptls) { jl_thread_heap_t *heap = &ptls->heap; - const int *szc = sizeclasses; jl_gc_pool_t *p = heap->norm_pools; for(int i=0; i < JL_GC_N_POOLS; i++) { - assert((szc[i] < 16 && szc[i] % sizeof(void*) == 0) || - (szc[i] % 16 == 0)); - p[i].osize = szc[i]; + assert((jl_gc_sizeclasses[i] < 16 && + jl_gc_sizeclasses[i] % sizeof(void*) == 0) || + (jl_gc_sizeclasses[i] % 16 == 0)); + p[i].osize = jl_gc_sizeclasses[i]; p[i].freelist = NULL; p[i].newpages = NULL; - p[i].end_offset = GC_POOL_END_OFS(szc[i]); + p[i].end_offset = GC_POOL_END_OFS(jl_gc_sizeclasses[i]); } arraylist_new(&heap->weak_refs, 0); heap->mallocarrays = NULL; @@ -2194,6 +2053,42 @@ JL_DLLEXPORT void jl_finalize(jl_value_t *o) jl_finalize_th(ptls, o); } +JL_DLLEXPORT jl_weakref_t *jl_gc_new_weakref(jl_value_t *value) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_new_weakref_th(ptls, value); +} + +JL_DLLEXPORT jl_value_t *jl_gc_allocobj(size_t sz) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sz, NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_0w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, 0, NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_1w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sizeof(void*), NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_2w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sizeof(void*) * 2, NULL); +} + +JL_DLLEXPORT jl_value_t *jl_gc_alloc_3w(void) +{ + jl_ptls_t ptls = jl_get_ptls_states(); + return jl_gc_alloc(ptls, sizeof(void*) * 3, NULL); +} + #ifdef __cplusplus } #endif diff --git a/src/gc.h b/src/gc.h index 51b99842372ec..d59c7db7069a4 100644 --- a/src/gc.h +++ b/src/gc.h @@ -32,15 +32,10 @@ extern "C" { // manipulating mark bits -#define GC_PAGE_LG2 14 // log2(size of a page) -#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k -#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT)) - // 8G * 32768 = 2^48 // It's really unlikely that we'll actually allocate that much though... #define REGION_COUNT 32768 -#define jl_buff_tag ((uintptr_t)0x4eade800) #define jl_malloc_tag ((void*)0xdeadaa01) #define jl_singleton_tag ((void*)0xdeadaa02) @@ -278,6 +273,7 @@ void pre_mark(jl_ptls_t ptls); void gc_mark_object_list(jl_ptls_t ptls, arraylist_t *list, size_t start); void visit_mark_stack(jl_ptls_t ptls); void gc_debug_init(void); +void jl_mark_box_caches(jl_ptls_t ptls); // GC pages diff --git a/src/interpreter.c b/src/interpreter.c index c8e844d8d07bc..e8c36ef9efbea 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -400,7 +400,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) } jl_compute_field_offsets(dt); if (para == (jl_value_t*)jl_emptysvec && jl_is_datatype_make_singleton(dt)) { - dt->instance = newstruct(dt); + dt->instance = newstruct(ptls, dt); jl_gc_wb(dt, dt->instance); } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 626774ff02be3..ffc6c4326f097 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -507,7 +507,8 @@ static jl_cgval_t generic_box(jl_value_t *targ, jl_value_t *x, jl_codectx_t *ctx return mark_julia_type(vx, false, bt, ctx); else return mark_julia_type( - init_bits_value(emit_allocobj(ctx, nb), boxed(bt_value, ctx), vx, tbaa_immut), + init_bits_value(emit_allocobj(ctx, nb, boxed(bt_value, ctx)), + vx, tbaa_immut), true, bt, ctx); } @@ -548,8 +549,7 @@ static jl_cgval_t generic_unbox(jl_value_t *targ, jl_value_t *x, jl_codectx_t *c Value *runtime_bt = boxed(bt_value, ctx); // XXX: emit type validity check on runtime_bt (bitstype of size nb) - Value *newobj = emit_allocobj(ctx, nb); - tbaa_decorate(tbaa_tag, builder.CreateStore(runtime_bt, emit_typeptr_addr(newobj))); + Value *newobj = emit_allocobj(ctx, nb, runtime_bt); if (!v.ispointer()) { tbaa_decorate(tbaa_value, builder.CreateAlignedStore(emit_unbox(llvmt, v, v.typ), builder.CreatePointerCast(newobj, llvmt->getPointerTo()), alignment)); } @@ -772,9 +772,8 @@ static jl_cgval_t emit_pointerref(jl_value_t *e, jl_value_t *i, jl_codectx_t *ct } assert(jl_is_datatype(ety)); uint64_t size = jl_datatype_size(ety); - Value *strct = emit_allocobj(ctx, size); - tbaa_decorate(tbaa_tag, builder.CreateStore(literal_pointer_val((jl_value_t*)ety), - emit_typeptr_addr(strct))); + Value *strct = emit_allocobj(ctx, size, + literal_pointer_val((jl_value_t*)ety)); im1 = builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, ((jl_datatype_t*)ety)->alignment))); thePtr = builder.CreateGEP(builder.CreateBitCast(thePtr, T_pint8), im1); diff --git a/src/jltypes.c b/src/jltypes.c index c8e8158e7c559..3668898d927ef 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -311,6 +311,7 @@ static jl_svec_t *jl_compute_type_union(jl_value_t **types, size_t ntypes) static jl_value_t *jl_type_union_v(jl_value_t **ts, size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); if (n == 0) return (jl_value_t*)jl_bottom_type; size_t i; for(i=0; i < n; i++) { @@ -323,7 +324,9 @@ static jl_value_t *jl_type_union_v(jl_value_t **ts, size_t n) if (jl_svec_len(types) == 0) return (jl_value_t*)jl_bottom_type; if (jl_svec_len(types) == 1) return jl_svecref(types, 0); JL_GC_PUSH1(&types); - jl_uniontype_t *tu = (jl_uniontype_t*)newobj((jl_value_t*)jl_uniontype_type,NWORDS(sizeof(jl_uniontype_t))); + jl_uniontype_t *tu = + (jl_uniontype_t*)jl_gc_alloc(ptls, sizeof(jl_uniontype_t), + jl_uniontype_type); tu->types = types; jl_gc_wb(tu, types); JL_GC_POP(); @@ -2129,6 +2132,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i int cacheable, jl_typestack_t *stack, jl_value_t **env, size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_typestack_t top; jl_typename_t *tn = dt->name; jl_value_t *tc = tn->primary; @@ -2245,7 +2249,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i ndt->pointerfree = dt->pointerfree; ndt->types = jl_emptysvec; if (jl_is_datatype_make_singleton(ndt)) { - ndt->instance = newstruct(ndt); + ndt->instance = jl_gc_alloc(ptls, 0, ndt); jl_gc_wb(ndt, ndt->instance); } } @@ -2266,7 +2270,7 @@ static jl_value_t *inst_datatype(jl_datatype_t *dt, jl_svec_t *p, jl_value_t **i if (cacheable) { jl_compute_field_offsets(ndt); if (jl_is_datatype_make_singleton(ndt)) { - ndt->instance = newstruct(ndt); + ndt->instance = jl_gc_alloc(ptls, 0, ndt); jl_gc_wb(ndt, ndt->instance); } } @@ -2511,6 +2515,7 @@ jl_datatype_t *jl_wrap_Type(jl_value_t *t) void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! { + jl_ptls_t ptls = jl_get_ptls_states(); inside_typedef = 0; assert(jl_is_datatype(t)); jl_typestack_t top; @@ -2550,7 +2555,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! if (ndt->uid) { // cacheable jl_compute_field_offsets(ndt); if (jl_is_datatype_make_singleton(ndt)) { - ndt->instance = newstruct(ndt); + ndt->instance = jl_gc_alloc(ptls, 0, ndt); jl_gc_wb(ndt, ndt->instance); } } @@ -3432,7 +3437,9 @@ jl_value_t *jl_type_match_morespecific(jl_value_t *a, jl_value_t *b) JL_DLLEXPORT jl_tvar_t *jl_new_typevar_(jl_sym_t *name, jl_value_t *lb, jl_value_t *ub, jl_value_t *b) { - jl_tvar_t *tv = (jl_tvar_t*)newobj((jl_value_t*)jl_tvar_type, 4); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_tvar_t *tv = (jl_tvar_t*)jl_gc_alloc(ptls, sizeof(jl_tvar_t), + jl_tvar_type); tv->name = name; tv->lb = lb; tv->ub = ub; @@ -3466,9 +3473,10 @@ void jl_init_types(void) jl_symbol_type = jl_sym_type; jl_simplevector_type = jl_new_uninitialized_datatype(1, 1); jl_methtable_type = jl_new_uninitialized_datatype(6, 1); - jl_nothing = jl_gc_alloc_0w(); + jl_nothing = jl_gc_alloc(ptls, 0, NULL); - jl_emptysvec = (jl_svec_t*)newobj((jl_value_t*)jl_simplevector_type, 1); + jl_emptysvec = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*), + jl_simplevector_type); jl_svec_set_len_unsafe(jl_emptysvec, 0); jl_any_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), NULL, jl_emptysvec); @@ -3628,7 +3636,7 @@ void jl_init_types(void) jl_tupletype_t *empty_tuple_type = jl_apply_tuple_type(jl_emptysvec); empty_tuple_type->uid = jl_assign_type_uid(); - jl_emptytuple = newstruct(empty_tuple_type); + jl_emptytuple = jl_gc_alloc(ptls, 0, empty_tuple_type); empty_tuple_type->instance = jl_emptytuple; // non-primitive definitions follow diff --git a/src/julia_internal.h b/src/julia_internal.h index 0e8215c1aa2fb..d7498b957539c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -40,35 +40,123 @@ extern unsigned sig_stack_size; JL_DLLEXPORT extern int jl_lineno; JL_DLLEXPORT extern const char *jl_filename; -JL_DLLEXPORT void *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, - int osize, int end_offset); +JL_DLLEXPORT jl_value_t *jl_gc_pool_alloc(jl_ptls_t ptls, jl_gc_pool_t *p, + int osize, int end_offset); JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t allocsz); int jl_gc_classify_pools(size_t sz, int *osize, int *end_offset); -STATIC_INLINE jl_value_t *newobj(jl_value_t *type, size_t nfields) +// pools are 16376 bytes large (GC_POOL_SZ - GC_PAGE_OFFSET) +static const int jl_gc_sizeclasses[JL_GC_N_POOLS] = { +#ifdef _P64 + 8, +#else + 4, 8, 12, +#endif + + // 16 pools at 16-byte spacing + 16, 32, 48, 64, 80, 96, 112, 128, + 144, 160, 176, 192, 208, 224, 240, 256, + + // the following tables are computed for maximum packing efficiency via the formula: + // sz=(div(2^14-8,rng)÷16)*16; hcat(sz, (2^14-8)÷sz, 2^14-(2^14-8)÷sz.*sz)' + + // rng = 60:-4:32 (8 pools) + 272, 288, 304, 336, 368, 400, 448, 496, +// 60, 56, 53, 48, 44, 40, 36, 33, /pool +// 64, 256, 272, 256, 192, 384, 256, 16, bytes lost + + // rng = 30:-2:16 (8 pools) + 544, 576, 624, 672, 736, 816, 896, 1008, +// 30, 28, 26, 24, 22, 20, 18, 16, /pool +// 64, 256, 160, 256, 192, 64, 256, 256, bytes lost + + // rng = 15:-1:8 (8 pools) + 1088, 1168, 1248, 1360, 1488, 1632, 1808, 2032 +// 15, 14, 13, 12, 11, 10, 9, 8, /pool +// 64, 32, 160, 64, 16, 64, 112, 128, bytes lost +}; + +STATIC_INLINE int JL_CONST_FUNC jl_gc_szclass(size_t sz) { - jl_value_t *jv = NULL; - switch (nfields) { - case 0: - jv = (jl_value_t*)jl_gc_alloc_0w(); break; - case 1: - jv = (jl_value_t*)jl_gc_alloc_1w(); break; - case 2: - jv = (jl_value_t*)jl_gc_alloc_2w(); break; - case 3: - jv = (jl_value_t*)jl_gc_alloc_3w(); break; - default: - jv = (jl_value_t*)jl_gc_allocobj(nfields * sizeof(void*)); +#ifdef _P64 + if (sz <= 8) + return 0; + const int N = 0; +#else + if (sz <= 12) + return (sz + 3) / 4 - 1; + const int N = 2; +#endif + if (sz <= 256) + return (sz + 15) / 16 + N; + if (sz <= 496) + return 16 - 16376 / 4 / LLT_ALIGN(sz, 16 * 4) + 16 + N; + if (sz <= 1008) + return 16 - 16376 / 2 / LLT_ALIGN(sz, 16 * 2) + 24 + N; + return 16 - 16376 / 1 / LLT_ALIGN(sz, 16 * 1) + 32 + N; +} + +#ifdef __GNUC__ +# define jl_is_constexpr(e) __builtin_constant_p(e) +#else +# define jl_is_constexpr(e) (0) +#endif +#define JL_SMALL_BYTE_ALIGNMENT 16 +#define JL_CACHE_BYTE_ALIGNMENT 64 +#define GC_POOL_END_OFS(osize) ((((GC_PAGE_SZ - GC_PAGE_OFFSET)/(osize)) - 1)*(osize) + GC_PAGE_OFFSET) +#define GC_PAGE_LG2 14 // log2(size of a page) +#define GC_PAGE_SZ (1 << GC_PAGE_LG2) // 16k +#define GC_PAGE_OFFSET (JL_SMALL_BYTE_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_SMALL_BYTE_ALIGNMENT)) +#define GC_MAX_SZCLASS (2032-sizeof(void*)) + +STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, + jl_datatype_t *ty) +{ + const size_t allocsz = sz + sizeof(jl_taggedvalue_t); + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + jl_value_t *v; + if (allocsz <= GC_MAX_SZCLASS + sizeof(jl_taggedvalue_t)) { + int pool_id = jl_gc_szclass(allocsz); + jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; + int osize; + int endoff; + if (jl_is_constexpr(allocsz)) { + osize = jl_gc_sizeclasses[pool_id]; + endoff = GC_POOL_END_OFS(osize); + } + else { + osize = p->osize; + endoff = p->end_offset; + } + v = jl_gc_pool_alloc(ptls, p, osize, endoff); } - jl_set_typeof(jv, type); - return jv; + else { + v = jl_gc_big_alloc(ptls, allocsz); + } + jl_set_typeof(v, ty); + return v; +} +JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, + jl_datatype_t *ty); +// On GCC, only inline when sz is constant +#ifdef __GNUC__ +# define jl_gc_alloc(ptls, sz, ty) \ + (__builtin_constant_p(sz) ? jl_gc_alloc_(ptls, sz, ty) : \ + jl_gc_alloc(ptls, sz, ty)) +#else +# define jl_gc_alloc(ptls, sz) jl_gc_alloc_(ptls, sz, ty) +#endif + +#define jl_buff_tag ((uintptr_t)0x4eade800) +STATIC_INLINE void *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz) +{ + return jl_gc_alloc(ptls, sz, (jl_datatype_t*)jl_buff_tag); } -STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) +STATIC_INLINE jl_value_t *newstruct(jl_ptls_t ptls, jl_datatype_t *type) { - jl_value_t *jv = (jl_value_t*)jl_gc_allocobj(type->size); - jl_set_typeof(jv, type); - return jv; + return jl_gc_alloc(ptls, type->size, type); } jl_lambda_info_t *jl_type_infer(jl_lambda_info_t *li, int force); @@ -100,13 +188,11 @@ jl_tupletype_t *jl_argtype_with_function(jl_function_t *f, jl_tupletype_t *types JL_DLLEXPORT jl_value_t *jl_apply_2va(jl_value_t *f, jl_value_t **args, uint32_t nargs); -#define GC_MAX_SZCLASS (2032-sizeof(void*)) -void jl_gc_setmark(jl_value_t *v); +void jl_gc_setmark(jl_ptls_t ptls, jl_value_t *v); void jl_gc_sync_total_bytes(void); -void jl_gc_track_malloced_array(jl_array_t *a); +void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a); void jl_gc_count_allocd(size_t sz); void jl_gc_run_all_finalizers(jl_ptls_t ptls); -void *allocb(size_t sz); void gc_queue_binding(jl_binding_t *bnd); void gc_setmark_buf(jl_ptls_t ptls, void *buf, int, size_t); @@ -584,10 +670,6 @@ STATIC_INLINE void jl_free_aligned(void *p) } #endif -#define JL_SMALL_BYTE_ALIGNMENT 16 -#define JL_CACHE_BYTE_ALIGNMENT 64 - - // -- typemap.c -- // STATIC_INLINE int is_kind(jl_value_t *v) diff --git a/src/julia_threads.h b/src/julia_threads.h index 48b03f91d93ea..b4d07e797d65d 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -33,7 +33,6 @@ typedef struct { jl_taggedvalue_t *newpages; // root of list of chunks of free objects uint16_t end_offset; // stored to avoid computing it at each allocation uint16_t osize; // size of objects in this pool - uint16_t nfree; // number of free objects in page pointed into by free_list } jl_gc_pool_t; typedef struct { diff --git a/src/module.c b/src/module.c index cdeb130cc8802..77856cf2796ad 100644 --- a/src/module.c +++ b/src/module.c @@ -18,8 +18,9 @@ jl_module_t *jl_top_module=NULL; JL_DLLEXPORT jl_module_t *jl_new_module(jl_sym_t *name) { - jl_module_t *m = (jl_module_t*)jl_gc_allocobj(sizeof(jl_module_t)); - jl_set_typeof(m, jl_module_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_module_t *m = (jl_module_t*)jl_gc_alloc(ptls, sizeof(jl_module_t), + jl_module_type); JL_GC_PUSH1(&m); assert(jl_is_symbol(name)); m->name = name; @@ -70,8 +71,9 @@ JL_DLLEXPORT uint8_t jl_istopmod(jl_module_t *mod) static jl_binding_t *new_binding(jl_sym_t *name) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_symbol(name)); - jl_binding_t *b = (jl_binding_t*)allocb(sizeof(jl_binding_t)); + jl_binding_t *b = (jl_binding_t*)jl_gc_alloc_buf(ptls, sizeof(jl_binding_t)); b->name = name; b->value = NULL; b->owner = NULL; diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 2375a340fa5a6..c7748a140e0b8 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -306,7 +306,8 @@ jl_value_t *jl_iintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); intrinsic_1_t op = select_intrinsic_1(osize2, (const intrinsic_1_t*)voidlist); op(osize * host_char_bit, pa, jl_data_ptr(newv)); return newv; @@ -314,7 +315,8 @@ static inline jl_value_t *jl_intrinsiclambda_ty1(jl_value_t *ty, void *pa, unsig static inline jl_value_t *jl_intrinsiclambda_u1(jl_value_t *ty, void *pa, unsigned osize, unsigned osize2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); intrinsic_u1_t op = select_intrinsic_u1(osize2, (const intrinsic_u1_t*)voidlist); unsigned cnt = op(osize * host_char_bit, pa); // TODO: the following memset/memcpy assumes little-endian @@ -342,6 +344,7 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *ty, jl_value_t *a) \ static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const char *name, intrinsic_cvt_t op, intrinsic_cvt_check_t check_op) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *aty = jl_typeof(a); if (!jl_is_bitstype(aty)) jl_errorf("%s: value is not a bitstype", name); @@ -352,7 +355,7 @@ static inline jl_value_t *jl_intrinsic_cvt(jl_value_t *ty, jl_value_t *a, const unsigned osize = jl_datatype_size(ty); if (check_op && check_op(isize, osize, pa)) jl_throw(jl_inexact_exception); - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); op(aty == (jl_value_t*)jl_bool_type ? 1 : isize * host_char_bit, pa, osize * host_char_bit, jl_data_ptr(newv)); if (ty == (jl_value_t*)jl_bool_type) @@ -381,11 +384,12 @@ typedef void (fintrinsic_op1)(unsigned, void*, void*); static inline jl_value_t *jl_fintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, fintrinsic_op1 *floatop, fintrinsic_op1 *doubleop) { + jl_ptls_t ptls = jl_get_ptls_states(); if (!jl_is_bitstype(jl_typeof(a))) jl_errorf("%s: value is not a bitstype", name); if (!jl_is_bitstype(ty)) jl_errorf("%s: type is not a bitstype", name); - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); unsigned sz = jl_datatype_size(jl_typeof(a)); unsigned sz2 = jl_datatype_size(ty); @@ -517,7 +521,8 @@ jl_value_t *jl_iintrinsic_2(jl_value_t *a, jl_value_t *b, const char *name, static inline jl_value_t *jl_intrinsiclambda_2(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); intrinsic_2_t op = select_intrinsic_2(sz2, (const intrinsic_2_t*)voidlist); op(sz * host_char_bit, pa, pb, jl_data_ptr(newv)); if (ty == (jl_value_t*)jl_bool_type) @@ -534,7 +539,8 @@ static inline jl_value_t *jl_intrinsiclambda_cmp(jl_value_t *ty, void *pa, void static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, void *pb, unsigned sz, unsigned sz2, const void *voidlist) { - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); intrinsic_checked_t op = select_intrinsic_checked(sz2, (const intrinsic_checked_t*)voidlist); int ovflw = op(sz * host_char_bit, pa, pb, jl_data_ptr(newv)); if (ovflw) @@ -551,12 +557,13 @@ static inline jl_value_t *jl_intrinsiclambda_checked(jl_value_t *ty, void *pa, v bi_intrinsic_ctype(OP, name, 64, double) \ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ { \ + jl_ptls_t ptls = jl_get_ptls_states();\ jl_value_t *ty = jl_typeof(a); \ if (jl_typeof(b) != ty) \ jl_error(#name ": types of a and b must match"); \ if (!jl_is_bitstype(ty)) \ jl_error(#name ": values are not bitstypes"); \ - jl_value_t *newv = newstruct((jl_datatype_t*)ty); \ + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); \ void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pr = jl_data_ptr(newv); \ int sz = jl_datatype_size(ty); \ switch (sz) { \ @@ -605,12 +612,13 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ ter_intrinsic_ctype(OP, name, 64, double) \ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c) \ { \ + jl_ptls_t ptls = jl_get_ptls_states();\ jl_value_t *ty = jl_typeof(a); \ if (jl_typeof(b) != ty || jl_typeof(c) != ty) \ jl_error(#name ": types of a, b, and c must match"); \ if (!jl_is_bitstype(ty)) \ jl_error(#name ": values are not bitstypes"); \ - jl_value_t *newv = newstruct((jl_datatype_t*)ty); \ + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); \ void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pc = jl_data_ptr(c), *pr = jl_data_ptr(newv); \ int sz = jl_datatype_size(ty); \ switch (sz) { \ @@ -892,12 +900,13 @@ un_fintrinsic(sqrt_float,sqrt_llvm) JL_DLLEXPORT jl_value_t *jl_powi_llvm(jl_value_t *a, jl_value_t *b) { + jl_ptls_t ptls = jl_get_ptls_states(); jl_value_t *ty = jl_typeof(a); if (!jl_is_bitstype(ty)) jl_error("powi_llvm: a is not a bitstype"); if (!jl_is_bitstype(jl_typeof(b)) || jl_datatype_size(jl_typeof(b)) != 4) jl_error("powi_llvm: b is not a 32-bit bitstype"); - jl_value_t *newv = newstruct((jl_datatype_t*)ty); + jl_value_t *newv = newstruct(ptls, (jl_datatype_t*)ty); void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); int sz = jl_datatype_size(ty); switch (sz) { diff --git a/src/simplevector.c b/src/simplevector.c index 16006831a2af5..9322692059840 100644 --- a/src/simplevector.c +++ b/src/simplevector.c @@ -21,8 +21,9 @@ JL_DLLEXPORT jl_svec_t *jl_svec(size_t n, ...) JL_DLLEXPORT jl_svec_t *jl_svec1(void *a) { - jl_svec_t *v = (jl_svec_t*)jl_gc_alloc_2w(); - jl_set_typeof(v, jl_simplevector_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*) * 2, + jl_simplevector_type); jl_svec_set_len_unsafe(v, 1); jl_svecset(v, 0, a); return v; @@ -30,8 +31,9 @@ JL_DLLEXPORT jl_svec_t *jl_svec1(void *a) JL_DLLEXPORT jl_svec_t *jl_svec2(void *a, void *b) { - jl_svec_t *v = (jl_svec_t*)jl_gc_alloc_3w(); - jl_set_typeof(v, jl_simplevector_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_svec_t *v = (jl_svec_t*)jl_gc_alloc(ptls, sizeof(void*) * 3, + jl_simplevector_type); jl_svec_set_len_unsafe(v, 2); jl_svecset(v, 0, a); jl_svecset(v, 1, b); @@ -40,8 +42,10 @@ JL_DLLEXPORT jl_svec_t *jl_svec2(void *a, void *b) JL_DLLEXPORT jl_svec_t *jl_alloc_svec_uninit(size_t n) { + jl_ptls_t ptls = jl_get_ptls_states(); if (n == 0) return jl_emptysvec; - jl_svec_t *jv = (jl_svec_t*)newobj((jl_value_t*)jl_simplevector_type, n+1); + jl_svec_t *jv = (jl_svec_t*)jl_gc_alloc(ptls, (n + 1) * sizeof(void*), + jl_simplevector_type); jl_svec_set_len_unsafe(jv, n); return jv; } diff --git a/src/task.c b/src/task.c index 5da2b1a7e35b4..bcadd9497f777 100644 --- a/src/task.c +++ b/src/task.c @@ -149,7 +149,7 @@ static void NOINLINE save_stack(jl_ptls_t ptls, jl_task_t *t) size_t nb = (char*)ptls->stackbase - frame_addr; char *buf; if (t->stkbuf == NULL || t->bufsz < nb) { - buf = (char*)allocb(nb); + buf = (char*)jl_gc_alloc_buf(ptls, nb); t->stkbuf = buf; t->bufsz = nb; } @@ -544,8 +544,8 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) { jl_ptls_t ptls = jl_get_ptls_states(); size_t pagesz = jl_page_size; - jl_task_t *t = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); - jl_set_typeof(t, jl_task_type); + jl_task_t *t = (jl_task_t*)jl_gc_alloc(ptls, sizeof(jl_task_t), + jl_task_type); #ifndef COPY_STACKS if (ssize == 0) // unspecified -- pick some default size ssize = 1*1024*1024; // 1M (for now) @@ -575,7 +575,7 @@ JL_DLLEXPORT jl_task_t *jl_new_task(jl_function_t *start, size_t ssize) JL_GC_PUSH1(&t); size_t stkbuf_sz = ssize + pagesz + (pagesz - 1); - char *stk = allocb(stkbuf_sz); + char *stk = (char*)jl_gc_alloc_buf(ptls, stkbuf_sz); t->stkbuf = stk; jl_gc_wb_buf(t, t->stkbuf, stkbuf_sz); stk = (char*)LLT_ALIGN((uintptr_t)stk, pagesz); @@ -658,8 +658,8 @@ void jl_init_tasks(void) void jl_init_root_task(void *stack, size_t ssize) { jl_ptls_t ptls = jl_get_ptls_states(); - ptls->current_task = (jl_task_t*)jl_gc_allocobj(sizeof(jl_task_t)); - jl_set_typeof(ptls->current_task, jl_task_type); + ptls->current_task = (jl_task_t*)jl_gc_alloc(ptls, sizeof(jl_task_t), + jl_task_type); #ifdef COPY_STACKS ptls->current_task->ssize = 0; // size of saved piece ptls->current_task->bufsz = 0; diff --git a/src/typemap.c b/src/typemap.c index 6e0764265219e..10fbd30db9a0f 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -826,8 +826,10 @@ static void jl_typemap_list_insert_sorted(jl_typemap_entry_t **pml, jl_value_t * static jl_typemap_level_t *jl_new_typemap_level(void) { - jl_typemap_level_t *cache = (jl_typemap_level_t*)jl_gc_allocobj(sizeof(jl_typemap_level_t)); - jl_set_typeof(cache, jl_typemap_level_type); + jl_ptls_t ptls = jl_get_ptls_states(); + jl_typemap_level_t *cache = + (jl_typemap_level_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_level_t), + jl_typemap_level_type); cache->key = NULL; cache->linear = (jl_typemap_entry_t*)jl_nothing; cache->any.unknown = jl_nothing; @@ -942,6 +944,7 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par const struct jl_typemap_info *tparams, jl_value_t **overwritten) { + jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_tuple_type(type)); if (!simpletype) { simpletype = (jl_tupletype_t*)jl_nothing; @@ -973,8 +976,9 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par if (overwritten != NULL) *overwritten = NULL; - jl_typemap_entry_t *newrec = (jl_typemap_entry_t*)jl_gc_allocobj(sizeof(jl_typemap_entry_t)); - jl_set_typeof(newrec, jl_typemap_entry_type); + jl_typemap_entry_t *newrec = + (jl_typemap_entry_t*)jl_gc_alloc(ptls, sizeof(jl_typemap_entry_t), + jl_typemap_entry_type); newrec->sig = type; newrec->simplesig = simpletype; newrec->tvars = tvars;