diff --git a/src/array.c b/src/array.c index 7d04711ecca82..0e11272f0a770 100644 --- a/src/array.c +++ b/src/array.c @@ -505,7 +505,11 @@ void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i) } else { ((jl_value_t**)a->data)[i] = rhs; - gc_wb(a, rhs); + jl_value_t *owner = (jl_value_t*)a; + if (a->how == 3) { + owner = jl_array_data_owner(a); + } + gc_wb(owner, rhs); } } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index e0640131658df..37c0397edc309 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1381,6 +1381,16 @@ static Value *emit_arraysize(Value *t, jl_value_t *ex, int dim, jl_codectx_t *ct return emit_arraysize(t, dim); } +static Value *emit_arrayflags(Value *t, jl_codectx_t *ctx) +{ + Value *addr = builder.CreateStructGEP( +#ifdef LLVM37 + nullptr, +#endif + builder.CreateBitCast(t,jl_parray_llvmt), 2); + return builder.CreateLoad(addr); // TODO tbaa +} + static void assign_arrayvar(jl_arrayvar_t &av, Value *ar) { tbaa_decorate(tbaa_arrayptr,builder.CreateStore(builder.CreateBitCast(emit_arrayptr(ar), diff --git a/src/codegen.cpp b/src/codegen.cpp index 69cf58a2c814f..69e875bcdd901 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2181,16 +2181,38 @@ static Value *emit_known_call(jl_value_t *ff, jl_value_t **args, size_t nargs, Value *ary = emit_expr(args[1], ctx); size_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : 1; Value *idx = emit_array_nd_index(ary, args[1], nd, &args[3], nargs-2, ctx); - if (jl_array_store_unboxed(ety) && - ((jl_datatype_t*)ety)->size == 0) { + bool isboxed = !jl_array_store_unboxed(ety); + if (!isboxed && ((jl_datatype_t*)ety)->size == 0) { // no-op, but emit expr for possible effects assert(jl_is_datatype(ety)); emit_expr(args[2],ctx,false); } else { Value* v = ety==(jl_value_t*)jl_any_type ? emit_expr(args[2],ctx) : emit_unboxed(args[2],ctx); + PHINode* data_owner = NULL; // owner object against which the write barrier must check + if (isboxed) { // if not boxed we don't need a write barrier + Value *flags = emit_arrayflags(ary,ctx); + // the owner of the data is ary itself except if ary->how == 3 + flags = builder.CreateAnd(flags, 3); + Value *is_owned = builder.CreateICmpEQ(flags, ConstantInt::get(T_int16, 3)); + BasicBlock *curBB = builder.GetInsertBlock(); + BasicBlock *ownedBB = BasicBlock::Create(getGlobalContext(), "array_owned", ctx->f); + BasicBlock *mergeBB = BasicBlock::Create(getGlobalContext(), "merge_own", ctx->f); + builder.CreateCondBr(is_owned, ownedBB, mergeBB); + builder.SetInsertPoint(ownedBB); + // load owner pointer + Value *own_ptr = builder.CreateLoad( + builder.CreateBitCast(builder.CreateConstGEP1_32( + builder.CreateBitCast(ary,T_pint8), jl_array_data_owner_offset(nd)), + jl_ppvalue_llvmt)); + builder.CreateBr(mergeBB); + builder.SetInsertPoint(mergeBB); + data_owner = builder.CreatePHI(jl_pvalue_llvmt, 2); + data_owner->addIncoming(ary, curBB); + data_owner->addIncoming(own_ptr, ownedBB); + } typed_store(emit_arrayptr(ary,args[1],ctx), idx, v, - ety, ctx, tbaa_user, ety == (jl_value_t*)jl_any_type ? ary : NULL); + ety, ctx, tbaa_user, data_owner); } JL_GC_POP(); return ary; @@ -4834,6 +4856,7 @@ static void init_julia_llvm_env(Module *m) #ifdef STORE_ARRAY_LEN , T_size #endif + , T_int16 }; Type* jl_array_llvmt = StructType::create(jl_LLVMContext, diff --git a/src/gc.c b/src/gc.c index ce08263cc2a90..fac43ed605fcb 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2036,7 +2036,7 @@ static void gc_verify(void) jl_printf(JL_STDERR, "val : "); jl_(jl_valueof(v)); jl_printf(JL_STDERR, "Let's try to backtrack the missing write barrier :\n"); - lostval = v; + lostval = jl_valueof(v); break; } } diff --git a/src/julia.h b/src/julia.h index 81af5c6bb33a8..d55528f51b6bd 100644 --- a/src/julia.h +++ b/src/julia.h @@ -145,23 +145,26 @@ typedef struct { #ifdef STORE_ARRAY_LEN size_t length; #endif - - unsigned short ndims:10; - unsigned short pooled:1; - unsigned short ptrarray:1; // representation is pointer array - /* - how - allocation style - 0 = data is inlined, or a foreign pointer we don't manage - 1 = julia-allocated buffer that needs to be marked - 2 = malloc-allocated pointer this array object manages - 3 = has a pointer to the Array that owns the data - */ - unsigned short how:2; - unsigned short isshared:1; // data is shared by multiple Arrays - unsigned short isaligned:1; // data allocated with memalign + union { + struct { + /* + how - allocation style + 0 = data is inlined, or a foreign pointer we don't manage + 1 = julia-allocated buffer that needs to be marked + 2 = malloc-allocated pointer this array object manages + 3 = has a pointer to the Array that owns the data + */ + unsigned short how:2; + unsigned short ndims:10; + unsigned short pooled:1; + unsigned short ptrarray:1; // representation is pointer array + unsigned short isshared:1; // data is shared by multiple Arrays + unsigned short isaligned:1; // data allocated with memalign + }; + unsigned short flags; + }; uint16_t elsize; uint32_t offset; // for 1-d only. does not need to get big. - size_t nrows; union { // 1d @@ -677,7 +680,8 @@ DLLEXPORT size_t jl_array_len_(jl_array_t *a); #define jl_array_dim0(a) (((jl_array_t*)(a))->nrows) #define jl_array_nrows(a) (((jl_array_t*)(a))->nrows) #define jl_array_ndims(a) ((int32_t)(((jl_array_t*)a)->ndims)) -#define jl_array_data_owner(a) (*((jl_value_t**)(&a->ncols+1+jl_array_ndimwords(jl_array_ndims(a))))) +#define jl_array_data_owner_offset(ndims) (offsetof(jl_array_t,ncols) + sizeof(size_t)*(1+jl_array_ndimwords(ndims))) // in bytes +#define jl_array_data_owner(a) (*((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a))))) STATIC_INLINE jl_value_t *jl_cellref(void *a, size_t i) { @@ -688,7 +692,12 @@ STATIC_INLINE jl_value_t *jl_cellset(void *a, size_t i, void *x) { assert(i < jl_array_len(a)); ((jl_value_t**)(jl_array_data(a)))[i] = (jl_value_t*)x; - if (x) gc_wb(a, x); + if (x) { + if (((jl_array_t*)a)->how == 3) { + a = jl_array_data_owner(a); + } + gc_wb(a, x); + } return (jl_value_t*)x; }