Skip to content

Commit

Permalink
Merge 2bd9348 into 430e5e0
Browse files Browse the repository at this point in the history
  • Loading branch information
@vtjnash
vtjnash committed Jul 15, 2021
2 parents 430e5e0 + 2bd9348 commit cb532d5
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 112 deletions.
2 changes: 1 addition & 1 deletion src/builtins.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -812,7 +812,7 @@ enum jl_memory_order jl_get_atomic_order(jl_sym_t *order, char loading, char sto
{
if (order == not_atomic_sym)
return jl_memory_order_notatomic;
if (order == unordered_sym && (loading || storing))
if (order == unordered_sym && (loading ^ storing))
return jl_memory_order_unordered;
if (order == monotonic_sym && (loading || storing))
return jl_memory_order_monotonic;
Expand Down
28 changes: 25 additions & 3 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1549,10 +1549,25 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
bool needlock, bool issetfield, bool isreplacefield, bool maybe_null_if_boxed)
{
assert(!needlock || parent != nullptr);
jl_cgval_t oldval = rhs;
Type *elty = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, jltype);
if (type_is_ghost(elty))
return oldval;
if (type_is_ghost(elty)) {
if (isStrongerThanMonotonic(Order))
ctx.builder.CreateFence(Order);
if (issetfield) {
return rhs;
}
else if (isreplacefield) {
Value *Success = emit_f_is(ctx, cmp, ghostValue(jltype));
Success = ctx.builder.CreateZExt(Success, T_int8);
jl_cgval_t argv[2] = {ghostValue(jltype), mark_julia_type(ctx, Success, false, jl_bool_type)};
// TODO: do better here
Value *instr = emit_jlcall(ctx, jltuple_func, V_rnull, argv, 2, JLCALL_F_CC);
return mark_julia_type(ctx, instr, true, jl_any_type);
}
else {
return ghostValue(jltype);
}
}
Value *intcast = nullptr;
if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) {
const DataLayout &DL = jl_data_layout;
Expand Down Expand Up @@ -1590,6 +1605,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
BasicBlock *DoneBB = issetfield || (!isreplacefield && !isboxed) ? nullptr : BasicBlock::Create(jl_LLVMContext, "done_xchg", ctx.f);
if (needlock)
emit_lockstate_value(ctx, parent, true);
jl_cgval_t oldval = rhs;
if (issetfield || Order == AtomicOrdering::NotAtomic) {
if (!issetfield) {
instr = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment));
Expand Down Expand Up @@ -3230,6 +3246,12 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx,
}
if (needlock)
emit_lockstate_value(ctx, strct, false);
if (isreplacefield) {
jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)};
// TODO: do better here
Value *instr = emit_jlcall(ctx, jltuple_func, V_rnull, argv, 2, JLCALL_F_CC);
oldval = mark_julia_type(ctx, instr, true, jl_any_type);
}
return oldval;
}
else {
Expand Down
196 changes: 101 additions & 95 deletions src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2883,7 +2883,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,

else if (f == jl_builtin_arrayset && nargs >= 4) {
const jl_cgval_t &ary = argv[2];
const jl_cgval_t &val = argv[3];
jl_cgval_t val = argv[3];
bool indices_ok = true;
for (size_t i = 4; i <= nargs; i++) {
if (argv[i].typ != (jl_value_t*)jl_long_type) {
Expand All @@ -2896,101 +2896,103 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
jl_value_t *ety = jl_tparam0(aty_dt);
jl_value_t *ndp = jl_tparam1(aty_dt);
if (!jl_has_free_typevars(ety) && (jl_is_long(ndp) || nargs == 4)) {
if (jl_subtype(val.typ, ety)) { // TODO: probably should just convert this to a type-assert
size_t elsz = 0, al = 0;
int union_max = jl_islayout_inline(ety, &elsz, &al);
bool isboxed = (union_max == 0);
if (isboxed)
ety = (jl_value_t*)jl_any_type;
jl_value_t *ary_ex = jl_exprarg(ex, 2);
ssize_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : -1;
jl_value_t *boundscheck = argv[1].constant;
emit_typecheck(ctx, argv[1], (jl_value_t*)jl_bool_type, "arrayset");
Value *idx = emit_array_nd_index(ctx, ary, ary_ex, nd, &argv[4], nargs - 3, boundscheck);
if (!isboxed && jl_is_datatype(ety) && jl_datatype_size(ety) == 0) {
// no-op
}
else {
PHINode *data_owner = NULL; // owner object against which the write barrier must check
if (isboxed || (jl_is_datatype(ety) && ((jl_datatype_t*)ety)->layout->npointers > 0)) { // if elements are just bits, don't need a write barrier
Value *aryv = boxed(ctx, ary);
Value *flags = emit_arrayflags(ctx, ary);
// the owner of the data is ary itself except if ary->how == 3
flags = ctx.builder.CreateAnd(flags, 3);
Value *is_owned = ctx.builder.CreateICmpEQ(flags, ConstantInt::get(T_int16, 3));
BasicBlock *curBB = ctx.builder.GetInsertBlock();
BasicBlock *ownedBB = BasicBlock::Create(jl_LLVMContext, "array_owned", ctx.f);
BasicBlock *mergeBB = BasicBlock::Create(jl_LLVMContext, "merge_own", ctx.f);
ctx.builder.CreateCondBr(is_owned, ownedBB, mergeBB);
ctx.builder.SetInsertPoint(ownedBB);
// load owner pointer
Instruction *own_ptr;
if (jl_is_long(ndp)) {
own_ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue,
ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue,
emit_bitcast(ctx, decay_derived(ctx, aryv), T_pprjlvalue),
jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)),
Align(sizeof(void*)));
tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type));
}
else {
own_ptr = ctx.builder.CreateCall(
prepare_call(jlarray_data_owner_func),
{aryv});
}
ctx.builder.CreateBr(mergeBB);
ctx.builder.SetInsertPoint(mergeBB);
data_owner = ctx.builder.CreatePHI(T_prjlvalue, 2);
data_owner->addIncoming(aryv, curBB);
data_owner->addIncoming(own_ptr, ownedBB);
if (!jl_subtype(val.typ, ety)) {
emit_typecheck(ctx, val, ety, "arrayset");
val = update_julia_type(ctx, val, ety);
}
size_t elsz = 0, al = 0;
int union_max = jl_islayout_inline(ety, &elsz, &al);
bool isboxed = (union_max == 0);
if (isboxed)
ety = (jl_value_t*)jl_any_type;
jl_value_t *ary_ex = jl_exprarg(ex, 2);
ssize_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : -1;
jl_value_t *boundscheck = argv[1].constant;
emit_typecheck(ctx, argv[1], (jl_value_t*)jl_bool_type, "arrayset");
Value *idx = emit_array_nd_index(ctx, ary, ary_ex, nd, &argv[4], nargs - 3, boundscheck);
if (!isboxed && jl_is_datatype(ety) && jl_datatype_size(ety) == 0) {
// no-op
}
else {
PHINode *data_owner = NULL; // owner object against which the write barrier must check
if (isboxed || (jl_is_datatype(ety) && ((jl_datatype_t*)ety)->layout->npointers > 0)) { // if elements are just bits, don't need a write barrier
Value *aryv = boxed(ctx, ary);
Value *flags = emit_arrayflags(ctx, ary);
// the owner of the data is ary itself except if ary->how == 3
flags = ctx.builder.CreateAnd(flags, 3);
Value *is_owned = ctx.builder.CreateICmpEQ(flags, ConstantInt::get(T_int16, 3));
BasicBlock *curBB = ctx.builder.GetInsertBlock();
BasicBlock *ownedBB = BasicBlock::Create(jl_LLVMContext, "array_owned", ctx.f);
BasicBlock *mergeBB = BasicBlock::Create(jl_LLVMContext, "merge_own", ctx.f);
ctx.builder.CreateCondBr(is_owned, ownedBB, mergeBB);
ctx.builder.SetInsertPoint(ownedBB);
// load owner pointer
Instruction *own_ptr;
if (jl_is_long(ndp)) {
own_ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue,
ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue,
emit_bitcast(ctx, decay_derived(ctx, aryv), T_pprjlvalue),
jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)),
Align(sizeof(void*)));
tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type));
}
if (!isboxed && jl_is_uniontype(ety)) {
Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (elsz + al - 1) / al);
Value *data = emit_bitcast(ctx, emit_arrayptr(ctx, ary, ary_ex), AT->getPointerTo());
// compute tindex from val
jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety);
Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety);
tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1));
Value *ndims = (nd == -1 ? emit_arrayndims(ctx, ary) : ConstantInt::get(T_int16, nd));
Value *is_vector = ctx.builder.CreateICmpEQ(ndims, ConstantInt::get(T_int16, 1));
Value *offset = emit_arrayoffset(ctx, ary, nd);
Value *selidx_v = ctx.builder.CreateSub(emit_vectormaxsize(ctx, ary), ctx.builder.CreateZExt(offset, T_size));
Value *selidx_m = emit_arraylen(ctx, ary);
Value *selidx = ctx.builder.CreateSelect(is_vector, selidx_v, selidx_m);
Value *ptindex = ctx.builder.CreateInBoundsGEP(AT, data, selidx);
ptindex = emit_bitcast(ctx, ptindex, T_pint8);
ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, offset);
ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, idx);
tbaa_decorate(tbaa_arrayselbyte, ctx.builder.CreateStore(tindex, ptindex));
if (jl_is_datatype(val.typ) && jl_datatype_size(val.typ) == 0) {
// no-op
}
else {
// copy data
Value *addr = ctx.builder.CreateInBoundsGEP(AT, data, idx);
emit_unionmove(ctx, addr, tbaa_arraybuf, val, nullptr);
}
else {
own_ptr = ctx.builder.CreateCall(
prepare_call(jlarray_data_owner_func),
{aryv});
}
ctx.builder.CreateBr(mergeBB);
ctx.builder.SetInsertPoint(mergeBB);
data_owner = ctx.builder.CreatePHI(T_prjlvalue, 2);
data_owner->addIncoming(aryv, curBB);
data_owner->addIncoming(own_ptr, ownedBB);
}
if (!isboxed && jl_is_uniontype(ety)) {
Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (elsz + al - 1) / al);
Value *data = emit_bitcast(ctx, emit_arrayptr(ctx, ary, ary_ex), AT->getPointerTo());
// compute tindex from val
jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety);
Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety);
tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1));
Value *ndims = (nd == -1 ? emit_arrayndims(ctx, ary) : ConstantInt::get(T_int16, nd));
Value *is_vector = ctx.builder.CreateICmpEQ(ndims, ConstantInt::get(T_int16, 1));
Value *offset = emit_arrayoffset(ctx, ary, nd);
Value *selidx_v = ctx.builder.CreateSub(emit_vectormaxsize(ctx, ary), ctx.builder.CreateZExt(offset, T_size));
Value *selidx_m = emit_arraylen(ctx, ary);
Value *selidx = ctx.builder.CreateSelect(is_vector, selidx_v, selidx_m);
Value *ptindex = ctx.builder.CreateInBoundsGEP(AT, data, selidx);
ptindex = emit_bitcast(ctx, ptindex, T_pint8);
ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, offset);
ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, idx);
tbaa_decorate(tbaa_arrayselbyte, ctx.builder.CreateStore(tindex, ptindex));
if (jl_is_datatype(val.typ) && jl_datatype_size(val.typ) == 0) {
// no-op
}
else {
typed_store(ctx,
emit_arrayptr(ctx, ary, ary_ex, isboxed),
idx, val, jl_cgval_t(), ety,
isboxed ? tbaa_ptrarraybuf : tbaa_arraybuf,
ctx.aliasscope,
data_owner,
isboxed,
isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0
isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0
0,
false,
true,
false,
false);
// copy data
Value *addr = ctx.builder.CreateInBoundsGEP(AT, data, idx);
emit_unionmove(ctx, addr, tbaa_arraybuf, val, nullptr);
}
}
*ret = ary;
return true;
else {
typed_store(ctx,
emit_arrayptr(ctx, ary, ary_ex, isboxed),
idx, val, jl_cgval_t(), ety,
isboxed ? tbaa_ptrarraybuf : tbaa_arraybuf,
ctx.aliasscope,
data_owner,
isboxed,
isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0
isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0
0,
false,
true,
false,
false);
}
}
*ret = ary;
return true;
}
}
}
Expand Down Expand Up @@ -3134,21 +3136,21 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
const jl_cgval_t undefval;
const jl_cgval_t &obj = argv[1];
const jl_cgval_t &fld = argv[2];
const jl_cgval_t &val = argv[isreplacefield ? 4 : 3];
jl_cgval_t val = argv[isreplacefield ? 4 : 3];
const jl_cgval_t &cmp = isreplacefield ? argv[3] : undefval;
enum jl_memory_order order = jl_memory_order_notatomic;
const std::string &fname = issetfield ? "setfield!" : isreplacefield ? "replacefield!" : "swapfield!";
if (nargs >= (isreplacefield ? 5 : 4)) {
const jl_cgval_t &ord = argv[isreplacefield ? 5 : 4];
emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type,
issetfield ? "setfield!" : isreplacefield ? "replacefield!" : "swapfield!");
emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname);
if (!ord.constant)
return false;
order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true);
}
enum jl_memory_order fail_order = order;
if (isreplacefield && nargs == 6) {
const jl_cgval_t &ord = argv[6];
emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, "replacefield!");
emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname);
if (!ord.constant)
return false;
fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false);
Expand All @@ -3172,7 +3174,11 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
}
if (idx != -1) {
jl_value_t *ft = jl_svecref(uty->types, idx);
if (!jl_has_free_typevars(ft) && jl_subtype(val.typ, ft)) {
if (!jl_has_free_typevars(ft)) {
if (!jl_subtype(val.typ, ft)) {
emit_typecheck(ctx, val, ft, fname);
val = update_julia_type(ctx, val, ft);
}
// TODO: attempt better codegen for approximate types
bool isboxed = jl_field_isptr(uty, idx);
bool isatomic = jl_field_isatomic(uty, idx);
Expand Down
Loading

0 comments on commit cb532d5

Please sign in to comment.