From cc09f64381a0b28d2c84ea9acf60867820b1d8a5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 14 Jul 2021 15:30:31 -0400 Subject: [PATCH 1/3] atomics: disable unordered RMW ordering This is not a particularly meaningful combination (LLVM dislikes it). --- src/builtins.c | 2 +- test/atomics.jl | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/builtins.c b/src/builtins.c index 7ef93faaa6368..4ef5d5ac5c2df 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -812,7 +812,7 @@ enum jl_memory_order jl_get_atomic_order(jl_sym_t *order, char loading, char sto { if (order == not_atomic_sym) return jl_memory_order_notatomic; - if (order == unordered_sym && (loading || storing)) + if (order == unordered_sym && (loading ^ storing)) return jl_memory_order_unordered; if (order == monotonic_sym && (loading || storing)) return jl_memory_order_monotonic; diff --git a/test/atomics.jl b/test/atomics.jl index e4202b5ce1aea..4c32fc12d87ed 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -184,7 +184,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test getfield(r, :y) === x @test_throws ConcurrencyViolationError("invalid atomic ordering") swapfield!(r, :y, y, :u) - @test_throws ConcurrencyViolationError("swapfield!: non-atomic field cannot be written atomically") swapfield!(r, :y, y, :unordered) + @test_throws ConcurrencyViolationError("invalid atomic ordering") swapfield!(r, :y, y, :unordered) @test_throws ConcurrencyViolationError("swapfield!: non-atomic field cannot be written atomically") swapfield!(r, :y, y, :monotonic) @test_throws ConcurrencyViolationError("swapfield!: non-atomic field cannot be written atomically") swapfield!(r, :y, y, :acquire) @test_throws ConcurrencyViolationError("swapfield!: non-atomic field cannot be written atomically") swapfield!(r, :y, y, :release) @@ -193,7 +193,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test swapfield!(r, :y, y, :not_atomic) === x @test_throws ConcurrencyViolationError("invalid atomic ordering") modifyfield!(r, :y, swap, y, :u) - @test_throws ConcurrencyViolationError("modifyfield!: non-atomic field cannot be written atomically") modifyfield!(r, :y, swap, y, :unordered) + @test_throws ConcurrencyViolationError("invalid atomic ordering") modifyfield!(r, :y, swap, y, :unordered) @test_throws ConcurrencyViolationError("modifyfield!: non-atomic field cannot be written atomically") modifyfield!(r, :y, swap, y, :monotonic) @test_throws ConcurrencyViolationError("modifyfield!: non-atomic field cannot be written atomically") modifyfield!(r, :y, swap, y, :acquire) @test_throws ConcurrencyViolationError("modifyfield!: non-atomic field cannot be written atomically") modifyfield!(r, :y, swap, y, :release) @@ -202,7 +202,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test modifyfield!(r, :y, swap, x, :not_atomic) === (y, x) @test_throws ConcurrencyViolationError("invalid atomic ordering") replacefield!(r, :y, y, y, :u, :not_atomic) - @test_throws ConcurrencyViolationError("replacefield!: non-atomic field cannot be written atomically") replacefield!(r, :y, y, y, :unordered, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replacefield!(r, :y, y, y, :unordered, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: non-atomic field cannot be written atomically") replacefield!(r, :y, y, y, :monotonic, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: non-atomic field cannot be written atomically") replacefield!(r, :y, y, y, :acquire, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: non-atomic field cannot be written atomically") replacefield!(r, :y, y, y, :release, :not_atomic) @@ -223,8 +223,8 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test_throws ConcurrencyViolationError("invalid atomic ordering") swapfield!(r, :x, x, :u) @test_throws ConcurrencyViolationError("swapfield!: atomic field cannot be written non-atomically") swapfield!(r, :x, x, :not_atomic) @test_throws ConcurrencyViolationError("swapfield!: atomic field cannot be written non-atomically") swapfield!(r, :x, x) - @test swapfield!(r, :x, x, :unordered) === y - @test swapfield!(r, :x, x, :monotonic) === x + @test_throws ConcurrencyViolationError("invalid atomic ordering") swapfield!(r, :x, x, :unordered) === y + @test swapfield!(r, :x, x, :monotonic) === y @test swapfield!(r, :x, x, :acquire) === x @test swapfield!(r, :x, x, :release) === x @test swapfield!(r, :x, x, :acquire_release) === x @@ -233,7 +233,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test_throws ConcurrencyViolationError("invalid atomic ordering") modifyfield!(r, :x, swap, x, :u) @test_throws ConcurrencyViolationError("modifyfield!: atomic field cannot be written non-atomically") modifyfield!(r, :x, swap, x, :not_atomic) @test_throws ConcurrencyViolationError("modifyfield!: atomic field cannot be written non-atomically") modifyfield!(r, :x, swap, x) - @test modifyfield!(r, :x, swap, x, :unordered) === (x, x) + @test_throws ConcurrencyViolationError("invalid atomic ordering") modifyfield!(r, :x, swap, x, :unordered) @test modifyfield!(r, :x, swap, x, :monotonic) === (x, x) @test modifyfield!(r, :x, swap, x, :acquire) === (x, x) @test modifyfield!(r, :x, swap, x, :release) === (x, x) @@ -243,7 +243,7 @@ test_field_operators(ARefxy{Float64}(123_10, 123_20)) @test_throws ConcurrencyViolationError("invalid atomic ordering") replacefield!(r, :x, x, x, :u, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: atomic field cannot be written non-atomically") replacefield!(r, :x, x, x) @test_throws ConcurrencyViolationError("replacefield!: atomic field cannot be written non-atomically") replacefield!(r, :x, y, x, :not_atomic, :not_atomic) - @test_throws ConcurrencyViolationError("replacefield!: atomic field cannot be accessed non-atomically") replacefield!(r, :x, x, x, :unordered, :not_atomic) + @test_throws ConcurrencyViolationError("invalid atomic ordering") replacefield!(r, :x, x, x, :unordered, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: atomic field cannot be accessed non-atomically") replacefield!(r, :x, x, x, :monotonic, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: atomic field cannot be accessed non-atomically") replacefield!(r, :x, x, x, :acquire, :not_atomic) @test_throws ConcurrencyViolationError("replacefield!: atomic field cannot be accessed non-atomically") replacefield!(r, :x, x, x, :release, :not_atomic) @@ -288,21 +288,21 @@ end Base.convert(T::Type{<:UndefComplex}, S) = T() @noinline function _test_field_undef(r) r = r[] - T = fieldtype(typeof(r), :x) - x = convert(T, 12345_10) + TT = fieldtype(typeof(r), :x) + x = convert(TT, 12345_10) @test_throws UndefRefError getfield(r, :x) @test_throws UndefRefError getfield(r, :x, :sequentially_consistent) @test_throws UndefRefError modifyfield!(r, :x, add, 1, :sequentially_consistent) - @test_throws (T === Any ? UndefRefError : TypeError) replacefield!(r, :x, 1, 1.0, :sequentially_consistent) + @test_throws (TT === Any ? UndefRefError : TypeError) replacefield!(r, :x, 1, 1.0, :sequentially_consistent) @test_throws UndefRefError replacefield!(r, :x, 1, x, :sequentially_consistent) @test_throws UndefRefError getfield(r, :x, :sequentially_consistent) @test_throws UndefRefError swapfield!(r, :x, x, :sequentially_consistent) @test getfield(r, :x, :sequentially_consistent) === x === getfield(r, :x) nothing end -@noinline function test_field_undef(T) - _test_field_undef(Ref(T())) - _test_field_undef(Ref{Any}(T())) +@noinline function test_field_undef(TT) + _test_field_undef(Ref(TT())) + _test_field_undef(Ref{Any}(TT())) nothing end test_field_undef(ARefxy{BigInt}) From 96e24002975e89224b60e320124453bdec299b59 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 14 Jul 2021 16:22:39 -0400 Subject: [PATCH 2/3] codegen: fix some atomic return types --- src/cgutils.cpp | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 8c7bdad2a5903..4ae6c4b21594a 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -1549,10 +1549,25 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, bool needlock, bool issetfield, bool isreplacefield, bool maybe_null_if_boxed) { assert(!needlock || parent != nullptr); - jl_cgval_t oldval = rhs; Type *elty = isboxed ? T_prjlvalue : julia_type_to_llvm(ctx, jltype); - if (type_is_ghost(elty)) - return oldval; + if (type_is_ghost(elty)) { + if (isStrongerThanMonotonic(Order)) + ctx.builder.CreateFence(Order); + if (issetfield) { + return rhs; + } + else if (isreplacefield) { + Value *Success = emit_f_is(ctx, cmp, ghostValue(jltype)); + Success = ctx.builder.CreateZExt(Success, T_int8); + jl_cgval_t argv[2] = {ghostValue(jltype), mark_julia_type(ctx, Success, false, jl_bool_type)}; + // TODO: do better here + Value *instr = emit_jlcall(ctx, jltuple_func, V_rnull, argv, 2, JLCALL_F_CC); + return mark_julia_type(ctx, instr, true, jl_any_type); + } + else { + return ghostValue(jltype); + } + } Value *intcast = nullptr; if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) { const DataLayout &DL = jl_data_layout; @@ -1590,6 +1605,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, BasicBlock *DoneBB = issetfield || (!isreplacefield && !isboxed) ? nullptr : BasicBlock::Create(jl_LLVMContext, "done_xchg", ctx.f); if (needlock) emit_lockstate_value(ctx, parent, true); + jl_cgval_t oldval = rhs; if (issetfield || Order == AtomicOrdering::NotAtomic) { if (!issetfield) { instr = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); @@ -3230,6 +3246,12 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, } if (needlock) emit_lockstate_value(ctx, strct, false); + if (isreplacefield) { + jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; + // TODO: do better here + Value *instr = emit_jlcall(ctx, jltuple_func, V_rnull, argv, 2, JLCALL_F_CC); + oldval = mark_julia_type(ctx, instr, true, jl_any_type); + } return oldval; } else { From 2bd93485faefb8adc5dc7420a20d575d901ebce3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 14 Jul 2021 15:32:18 -0400 Subject: [PATCH 3/3] codegen: optimize setfield/arrayset with inline isa test --- src/codegen.cpp | 196 +++++++++++++++++++++++++----------------------- 1 file changed, 101 insertions(+), 95 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 68300633f293f..694d0c6e64ddf 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2883,7 +2883,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (f == jl_builtin_arrayset && nargs >= 4) { const jl_cgval_t &ary = argv[2]; - const jl_cgval_t &val = argv[3]; + jl_cgval_t val = argv[3]; bool indices_ok = true; for (size_t i = 4; i <= nargs; i++) { if (argv[i].typ != (jl_value_t*)jl_long_type) { @@ -2896,101 +2896,103 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, jl_value_t *ety = jl_tparam0(aty_dt); jl_value_t *ndp = jl_tparam1(aty_dt); if (!jl_has_free_typevars(ety) && (jl_is_long(ndp) || nargs == 4)) { - if (jl_subtype(val.typ, ety)) { // TODO: probably should just convert this to a type-assert - size_t elsz = 0, al = 0; - int union_max = jl_islayout_inline(ety, &elsz, &al); - bool isboxed = (union_max == 0); - if (isboxed) - ety = (jl_value_t*)jl_any_type; - jl_value_t *ary_ex = jl_exprarg(ex, 2); - ssize_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : -1; - jl_value_t *boundscheck = argv[1].constant; - emit_typecheck(ctx, argv[1], (jl_value_t*)jl_bool_type, "arrayset"); - Value *idx = emit_array_nd_index(ctx, ary, ary_ex, nd, &argv[4], nargs - 3, boundscheck); - if (!isboxed && jl_is_datatype(ety) && jl_datatype_size(ety) == 0) { - // no-op - } - else { - PHINode *data_owner = NULL; // owner object against which the write barrier must check - if (isboxed || (jl_is_datatype(ety) && ((jl_datatype_t*)ety)->layout->npointers > 0)) { // if elements are just bits, don't need a write barrier - Value *aryv = boxed(ctx, ary); - Value *flags = emit_arrayflags(ctx, ary); - // the owner of the data is ary itself except if ary->how == 3 - flags = ctx.builder.CreateAnd(flags, 3); - Value *is_owned = ctx.builder.CreateICmpEQ(flags, ConstantInt::get(T_int16, 3)); - BasicBlock *curBB = ctx.builder.GetInsertBlock(); - BasicBlock *ownedBB = BasicBlock::Create(jl_LLVMContext, "array_owned", ctx.f); - BasicBlock *mergeBB = BasicBlock::Create(jl_LLVMContext, "merge_own", ctx.f); - ctx.builder.CreateCondBr(is_owned, ownedBB, mergeBB); - ctx.builder.SetInsertPoint(ownedBB); - // load owner pointer - Instruction *own_ptr; - if (jl_is_long(ndp)) { - own_ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue, - ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue, - emit_bitcast(ctx, decay_derived(ctx, aryv), T_pprjlvalue), - jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)), - Align(sizeof(void*))); - tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type)); - } - else { - own_ptr = ctx.builder.CreateCall( - prepare_call(jlarray_data_owner_func), - {aryv}); - } - ctx.builder.CreateBr(mergeBB); - ctx.builder.SetInsertPoint(mergeBB); - data_owner = ctx.builder.CreatePHI(T_prjlvalue, 2); - data_owner->addIncoming(aryv, curBB); - data_owner->addIncoming(own_ptr, ownedBB); + if (!jl_subtype(val.typ, ety)) { + emit_typecheck(ctx, val, ety, "arrayset"); + val = update_julia_type(ctx, val, ety); + } + size_t elsz = 0, al = 0; + int union_max = jl_islayout_inline(ety, &elsz, &al); + bool isboxed = (union_max == 0); + if (isboxed) + ety = (jl_value_t*)jl_any_type; + jl_value_t *ary_ex = jl_exprarg(ex, 2); + ssize_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : -1; + jl_value_t *boundscheck = argv[1].constant; + emit_typecheck(ctx, argv[1], (jl_value_t*)jl_bool_type, "arrayset"); + Value *idx = emit_array_nd_index(ctx, ary, ary_ex, nd, &argv[4], nargs - 3, boundscheck); + if (!isboxed && jl_is_datatype(ety) && jl_datatype_size(ety) == 0) { + // no-op + } + else { + PHINode *data_owner = NULL; // owner object against which the write barrier must check + if (isboxed || (jl_is_datatype(ety) && ((jl_datatype_t*)ety)->layout->npointers > 0)) { // if elements are just bits, don't need a write barrier + Value *aryv = boxed(ctx, ary); + Value *flags = emit_arrayflags(ctx, ary); + // the owner of the data is ary itself except if ary->how == 3 + flags = ctx.builder.CreateAnd(flags, 3); + Value *is_owned = ctx.builder.CreateICmpEQ(flags, ConstantInt::get(T_int16, 3)); + BasicBlock *curBB = ctx.builder.GetInsertBlock(); + BasicBlock *ownedBB = BasicBlock::Create(jl_LLVMContext, "array_owned", ctx.f); + BasicBlock *mergeBB = BasicBlock::Create(jl_LLVMContext, "merge_own", ctx.f); + ctx.builder.CreateCondBr(is_owned, ownedBB, mergeBB); + ctx.builder.SetInsertPoint(ownedBB); + // load owner pointer + Instruction *own_ptr; + if (jl_is_long(ndp)) { + own_ptr = ctx.builder.CreateAlignedLoad(T_prjlvalue, + ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue, + emit_bitcast(ctx, decay_derived(ctx, aryv), T_pprjlvalue), + jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)), + Align(sizeof(void*))); + tbaa_decorate(tbaa_const, maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type)); } - if (!isboxed && jl_is_uniontype(ety)) { - Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (elsz + al - 1) / al); - Value *data = emit_bitcast(ctx, emit_arrayptr(ctx, ary, ary_ex), AT->getPointerTo()); - // compute tindex from val - jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety); - Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety); - tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1)); - Value *ndims = (nd == -1 ? emit_arrayndims(ctx, ary) : ConstantInt::get(T_int16, nd)); - Value *is_vector = ctx.builder.CreateICmpEQ(ndims, ConstantInt::get(T_int16, 1)); - Value *offset = emit_arrayoffset(ctx, ary, nd); - Value *selidx_v = ctx.builder.CreateSub(emit_vectormaxsize(ctx, ary), ctx.builder.CreateZExt(offset, T_size)); - Value *selidx_m = emit_arraylen(ctx, ary); - Value *selidx = ctx.builder.CreateSelect(is_vector, selidx_v, selidx_m); - Value *ptindex = ctx.builder.CreateInBoundsGEP(AT, data, selidx); - ptindex = emit_bitcast(ctx, ptindex, T_pint8); - ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, offset); - ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, idx); - tbaa_decorate(tbaa_arrayselbyte, ctx.builder.CreateStore(tindex, ptindex)); - if (jl_is_datatype(val.typ) && jl_datatype_size(val.typ) == 0) { - // no-op - } - else { - // copy data - Value *addr = ctx.builder.CreateInBoundsGEP(AT, data, idx); - emit_unionmove(ctx, addr, tbaa_arraybuf, val, nullptr); - } + else { + own_ptr = ctx.builder.CreateCall( + prepare_call(jlarray_data_owner_func), + {aryv}); + } + ctx.builder.CreateBr(mergeBB); + ctx.builder.SetInsertPoint(mergeBB); + data_owner = ctx.builder.CreatePHI(T_prjlvalue, 2); + data_owner->addIncoming(aryv, curBB); + data_owner->addIncoming(own_ptr, ownedBB); + } + if (!isboxed && jl_is_uniontype(ety)) { + Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (elsz + al - 1) / al); + Value *data = emit_bitcast(ctx, emit_arrayptr(ctx, ary, ary_ex), AT->getPointerTo()); + // compute tindex from val + jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety); + Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety); + tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1)); + Value *ndims = (nd == -1 ? emit_arrayndims(ctx, ary) : ConstantInt::get(T_int16, nd)); + Value *is_vector = ctx.builder.CreateICmpEQ(ndims, ConstantInt::get(T_int16, 1)); + Value *offset = emit_arrayoffset(ctx, ary, nd); + Value *selidx_v = ctx.builder.CreateSub(emit_vectormaxsize(ctx, ary), ctx.builder.CreateZExt(offset, T_size)); + Value *selidx_m = emit_arraylen(ctx, ary); + Value *selidx = ctx.builder.CreateSelect(is_vector, selidx_v, selidx_m); + Value *ptindex = ctx.builder.CreateInBoundsGEP(AT, data, selidx); + ptindex = emit_bitcast(ctx, ptindex, T_pint8); + ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, offset); + ptindex = ctx.builder.CreateInBoundsGEP(T_int8, ptindex, idx); + tbaa_decorate(tbaa_arrayselbyte, ctx.builder.CreateStore(tindex, ptindex)); + if (jl_is_datatype(val.typ) && jl_datatype_size(val.typ) == 0) { + // no-op } else { - typed_store(ctx, - emit_arrayptr(ctx, ary, ary_ex, isboxed), - idx, val, jl_cgval_t(), ety, - isboxed ? tbaa_ptrarraybuf : tbaa_arraybuf, - ctx.aliasscope, - data_owner, - isboxed, - isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - 0, - false, - true, - false, - false); + // copy data + Value *addr = ctx.builder.CreateInBoundsGEP(AT, data, idx); + emit_unionmove(ctx, addr, tbaa_arraybuf, val, nullptr); } } - *ret = ary; - return true; + else { + typed_store(ctx, + emit_arrayptr(ctx, ary, ary_ex, isboxed), + idx, val, jl_cgval_t(), ety, + isboxed ? tbaa_ptrarraybuf : tbaa_arraybuf, + ctx.aliasscope, + data_owner, + isboxed, + isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 + isboxed ? AtomicOrdering::Unordered : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 + 0, + false, + true, + false, + false); + } } + *ret = ary; + return true; } } } @@ -3134,13 +3136,13 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, const jl_cgval_t undefval; const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; - const jl_cgval_t &val = argv[isreplacefield ? 4 : 3]; + jl_cgval_t val = argv[isreplacefield ? 4 : 3]; const jl_cgval_t &cmp = isreplacefield ? argv[3] : undefval; enum jl_memory_order order = jl_memory_order_notatomic; + const std::string &fname = issetfield ? "setfield!" : isreplacefield ? "replacefield!" : "swapfield!"; if (nargs >= (isreplacefield ? 5 : 4)) { const jl_cgval_t &ord = argv[isreplacefield ? 5 : 4]; - emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, - issetfield ? "setfield!" : isreplacefield ? "replacefield!" : "swapfield!"); + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; order = jl_get_atomic_order((jl_sym_t*)ord.constant, !issetfield, true); @@ -3148,7 +3150,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, enum jl_memory_order fail_order = order; if (isreplacefield && nargs == 6) { const jl_cgval_t &ord = argv[6]; - emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, "replacefield!"); + emit_typecheck(ctx, ord, (jl_value_t*)jl_symbol_type, fname); if (!ord.constant) return false; fail_order = jl_get_atomic_order((jl_sym_t*)ord.constant, true, false); @@ -3172,7 +3174,11 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } if (idx != -1) { jl_value_t *ft = jl_svecref(uty->types, idx); - if (!jl_has_free_typevars(ft) && jl_subtype(val.typ, ft)) { + if (!jl_has_free_typevars(ft)) { + if (!jl_subtype(val.typ, ft)) { + emit_typecheck(ctx, val, ft, fname); + val = update_julia_type(ctx, val, ft); + } // TODO: attempt better codegen for approximate types bool isboxed = jl_field_isptr(uty, idx); bool isatomic = jl_field_isatomic(uty, idx);