From 5fd24d28415c9f2b6bcdc2dfab827704519aa0a5 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 5 Apr 2019 15:23:27 -0400 Subject: [PATCH] Implement ImmutableArray This rebases #31630 with several fixed and modifications. After #31630, we had originally decided to hold off on said PR in favor of implementing either more efficient layouts for tuples or some sort of variable-sized struct type. However, in the two years since, neither of those have happened (I had a go at improving tuples and made some progress, but there is much still to be done there). In the meantime, all across the package ecosystem, we've seen an increasing creep of pre-allocation and mutating operations, primarily caused by our lack of sufficiently powerful immutable array abstractions and array optimizations. This works fine for the individual packages in question, but it causes a fair bit of trouble when trying to compose these packages with transformation passes such as AD or domain specific optimizations, since many of those passes do not play well with mutation. More generally, we would like to avoid people needing to pierce abstractions for performance reasons. Given these developments, I think it's getting quite important that we start to seriously look at arrays and try to provide performant and well-optimized arrays in the language. More importantly, I think this is somewhat independent from the actual implementation details. To be sure, it would be nice to move more of the array implementation into Julia by making use of one of the abovementioned langugage features, but that is a bit of an orthogonal concern and not absolutely required. This PR provides an `ImmutableArray` type that is identical in functionality and implementation to `Array`, except that it is immutable. Two new intrinsics `Core.arrayfreeze` and `Core.arraythaw` are provided which are semantically copies and turn a mutable array into an immutable array and vice versa. In the original PR, I additionally provided generic functions `freeze` and `thaw` that would simply forward to these intrinsics. However, said generic functions have been omitted from this PR in favor of simply using constructors to go between mutable and immutable arrays at the high level. Generic `freeze`/`thaw` functions can always be added later, once we have a more complete picture of how these functions would work on non-Array datatypes. Some basic compiler support is provided to elide these copies when the compiler can prove that the original object is dead after the copy. For instance, in the following example: ``` function simple() a = Vector{Float64}(undef, 5) for i = 1:5 a[i] = i end ImmutableArray(a) end ``` the compiler will recognize that the array `a` is dead after its use in `ImmutableArray` and the optimized implementation will simply rewrite the type tag in the originally allocated array to now mark it as immutable. It should be pointed out however, that *semantically* there is still no mutation of the original array, this is simply an optimization. At the moment this compiler transform is rather limited, since the analysis requires escape information in order to compute whether or not the copy may be elided. However, more complete escape analysis is being worked on at the moment, so hopefully this analysis should become more powerful in the very near future. I would like to get this cleaned up and merged resonably quickly, and then crowdsource some improvements to the Array APIs more generally. There are still a number of APIs that are quite bound to the notion of mutable `Array`s. StaticArrays and other packages have been inventing conventions for how to generalize those, but we should form a view in Base what those APIs should look like and harmonize them. Having the `ImmutableArray` in Base should help with that. --- base/array.jl | 28 ++++++++++--- base/compiler/optimize.jl | 2 +- base/compiler/ssair/ir.jl | 7 ++++ base/compiler/ssair/passes.jl | 74 +++++++++++++++++++++++++++++++++ base/compiler/tfuncs.jl | 15 +++++++ base/dict.jl | 2 +- base/experimental.jl | 2 + src/Makefile | 2 +- src/builtin_proto.h | 3 ++ src/builtins.c | 59 +++++++++++++++++++++++++- src/cgutils.cpp | 2 +- src/codegen.cpp | 28 ++++++++++++- src/datatype.c | 7 +++- src/gc.c | 6 ++- src/intrinsics.cpp | 2 +- src/jl_exported_data.inc | 2 + src/jltypes.c | 10 +++++ src/julia.h | 29 ++++++++++--- src/llvm-late-gc-lowering.cpp | 24 +++++++++++ src/llvm-pass-helpers.cpp | 4 +- src/llvm-pass-helpers.h | 1 + src/rtutils.c | 4 +- src/staticdata.c | 11 ++++- test/choosetests.jl | 2 +- test/compiler/immutablearray.jl | 12 ++++++ 25 files changed, 309 insertions(+), 29 deletions(-) create mode 100644 test/compiler/immutablearray.jl diff --git a/base/array.jl b/base/array.jl index 15c354dce6085..f38e2e10c08e2 100644 --- a/base/array.jl +++ b/base/array.jl @@ -147,12 +147,20 @@ function vect(X...) return copyto!(Vector{T}(undef, length(X)), X) end -size(a::Array, d::Integer) = arraysize(a, convert(Int, d)) -size(a::Vector) = (arraysize(a,1),) -size(a::Matrix) = (arraysize(a,1), arraysize(a,2)) -size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(N))::Dims) +const ImmutableArray = Core.ImmutableArray +const IMArray{T,N} = Union{Array{T, N}, ImmutableArray{T,N}} +const IMVector{T} = IMArray{T, 1} +const IMMatrix{T} = IMArray{T, 2} -asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...) +ImmutableArray(a::Array) = Core.arrayfreeze(a) +Array(a::ImmutableArray) = Core.arraythaw(a) + +size(a::IMArray, d::Integer) = arraysize(a, convert(Int, d)) +size(a::IMVector) = (arraysize(a,1),) +size(a::IMMatrix) = (arraysize(a,1), arraysize(a,2)) +size(a::IMArray{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(N))::Dims) + +asize_from(a::IMArray, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...) allocatedinline(T::Type) = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0)) @@ -223,6 +231,13 @@ function isassigned(a::Array, i::Int...) ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 end +function isassigned(a::ImmutableArray, i::Int...) + @_inline_meta + ii = (_sub2ind(size(a), i...) % UInt) - 1 + @boundscheck ii < length(a) % UInt || return false + ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 +end + ## copy ## """ @@ -895,6 +910,9 @@ function getindex end @eval getindex(A::Array, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) @eval getindex(A::Array, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) +@eval getindex(A::ImmutableArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1) +@eval getindex(A::ImmutableArray, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)) + # Faster contiguous indexing using copyto! for UnitRange and Colon function getindex(A::Array, I::AbstractUnitRange{<:Integer}) @_inline_meta diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 349ed52c01529..62f4e32b90aea 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -307,7 +307,7 @@ function run_passes(ci::CodeInfo, sv::OptimizationState) ir = adce_pass!(ir) #@Base.show ("after_adce", ir) @timeit "type lift" ir = type_lift_pass!(ir) - @timeit "compact 3" ir = compact!(ir) + ir = memory_opt!(ir) #@Base.show ir if JLOptions().debug_level == 2 @timeit "verify 3" (verify_ir(ir); verify_linetable(ir.linetable)) diff --git a/base/compiler/ssair/ir.jl b/base/compiler/ssair/ir.jl index bc268d33b1a30..7029b3350b6f7 100644 --- a/base/compiler/ssair/ir.jl +++ b/base/compiler/ssair/ir.jl @@ -319,6 +319,13 @@ function setindex!(x::IRCode, @nospecialize(repl), s::SSAValue) return x end +function ssadominates(ir::IRCode, domtree::DomTree, ssa1::Int, ssa2::Int) + bb1 = block_for_inst(ir.cfg, ssa1) + bb2 = block_for_inst(ir.cfg, ssa2) + bb1 == bb2 && return ssa1 < ssa2 + return dominates(domtree, bb1, bb2) +end + # SSA values that need renaming struct OldSSAValue id::Int diff --git a/base/compiler/ssair/passes.jl b/base/compiler/ssair/passes.jl index 838ab5bd21755..54d4d46b2c5ab 100644 --- a/base/compiler/ssair/passes.jl +++ b/base/compiler/ssair/passes.jl @@ -1255,3 +1255,77 @@ function cfg_simplify!(ir::IRCode) compact.active_result_bb = length(bb_starts) return finish(compact) end + +function is_allocation(stmt) + isexpr(stmt, :foreigncall) || return false + s = stmt.args[1] + isa(s, QuoteNode) && (s = s.value) + return s === :jl_alloc_array_1d +end + +function memory_opt!(ir::IRCode) + compact = IncrementalCompact(ir, false) + uses = IdDict{Int, Vector{Int}}() + relevant = IdSet{Int}() + revisit = Int[] + function mark_val(val) + isa(val, SSAValue) || return + val.id in relevant && pop!(relevant, val.id) + end + for ((_, idx), stmt) in compact + if isa(stmt, ReturnNode) + isdefined(stmt, :val) || continue + val = stmt.val + if isa(val, SSAValue) && val.id in relevant + (haskey(uses, val.id)) || (uses[val.id] = Int[]) + push!(uses[val.id], idx) + end + continue + end + (isexpr(stmt, :call) || isexpr(stmt, :foreigncall)) || continue + if is_allocation(stmt) + push!(relevant, idx) + # TODO: Mark everything else here + continue + end + # TODO: Replace this by interprocedural escape analysis + if is_known_call(stmt, arrayset, compact) + # The value being set escapes, everything else doesn't + mark_val(stmt.args[4]) + arr = stmt.args[3] + if isa(arr, SSAValue) && arr.id in relevant + (haskey(uses, arr.id)) || (uses[arr.id] = Int[]) + push!(uses[arr.id], idx) + end + elseif is_known_call(stmt, Core.arrayfreeze, compact) && isa(stmt.args[2], SSAValue) + push!(revisit, idx) + else + # For now we assume everything escapes + # TODO: We could handle PhiNodes specially and improve this + for ur in userefs(stmt) + mark_val(ur[]) + end + end + end + ir = finish(compact) + isempty(revisit) && return ir + domtree = construct_domtree(ir.cfg.blocks) + for idx in revisit + # Make sure that the value we reference didn't escape + id = ir.stmts[idx][:inst].args[2].id + (id in relevant) || continue + + # We're ok to steal the memory if we don't dominate any uses + ok = true + for use in uses[id] + if ssadominates(ir, domtree, idx, use) + ok = false + break + end + end + ok || continue + + ir.stmts[idx][:inst].args[1] = Core.mutating_arrayfreeze + end + return ir +end diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 772db09417393..0961d28a4d0ec 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -1532,6 +1532,21 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp sv::Union{InferenceState,Nothing}) if f === tuple return tuple_tfunc(argtypes) + elseif f === Core.arrayfreeze || f === Core.arraythaw + if length(argtypes) != 1 + isva && return Any + return Bottom + end + a = widenconst(argtypes[1]) + at = (f === Core.arrayfreeze ? Array : ImmutableArray) + rt = (f === Core.arrayfreeze ? ImmutableArray : Array) + if a <: at + unw = unwrap_unionall(a) + if isa(unw, DataType) + return rewrap_unionall(rt{unw.parameters[1], unw.parameters[2]}, a) + end + end + return rt end if isa(f, IntrinsicFunction) if is_pure_intrinsic_infer(f) && _all(@nospecialize(a) -> isa(a, Const), argtypes) diff --git a/base/dict.jl b/base/dict.jl index 6918677c4f0bb..6d8bea2d33f20 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -372,7 +372,7 @@ end function setindex!(h::Dict{K,V}, v0, key0) where V where K key = convert(K, key0) if !isequal(key, key0) - throw(ArgumentError("$(limitrepr(key0)) is not a valid key for type $K")) + throw(KeyTypeError(K, key0)) end setindex!(h, v0, key) end diff --git a/base/experimental.jl b/base/experimental.jl index 232d2efd11d21..de51c04f56c66 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -11,6 +11,8 @@ module Experimental using Base: Threads, sync_varname using Base.Meta +using Base: ImmutableArray + """ Const(A::Array) diff --git a/src/Makefile b/src/Makefile index c61523a2bacf7..25e8d931d83c9 100644 --- a/src/Makefile +++ b/src/Makefile @@ -261,7 +261,7 @@ $(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/builtin_pro $(BUILDDIR)/jitlayers.o $(BUILDDIR)/jitlayers.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/codegen_shared.h $(BUILDDIR)/jltypes.o $(BUILDDIR)/jltypes.dbg.obj: $(SRCDIR)/builtin_proto.h $(build_shlibdir)/libllvmcalltest.$(SHLIB_EXT): $(SRCDIR)/codegen_shared.h $(BUILDDIR)/julia_version.h -$(BUILDDIR)/llvm-alloc-opt.o $(BUILDDIR)/llvm-alloc-opt.dbg.obj: $(SRCDIR)/codegen_shared.h +$(BUILDDIR)/llvm-alloc-opt.o $(BUILDDIR)/llvm-alloc-opt.dbg.obj: $(SRCDIR)/codegen_shared.h $(SRCDIR)/llvm-pass-helpers.h $(BUILDDIR)/llvm-final-gc-lowering.o $(BUILDDIR)/llvm-final-gc-lowering.dbg.obj: $(SRCDIR)/llvm-pass-helpers.h $(BUILDDIR)/llvm-gc-invariant-verifier.o $(BUILDDIR)/llvm-gc-invariant-verifier.dbg.obj: $(SRCDIR)/codegen_shared.h $(BUILDDIR)/llvm-late-gc-lowering.o $(BUILDDIR)/llvm-late-gc-lowering.dbg.obj: $(SRCDIR)/llvm-pass-helpers.h diff --git a/src/builtin_proto.h b/src/builtin_proto.h index 49d3cd7fe87e1..f781658ed55f9 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -51,6 +51,9 @@ DECLARE_BUILTIN(typeassert); DECLARE_BUILTIN(_typebody); DECLARE_BUILTIN(typeof); DECLARE_BUILTIN(_typevar); +DECLARE_BUILTIN(arrayfreeze); +DECLARE_BUILTIN(arraythaw); +DECLARE_BUILTIN(mutating_arrayfreeze); JL_CALLABLE(jl_f_invoke_kwsorter); JL_CALLABLE(jl_f__structtype); diff --git a/src/builtins.c b/src/builtins.c index 32afff52e0b5f..5a612da44363a 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -1330,7 +1330,9 @@ JL_CALLABLE(jl_f__typevar) JL_CALLABLE(jl_f_arraysize) { JL_NARGS(arraysize, 2, 2); - JL_TYPECHK(arraysize, array, args[0]); + if (!jl_is_arrayish(args[0])) { + jl_type_error("arraysize", (jl_value_t*)jl_array_type, args[0]); + } jl_array_t *a = (jl_array_t*)args[0]; size_t nd = jl_array_ndims(a); JL_TYPECHK(arraysize, long, args[1]); @@ -1369,7 +1371,9 @@ JL_CALLABLE(jl_f_arrayref) { JL_NARGSV(arrayref, 3); JL_TYPECHK(arrayref, bool, args[0]); - JL_TYPECHK(arrayref, array, args[1]); + if (!jl_is_arrayish(args[1])) { + jl_type_error("arrayref", (jl_value_t*)jl_array_type, args[1]); + } jl_array_t *a = (jl_array_t*)args[1]; size_t i = array_nd_index(a, &args[2], nargs - 2, "arrayref"); return jl_arrayref(a, i); @@ -1645,6 +1649,52 @@ JL_CALLABLE(jl_f__equiv_typedef) return equiv_type(args[0], args[1]) ? jl_true : jl_false; } +JL_CALLABLE(jl_f_arrayfreeze) +{ + JL_NARGSV(arrayfreeze, 1); + JL_TYPECHK(arrayfreeze, array, args[0]); + jl_array_t *a = (jl_array_t*)args[0]; + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, + jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a))); + // The idea is to elide this copy if the compiler or runtime can prove that + // doing so is safe to do. + jl_array_t *na = jl_array_copy(a); + jl_set_typeof(na, it); + return (jl_value_t*)na; +} + +JL_CALLABLE(jl_f_mutating_arrayfreeze) +{ + // N.B.: These error checks pretend to be arrayfreeze since this is a drop + // in replacement and we don't want to change the visible error type in the + // optimizer + JL_NARGSV(arrayfreeze, 1); + JL_TYPECHK(arrayfreeze, array, args[0]); + jl_array_t *a = (jl_array_t*)args[0]; + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, + jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a))); + // The idea is to elide this copy if the compiler or runtime can prove that + // doing so is safe to do. + jl_set_typeof(a, it); + return (jl_value_t*)a; +} + +JL_CALLABLE(jl_f_arraythaw) +{ + JL_NARGSV(arraythaw, 1); + if (((jl_datatype_t*)jl_typeof(args[0]))->name != jl_immutable_array_typename) { + jl_type_error("arraythaw", (jl_value_t*)jl_immutable_array_type, args[0]); + } + jl_array_t *a = (jl_array_t*)args[0]; + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_array_type, + jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a))); + // The idea is to elide this copy if the compiler or runtime can prove that + // doing so is safe to do. + jl_array_t *na = jl_array_copy(a); + jl_set_typeof(na, it); + return (jl_value_t*)na; +} + // IntrinsicFunctions --------------------------------------------------------- static void (*runtime_fp[num_intrinsics])(void); @@ -1797,6 +1847,10 @@ void jl_init_primitives(void) JL_GC_DISABLED jl_builtin_arrayset = add_builtin_func("arrayset", jl_f_arrayset); jl_builtin_arraysize = add_builtin_func("arraysize", jl_f_arraysize); + jl_builtin_arrayfreeze = add_builtin_func("arrayfreeze", jl_f_arrayfreeze); + jl_builtin_mutating_arrayfreeze = add_builtin_func("mutating_arrayfreeze", jl_f_mutating_arrayfreeze); + jl_builtin_arraythaw = add_builtin_func("arraythaw", jl_f_arraythaw); + // method table utils jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); jl_builtin_invoke = add_builtin_func("invoke", jl_f_invoke); @@ -1868,6 +1922,7 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("AbstractArray", (jl_value_t*)jl_abstractarray_type); add_builtin("DenseArray", (jl_value_t*)jl_densearray_type); add_builtin("Array", (jl_value_t*)jl_array_type); + add_builtin("ImmutableArray", (jl_value_t*)jl_immutable_array_type); add_builtin("Expr", (jl_value_t*)jl_expr_type); add_builtin("LineNumberNode", (jl_value_t*)jl_linenumbernode_type); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 94ac4c071770e..f5314863e94f9 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -500,7 +500,7 @@ static Type *_julia_type_to_llvm(jl_codegen_params_t *ctx, jl_value_t *jt, bool if (isboxed) *isboxed = false; if (jt == (jl_value_t*)jl_bottom_type) return T_void; - if (jl_is_concrete_immutable(jt)) { + if (jl_is_concrete_immutable(jt) && !jl_is_arrayish_type(jt)) { if (jl_datatype_nbits(jt) == 0) return T_void; Type *t = _julia_struct_to_llvm(ctx, jt, isboxed); diff --git a/src/codegen.cpp b/src/codegen.cpp index ba583799e1c97..8143f90d5ef6d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -860,6 +860,15 @@ static const auto pointer_from_objref_func = new JuliaFunction{ Attributes(C, {Attribute::NonNull}), None); }, }; +static const auto mutating_arrayfreeze_func = new JuliaFunction{ + "julia.mutating_arrayfreeze", + [](LLVMContext &C) { return FunctionType::get(T_prjlvalue, + {T_prjlvalue}, false); }, + [](LLVMContext &C) { return AttributeList::get(C, + Attributes(C, {Attribute::NoUnwind, Attribute::NoRecurse}), + Attributes(C, {Attribute::NonNull}), + None); }, +}; static const auto jltuple_func = new JuliaFunction{"jl_f_tuple", get_func_sig, get_func_attrs}; static const std::map builtin_func_map = { @@ -894,6 +903,9 @@ static const std::map builtin_func_map = { { &jl_f_arrayset, new JuliaFunction{"jl_f_arrayset", get_func_sig, get_func_attrs} }, { &jl_f_arraysize, new JuliaFunction{"jl_f_arraysize", get_func_sig, get_func_attrs} }, { &jl_f_apply_type, new JuliaFunction{"jl_f_apply_type", get_func_sig, get_func_attrs} }, + { &jl_f_arrayfreeze, new JuliaFunction{"jl_f_arrayfreeze", get_func_sig, get_func_attrs} }, + { &jl_f_arraythaw, new JuliaFunction{"jl_f_arraythaw", get_func_sig, get_func_attrs} }, + { &jl_f_mutating_arrayfreeze,new JuliaFunction{"jl_f_mutating_arrayfreeze", get_func_sig, get_func_attrs} }, }; static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction{"jl_new_opaque_closure_jlcall", get_func_sig, get_func_attrs}; @@ -969,7 +981,7 @@ static bool deserves_retbox(jl_value_t* t) static bool deserves_sret(jl_value_t *dt, Type *T) { assert(jl_is_datatype(dt)); - return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy(); + return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy() && !jl_is_arrayish_type(dt); } @@ -2887,6 +2899,20 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } } + else if (f == jl_builtin_mutating_arrayfreeze && nargs == 1) { + const jl_cgval_t &ary = argv[1]; + jl_value_t *aty_dt = jl_unwrap_unionall(ary.typ); + if (jl_is_array_type(aty_dt)) { + jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type, + jl_tparam0(aty_dt), jl_tparam1(aty_dt)); + *ret = mark_julia_type(ctx, + ctx.builder.CreateCall(prepare_call(mutating_arrayfreeze_func), + { boxed(ctx, ary), literal_pointer_val(ctx, (jl_value_t*)it) }), true, it); + return true; + } + return false; + } + else if (f == jl_builtin_arrayset && nargs >= 4) { const jl_cgval_t &ary = argv[2]; jl_cgval_t val = argv[3]; diff --git a/src/datatype.c b/src/datatype.c index 1a3ffa78170ac..f751d744dff1c 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -222,7 +222,8 @@ unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) STATIC_INLINE int jl_is_datatype_make_singleton(jl_datatype_t *d) JL_NOTSAFEPOINT { - return (!d->name->abstract && jl_datatype_size(d) == 0 && d != jl_symbol_type && d->name != jl_array_typename && + return (!d->name->abstract && jl_datatype_size(d) == 0 && d != jl_symbol_type && + d->name != jl_array_typename && d->name != jl_immutable_array_typename && d->isconcretetype && !d->name->mutabl); } @@ -389,7 +390,9 @@ void jl_compute_field_offsets(jl_datatype_t *st) st->layout = &opaque_byte_layout; return; } - else if (st == jl_simplevector_type || st == jl_module_type || st->name == jl_array_typename) { + else if (st == jl_simplevector_type || st == jl_module_type || + st->name == jl_array_typename || + st->name == jl_immutable_array_typename) { static const jl_datatype_layout_t opaque_ptr_layout = {0, 1, -1, sizeof(void*), 0, 0}; st->layout = &opaque_ptr_layout; return; diff --git a/src/gc.c b/src/gc.c index f923b826de544..84b1cf695a4cd 100644 --- a/src/gc.c +++ b/src/gc.c @@ -856,7 +856,8 @@ void jl_gc_force_mark_old(jl_ptls_t ptls, jl_value_t *v) JL_NOTSAFEPOINT size_t l = jl_svec_len(v); dtsz = l * sizeof(void*) + sizeof(jl_svec_t); } - else if (dt->name == jl_array_typename) { + else if (dt->name == jl_array_typename || + dt->name == jl_immutable_array_typename) { jl_array_t *a = (jl_array_t*)v; if (!a->flags.pooled) dtsz = GC_MAX_SZCLASS + 1; @@ -2520,7 +2521,8 @@ mark: { objary = (gc_mark_objarray_t*)sp.data; goto objarray_loaded; } - else if (vt->name == jl_array_typename) { + else if (vt->name == jl_array_typename || + vt->name == jl_immutable_array_typename) { jl_array_t *a = (jl_array_t*)new_obj; jl_array_flags_t flags = a->flags; if (update_meta) { diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index e1d821a34e42d..2c1b8a55754a3 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1073,7 +1073,7 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar case arraylen: { const jl_cgval_t &x = argv[0]; jl_value_t *typ = jl_unwrap_unionall(x.typ); - if (!jl_is_datatype(typ) || ((jl_datatype_t*)typ)->name != jl_array_typename) + if (!jl_is_arrayish_type(typ)) return emit_runtime_call(ctx, f, argv, nargs); return mark_julia_type(ctx, emit_arraylen(ctx, x), false, jl_long_type); } diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 3cebe459bf643..32dd6cb5079e4 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -15,6 +15,8 @@ XX(jl_array_symbol_type) \ XX(jl_array_type) \ XX(jl_array_typename) \ + XX(jl_immutable_array_type) \ + XX(jl_immutable_array_typename) \ XX(jl_array_uint8_type) \ XX(jl_atomicerror_type) \ XX(jl_base_module) \ diff --git a/src/jltypes.c b/src/jltypes.c index 1ae49c0a32eab..f0efc0e37ce42 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2218,6 +2218,15 @@ void jl_init_types(void) JL_GC_DISABLED jl_nonfunction_mt->leafcache = (jl_array_t*)jl_an_empty_vec_any; jl_type_type_mt->leafcache = (jl_array_t*)jl_an_empty_vec_any; + tv = jl_svec2(tvar("T"), tvar("N")); + jl_immutable_array_type = (jl_unionall_t*) + jl_new_datatype(jl_symbol("ImmutableArray"), core, + (jl_datatype_t*) + jl_apply_type((jl_value_t*)jl_densearray_type, jl_svec_data(tv), 2), + tv, jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 0, 0)->name->wrapper; + jl_immutable_array_typename = ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_immutable_array_type))->name; + jl_compute_field_offsets((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_immutable_array_type)); + jl_expr_type = jl_new_datatype(jl_symbol("Expr"), core, jl_any_type, jl_emptysvec, @@ -2629,6 +2638,7 @@ void jl_init_types(void) JL_GC_DISABLED // override the preferred layout for a couple types jl_lineinfonode_type->name->mayinlinealloc = 0; // FIXME: assumed to be a pointer by codegen + jl_immutable_array_typename->mayinlinealloc = 0; // It seems like we probably usually end up needing the box for kinds (used in an Any context)--but is that true? jl_uniontype_type->name->mayinlinealloc = 0; jl_unionall_type->name->mayinlinealloc = 0; diff --git a/src/julia.h b/src/julia.h index 9afd7301fc5bc..292083225b30e 100644 --- a/src/julia.h +++ b/src/julia.h @@ -657,6 +657,8 @@ extern JL_DLLIMPORT jl_unionall_t *jl_abstractarray_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_unionall_t *jl_densearray_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_unionall_t *jl_array_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_typename_t *jl_array_typename JL_GLOBALLY_ROOTED; +extern JL_DLLEXPORT jl_unionall_t *jl_immutable_array_type JL_GLOBALLY_ROOTED; +extern JL_DLLEXPORT jl_typename_t *jl_immutable_array_typename JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_weakref_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_abstractstring_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_string_type JL_GLOBALLY_ROOTED; @@ -1200,11 +1202,25 @@ STATIC_INLINE int jl_is_primitivetype(void *v) JL_NOTSAFEPOINT jl_datatype_size(v) > 0); } +STATIC_INLINE int jl_is_array_type(void *t) JL_NOTSAFEPOINT +{ + return (jl_is_datatype(t) && + (((jl_datatype_t*)(t))->name == jl_array_typename)); +} + +STATIC_INLINE int jl_is_arrayish_type(void *t) JL_NOTSAFEPOINT +{ + return (jl_is_datatype(t) && + (((jl_datatype_t*)(t))->name == jl_array_typename || + ((jl_datatype_t*)(t))->name == jl_immutable_array_typename)); +} + STATIC_INLINE int jl_is_structtype(void *v) JL_NOTSAFEPOINT { return (jl_is_datatype(v) && !((jl_datatype_t*)(v))->name->abstract && - !jl_is_primitivetype(v)); + !jl_is_primitivetype(v) && + !jl_is_arrayish_type(v)); } STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbits() in julia @@ -1222,16 +1238,16 @@ STATIC_INLINE int jl_is_abstracttype(void *v) JL_NOTSAFEPOINT return (jl_is_datatype(v) && ((jl_datatype_t*)(v))->name->abstract); } -STATIC_INLINE int jl_is_array_type(void *t) JL_NOTSAFEPOINT +STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT { - return (jl_is_datatype(t) && - ((jl_datatype_t*)(t))->name == jl_array_typename); + jl_value_t *t = jl_typeof(v); + return jl_is_array_type(t); } -STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT +STATIC_INLINE int jl_is_arrayish(void *v) JL_NOTSAFEPOINT { jl_value_t *t = jl_typeof(v); - return jl_is_array_type(t); + return jl_is_arrayish_type(t); } @@ -1500,6 +1516,7 @@ JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a); JL_DLLEXPORT jl_array_t *jl_alloc_vec_any(size_t n); JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i); // 0-indexed JL_DLLEXPORT jl_value_t *jl_ptrarrayref(jl_array_t *a JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; // 0-indexed +JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary); JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i); // 0-indexed JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i); // 0-indexed JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i); // 0-indexed diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index d8ad3d62d4cc1..f8a450e1f49ca 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -359,6 +359,7 @@ struct LateLowerGCFrame: public FunctionPass, private JuliaPassContext { void RefineLiveSet(BitVector &LS, State &S, const std::vector &CalleeRoots); Value *EmitTagPtr(IRBuilder<> &builder, Type *T, Value *V); Value *EmitLoadTag(IRBuilder<> &builder, Value *V); + Value *EmitStoreTag(IRBuilder<> &builder, Value *V, Value *Typ); }; static unsigned getValueAddrSpace(Value *V) { @@ -2186,6 +2187,16 @@ Value *LateLowerGCFrame::EmitLoadTag(IRBuilder<> &builder, Value *V) return load; } +Value *LateLowerGCFrame::EmitStoreTag(IRBuilder<> &builder, Value *V, Value *Typ) +{ + auto addr = EmitTagPtr(builder, T_size, V); + StoreInst *store = builder.CreateAlignedStore(Typ, addr, Align(sizeof(size_t))); + store->setOrdering(AtomicOrdering::Unordered); + store->setMetadata(LLVMContext::MD_tbaa, tbaa_tag); + return store; +} + + // Enable this optimization only on LLVM 4.0+ since this cause LLVM to optimize // constant store loop to produce a `memset_pattern16` with a global variable // that's initialized by `addrspacecast`. Such a global variable is not supported by the backend. @@ -2358,6 +2369,19 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S) { typ->takeName(CI); CI->replaceAllUsesWith(typ); UpdatePtrNumbering(CI, typ, S); + } else if (mutating_arrayfreeze_func && callee == mutating_arrayfreeze_func) { + assert(CI->getNumArgOperands() == 1); + IRBuilder<> builder(CI); + builder.SetCurrentDebugLocation(CI->getDebugLoc()); + auto array = CI->getArgOperand(0); + auto tag = EmitLoadTag(builder, array); + auto mark_bits = builder.CreateAnd(tag, ConstantInt::get(T_size, (uintptr_t)15)); + auto new_typ = builder.CreateAddrSpaceCast(CI->getArgOperand(1), + T_pjlvalue); + auto new_typ_marked = builder.CreateOr(builder.CreatePtrToInt(new_typ, T_size), mark_bits); + EmitStoreTag(builder, array, new_typ_marked); + CI->replaceAllUsesWith(array); + UpdatePtrNumbering(CI, array, S); } else if (write_barrier_func && callee == write_barrier_func) { // The replacement for this requires creating new BasicBlocks // which messes up the loop. Queue all of them to be replaced later. diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 0eed7aec98f0b..eea7b072284ff 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -27,7 +27,8 @@ JuliaPassContext::JuliaPassContext() T_ppjlvalue_der(nullptr), pgcstack_getter(nullptr), gc_flush_func(nullptr), gc_preserve_begin_func(nullptr), gc_preserve_end_func(nullptr), pointer_from_objref_func(nullptr), alloc_obj_func(nullptr), - typeof_func(nullptr), write_barrier_func(nullptr), module(nullptr) + typeof_func(nullptr), mutating_arrayfreeze_func(nullptr), + write_barrier_func(nullptr), module(nullptr) { tbaa_gcframe = tbaa_make_child("jtbaa_gcframe").first; MDNode *tbaa_data; @@ -46,6 +47,7 @@ void JuliaPassContext::initFunctions(Module &M) gc_preserve_end_func = M.getFunction("llvm.julia.gc_preserve_end"); pointer_from_objref_func = M.getFunction("julia.pointer_from_objref"); typeof_func = M.getFunction("julia.typeof"); + mutating_arrayfreeze_func = M.getFunction("julia.mutating_arrayfreeze"); write_barrier_func = M.getFunction("julia.write_barrier"); alloc_obj_func = M.getFunction("julia.gc_alloc_obj"); } diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index f80786d1e7149..9352d01e2fbe9 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -67,6 +67,7 @@ struct JuliaPassContext { llvm::Function *pointer_from_objref_func; llvm::Function *alloc_obj_func; llvm::Function *typeof_func; + llvm::Function *mutating_arrayfreeze_func; llvm::Function *write_barrier_func; // Creates a pass context. Type and function pointers diff --git a/src/rtutils.c b/src/rtutils.c index 67d17c39c67ec..793a5757b10c4 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -988,8 +988,8 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, ")"); } } - else if (jl_array_type && jl_is_array_type(vt)) { - n += jl_printf(out, "Array{"); + else if (jl_array_type && jl_is_arrayish_type(vt)) { + n += jl_printf(out, jl_is_array_type(vt) ? "Array{" : "ImmutableArray{"); n += jl_static_show_x(out, (jl_value_t*)jl_tparam0(vt), depth); n += jl_printf(out, ", ("); size_t i, ndims = jl_array_ndims(v); diff --git a/src/staticdata.c b/src/staticdata.c index 8fa1613b075a8..53e83a0aa2df9 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -30,7 +30,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 150 +#define NUM_TAGS 155 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -50,6 +50,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_slotnumber_type); INSERT_TAG(jl_simplevector_type); INSERT_TAG(jl_array_type); + INSERT_TAG(jl_immutable_array_type); INSERT_TAG(jl_typedslot_type); INSERT_TAG(jl_expr_type); INSERT_TAG(jl_globalref_type); @@ -133,6 +134,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_pointer_typename); INSERT_TAG(jl_llvmpointer_typename); INSERT_TAG(jl_array_typename); + INSERT_TAG(jl_immutable_array_typename); INSERT_TAG(jl_type_typename); INSERT_TAG(jl_namedtuple_typename); INSERT_TAG(jl_vecelement_typename); @@ -200,6 +202,9 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin__expr); INSERT_TAG(jl_builtin_ifelse); INSERT_TAG(jl_builtin__typebody); + INSERT_TAG(jl_builtin_arrayfreeze); + INSERT_TAG(jl_builtin_mutating_arrayfreeze); + INSERT_TAG(jl_builtin_arraythaw); // All optional tags must be placed at the end, so that we // don't accidentally have a `NULL` in the middle @@ -250,7 +255,9 @@ static const jl_fptr_args_t id_to_fptrs[] = { &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f_opaque_closure_call, - NULL }; + &jl_f_arrayfreeze, &jl_f_mutating_arrayfreeze, &jl_f_arraythaw, + NULL +}; typedef struct { ios_t *s; diff --git a/test/choosetests.jl b/test/choosetests.jl index 21f313fdbbb34..7904df7d405b2 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -100,7 +100,7 @@ function choosetests(choices = []) filtertests!(tests, "subarray") filtertests!(tests, "compiler", ["compiler/inference", "compiler/validation", "compiler/ssair", "compiler/irpasses", "compiler/codegen", - "compiler/inline", "compiler/contextual"]) + "compiler/inline", "compiler/contextual", "compiler/immutablearray"]) filtertests!(tests, "stdlib", STDLIBS) # do ambiguous first to avoid failing if ambiguities are introduced by other tests filtertests!(tests, "ambiguous") diff --git a/test/compiler/immutablearray.jl b/test/compiler/immutablearray.jl new file mode 100644 index 0000000000000..474e5dfc0f657 --- /dev/null +++ b/test/compiler/immutablearray.jl @@ -0,0 +1,12 @@ +using Base.Experimental: ImmutableArray +function simple() + a = Vector{Float64}(undef, 5) + for i = 1:5 + a[i] = i + end + ImmutableArray(a) +end +let + @allocated(simple()) + @test @allocated(simple()) < 100 +end