Merge 678e2ad into 21529a9

base/boot.jl

@@ -425,13 +425,14 @@ eval(Core, quote
     function CodeInstance(
         mi::MethodInstance, @nospecialize(rettype), @nospecialize(inferred_const),
         @nospecialize(inferred), const_flags::Int32, min_world::UInt, max_world::UInt,
-        ipo_effects::UInt32, effects::UInt32, @nospecialize(argescapes#=::Union{Nothing,Vector{ArgEscapeInfo}}=#),
+        ipo_effects::UInt32, effects::UInt32,
+        @nospecialize(overrides#=::Union{Nothing,Vector{SSAValueTypeOverride}}=#),
+        @nospecialize(argescapes#=::Union{Nothing,Vector{ArgEscapeInfo}}=#),
         relocatability::UInt8)
         return ccall(:jl_new_codeinst, Ref{CodeInstance},
-            (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, UInt32, Any, UInt8),
+            (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, UInt32, Any, Any, UInt8),
             mi, rettype, inferred_const, inferred, const_flags, min_world, max_world,
-            ipo_effects, effects, argescapes,
-            relocatability)
+            ipo_effects, effects, overrides, argescapes, relocatability)
     end
     Const(@nospecialize(v)) = $(Expr(:new, :Const, :v))
     # NOTE the main constructor is defined within `Core.Compiler`

base/compiler/optimize.jl

@@ -179,18 +179,34 @@ function ir_to_codeinf!(opt::OptimizationState)
     optdef = linfo.def
     replace_code_newstyle!(src, opt.ir::IRCode, isa(optdef, Method) ? Int(optdef.nargs) : 0)
     opt.ir = nothing
-    widen_all_consts!(src)
+    overrides = widen_all_consts!(src)
     src.inferred = true
     # finish updating the result struct
     validate_code_in_debug_mode(linfo, src, "optimized")
-    return src
+    return src, overrides
+end
+
+struct SSAValueTypeOverride
+    idx::Int
+    typ
+    SSAValueTypeOverride(idx::Int, @nospecialize(typ)) = new(idx, typ)
 end
 
 # widen all Const elements in type annotations
 function widen_all_consts!(src::CodeInfo)
+    local overrides = nothing
+
     ssavaluetypes = src.ssavaluetypes::Vector{Any}
     for i = 1:length(ssavaluetypes)
-        ssavaluetypes[i] = widenconst(ssavaluetypes[i])
+        extended = ssavaluetypes[i]
+        widened = widenconst(extended)
+        if widened !== extended
+            ssavaluetypes[i] = widened
+            if overrides === nothing
+                overrides = SSAValueTypeOverride[]
+            end
+            push!(overrides, SSAValueTypeOverride(i, extended))
+        end
     end
 
     for i = 1:length(src.code)
@@ -202,7 +218,7 @@ function widen_all_consts!(src::CodeInfo)
 
     src.rettype = widenconst(src.rettype)
 
-    return src
+    return overrides
 end
 
 #########

base/compiler/ssair/irinterp.jl

@@ -126,6 +126,13 @@ function codeinst_to_ir(interp::AbstractInterpreter, code::CodeInstance)
     else
         isa(src, CodeInfo) || return nothing
     end
+    # override `ssavaluetypes` with extended lattice information
+    overrides = code.overrides
+    if isa(overrides, Vector{SSAValueTypeOverride})
+        for (; idx, typ) = overrides
+            src.ssavaluetypes[idx] = typ
+        end
+    end
     return inflate_ir(src, mi)
 end
 

base/compiler/typeinfer.jl

@@ -223,7 +223,7 @@ function finish!(interp::AbstractInterpreter, caller::InferenceResult)
     if opt isa OptimizationState{typeof(interp)} # implies `may_optimize(interp) === true`
         if opt.ir !== nothing
             if caller.must_be_codeinf
-                caller.src = ir_to_codeinf!(opt)
+                caller.src, caller.overrides = ir_to_codeinf!(opt)
             elseif is_inlineable(opt.src)
                 # TODO: If the CFG is too big, inlining becomes more expensive and if we're going to
                 # use this IR over and over, it's worth simplifying it. Round trips through
@@ -274,7 +274,7 @@ function _typeinf(interp::AbstractInterpreter, frame::InferenceState)
                 # we're doing it is so that code_llvm can return the code
                 # for the `return ...::Const` (which never runs anyway). We should do this
                 # as a post processing step instead.
-                ir_to_codeinf!(opt)
+                _, caller.overrides = ir_to_codeinf!(opt)
                 caller.src = analyzed
             end
             caller.valid_worlds = (opt.inlining.et::EdgeTracker).valid_worlds[]
@@ -336,8 +336,8 @@ function CodeInstance(
         widenconst(result_type), rettype_const, inferred_result,
         const_flags, first(valid_worlds), last(valid_worlds),
         # TODO: Actually do something with non-IPO effects
-	    encode_effects(result.ipo_effects), encode_effects(result.ipo_effects), result.argescapes,
-        relocatability)
+	    encode_effects(result.ipo_effects), encode_effects(result.ipo_effects),
+        result.overrides, result.argescapes, relocatability)
 end
 
 function maybe_compress_codeinfo(interp::AbstractInterpreter, linfo::MethodInstance, ci::CodeInfo)
@@ -368,10 +368,8 @@ end
 function transform_result_for_cache(interp::AbstractInterpreter,
     linfo::MethodInstance, valid_worlds::WorldRange, result::InferenceResult)
     inferred_result = result.src
-    # If we decided not to optimize, drop the OptimizationState now.
-    # External interpreters can override as necessary to cache additional information
     if inferred_result isa OptimizationState{typeof(interp)}
-        inferred_result = ir_to_codeinf!(inferred_result)
+        inferred_result, result.overrides = ir_to_codeinf!(inferred_result)
     end
     if inferred_result isa CodeInfo
         inferred_result.min_world = first(valid_worlds)

base/compiler/types.jl

@@ -42,19 +42,20 @@ A type that represents the result of running type inference on a chunk of code.
 See also [`matching_cache_argtypes`](@ref).
 """
 mutable struct InferenceResult
-    linfo::MethodInstance
-    argtypes::Vector{Any}
-    overridden_by_const::BitVector
+    const linfo::MethodInstance
+    const argtypes::Vector{Any}
+    const overridden_by_const::BitVector
     result                   # ::Type, or InferenceState if WIP
     src                      # ::Union{CodeInfo, IRCode, OptimizationState} if inferred copy is available, nothing otherwise
     valid_worlds::WorldRange # if inference and optimization is finished
     ipo_effects::Effects     # if inference is finished
     effects::Effects         # if optimization is finished
+    overrides                # ::Vector{SSAValueTypeOverride} if optimized, nothing otherwise
     argescapes               # ::ArgEscapeCache if optimized, nothing otherwise
     must_be_codeinf::Bool    # if this must come out as CodeInfo or leaving it as IRCode is ok
     function InferenceResult(linfo::MethodInstance, cache_argtypes::Vector{Any}, overridden_by_const::BitVector)
         return new(linfo, cache_argtypes, overridden_by_const, Any, nothing,
-            WorldRange(), Effects(), Effects(), nothing, true)
+            WorldRange(), Effects(), Effects(), nothing, nothing, true)
     end
 end
 function InferenceResult(linfo::MethodInstance)

src/gf.c

@@ -216,13 +216,6 @@ JL_DLLEXPORT jl_value_t *jl_methtable_lookup(jl_methtable_t *mt, jl_value_t *typ
 
 // ----- MethodInstance specialization instantiation ----- //
 
-JL_DLLEXPORT jl_code_instance_t* jl_new_codeinst(
-        jl_method_instance_t *mi, jl_value_t *rettype,
-        jl_value_t *inferred_const, jl_value_t *inferred,
-        int32_t const_flags, size_t min_world, size_t max_world,
-        uint32_t ipo_effects, uint32_t effects, jl_value_t *argescapes,
-        uint8_t relocatability);
-
 jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_args_t fptr) JL_GC_DISABLED
 {
     jl_sym_t *sname = jl_symbol(name);
@@ -256,7 +249,7 @@ jl_datatype_t *jl_mk_builtin_func(jl_datatype_t *dt, const char *name, jl_fptr_a
 
     jl_code_instance_t *codeinst = jl_new_codeinst(mi,
         (jl_value_t*)jl_any_type, jl_nothing, jl_nothing,
-        0, 1, ~(size_t)0, 0, 0, jl_nothing, 0);
+        0, 1, ~(size_t)0, 0, 0, jl_nothing, jl_nothing, 0);
     jl_mi_cache_insert(mi, codeinst);
     jl_atomic_store_relaxed(&codeinst->specptr.fptr1, fptr);
     jl_atomic_store_relaxed(&codeinst->invoke, jl_fptr_args);
@@ -383,7 +376,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_get_method_inferred(
     }
     codeinst = jl_new_codeinst(
         mi, rettype, NULL, NULL,
-        0, min_world, max_world, 0, 0, jl_nothing, 0);
+        0, min_world, max_world, 0, 0, jl_nothing, jl_nothing, 0);
     jl_mi_cache_insert(mi, codeinst);
     return codeinst;
 }
@@ -392,8 +385,8 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst(
         jl_method_instance_t *mi, jl_value_t *rettype,
         jl_value_t *inferred_const, jl_value_t *inferred,
         int32_t const_flags, size_t min_world, size_t max_world,
-        uint32_t ipo_effects, uint32_t effects, jl_value_t *argescapes,
-        uint8_t relocatability
+        uint32_t ipo_effects, uint32_t effects,
+        jl_value_t *overrides, jl_value_t *argescapes, uint8_t relocatability
         /*, jl_array_t *edges, int absolute_max*/)
 {
     jl_task_t *ct = jl_current_task;
@@ -420,6 +413,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst(
     jl_atomic_store_relaxed(&codeinst->next, NULL);
     codeinst->ipo_purity_bits = ipo_effects;
     jl_atomic_store_relaxed(&codeinst->purity_bits, effects);
+    codeinst->overrides = overrides;
     codeinst->argescapes = argescapes;
     codeinst->relocatability = relocatability;
     return codeinst;
@@ -2208,7 +2202,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t
                 if (unspec && jl_atomic_load_acquire(&unspec->invoke)) {
                     jl_code_instance_t *codeinst = jl_new_codeinst(mi,
                         (jl_value_t*)jl_any_type, NULL, NULL,
-                        0, 1, ~(size_t)0, 0, 0, jl_nothing, 0);
+                        0, 1, ~(size_t)0, 0, 0, jl_nothing, jl_nothing, 0);
                     codeinst->isspecsig = 0;
                     codeinst->specptr = unspec->specptr;
                     codeinst->rettype_const = unspec->rettype_const;
@@ -2228,7 +2222,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t
         if (!jl_code_requires_compiler(src, 0)) {
             jl_code_instance_t *codeinst = jl_new_codeinst(mi,
                 (jl_value_t*)jl_any_type, NULL, NULL,
-                0, 1, ~(size_t)0, 0, 0, jl_nothing, 0);
+                0, 1, ~(size_t)0, 0, 0, jl_nothing, jl_nothing, 0);
             jl_atomic_store_relaxed(&codeinst->invoke, jl_fptr_interpret_call);
             jl_mi_cache_insert(mi, codeinst);
             record_precompile_statement(mi);
@@ -2263,7 +2257,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t
             return ucache;
         }
         codeinst = jl_new_codeinst(mi, (jl_value_t*)jl_any_type, NULL, NULL,
-            0, 1, ~(size_t)0, 0, 0, jl_nothing, 0);
+            0, 1, ~(size_t)0, 0, 0, jl_nothing, jl_nothing, 0);
         codeinst->isspecsig = 0;
         codeinst->specptr = ucache->specptr;
         codeinst->rettype_const = ucache->rettype_const;

src/jltypes.c

@@ -2579,7 +2579,7 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_code_instance_type =
         jl_new_datatype(jl_symbol("CodeInstance"), core,
                         jl_any_type, jl_emptysvec,
-                        jl_perm_symsvec(15,
+                        jl_perm_symsvec(16,
                             "def",
                             "next",
                             "min_world",
@@ -2590,10 +2590,11 @@ void jl_init_types(void) JL_GC_DISABLED
                             //"edges",
                             //"absolute_max",
                             "ipo_purity_bits", "purity_bits",
+                            "overrides",
                             "argescapes",
                             "isspecsig", "precompile", "relocatability",
                             "invoke", "specptr"), // function object decls
-                        jl_svec(15,
+                        jl_svec(16,
                             jl_method_instance_type,
                             jl_any_type,
                             jl_ulong_type,
@@ -2605,15 +2606,16 @@ void jl_init_types(void) JL_GC_DISABLED
                             //jl_bool_type,
                             jl_uint32_type, jl_uint32_type,
                             jl_any_type,
+                            jl_any_type,
                             jl_bool_type,
                             jl_bool_type,
                             jl_uint8_type,
                             jl_any_type, jl_any_type), // fptrs
                         jl_emptysvec,
                         0, 1, 1);
     jl_svecset(jl_code_instance_type->types, 1, jl_code_instance_type);
-    const static uint32_t code_instance_constfields[1]  = { 0b000001010110001 }; // Set fields 1, 5-6, 8, 10 as const
-    const static uint32_t code_instance_atomicfields[1] = { 0b110100101000010 }; // Set fields 2, 7, 9, 12, 14-15 as atomic
+    const static uint32_t code_instance_constfields[1]  = { 0b0000011010110001 }; // Set fields 1, 5-6, 8, 10-11 as const
+    const static uint32_t code_instance_atomicfields[1] = { 0b1101000101000010 }; // Set fields 2, 7, 9, 13, 15-16 as atomic
     //Fields 3-4 are only operated on by construction and deserialization, so are const at runtime
     //Fields 11 and 15 must be protected by locks, and thus all operations on jl_code_instance_t are threadsafe
     jl_code_instance_type->name->constfields = code_instance_constfields;
@@ -2780,8 +2782,8 @@ void jl_init_types(void) JL_GC_DISABLED
     jl_svecset(jl_methtable_type->types, 10, jl_uint8_type);
     jl_svecset(jl_method_type->types, 12, jl_method_instance_type);
     jl_svecset(jl_method_instance_type->types, 6, jl_code_instance_type);
-    jl_svecset(jl_code_instance_type->types, 13, jl_voidpointer_type);
     jl_svecset(jl_code_instance_type->types, 14, jl_voidpointer_type);
+    jl_svecset(jl_code_instance_type->types, 15, jl_voidpointer_type);
     jl_svecset(jl_binding_type->types, 2, jl_globalref_type);
 
     jl_compute_field_offsets(jl_datatype_type);

src/julia.h

@@ -419,6 +419,7 @@ typedef struct _jl_code_instance_t {
     //     uint8_t nonoverlayed        : 1;
     //     uint8_t notaskstate         : 2;
     //     uint8_t inaccessiblememonly : 2;
+    jl_value_t *overrides;  // override information of `inferred`'s ssavaluetypes
     jl_value_t *argescapes; // escape information of call arguments
 
     // compilation state cache
@@ -1474,6 +1475,12 @@ JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args,
 JL_DLLEXPORT jl_value_t *jl_new_structt(jl_datatype_t *type, jl_value_t *tup);
 JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type);
 JL_DLLEXPORT jl_method_instance_t *jl_new_method_instance_uninit(void);
+JL_DLLEXPORT jl_code_instance_t* jl_new_codeinst(
+        jl_method_instance_t *mi, jl_value_t *rettype,
+        jl_value_t *inferred_const, jl_value_t *inferred,
+        int32_t const_flags, size_t min_world, size_t max_world,
+        uint32_t ipo_effects, uint32_t effects,
+        jl_value_t *overrrides, jl_value_t *argescapes, uint8_t relocatability);
 JL_DLLEXPORT jl_svec_t *jl_svec(size_t n, ...) JL_MAYBE_UNROOTED;
 JL_DLLEXPORT jl_svec_t *jl_svec1(void *a);
 JL_DLLEXPORT jl_svec_t *jl_svec2(void *a, void *b);

src/opaque_closure.c

@@ -105,13 +105,6 @@ jl_opaque_closure_t *jl_new_opaque_closure(jl_tupletype_t *argt, jl_value_t *rt_
 jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name,
     int nargs, jl_value_t *functionloc, jl_code_info_t *ci, int isva);
 
-JL_DLLEXPORT jl_code_instance_t* jl_new_codeinst(
-        jl_method_instance_t *mi, jl_value_t *rettype,
-        jl_value_t *inferred_const, jl_value_t *inferred,
-        int32_t const_flags, size_t min_world, size_t max_world,
-        uint32_t ipo_effects, uint32_t effects, jl_value_t *argescapes,
-        uint8_t relocatability);
-
 JL_DLLEXPORT jl_opaque_closure_t *jl_new_opaque_closure_from_code_info(jl_tupletype_t *argt, jl_value_t *rt_lb, jl_value_t *rt_ub,
     jl_module_t *mod, jl_code_info_t *ci, int lineno, jl_value_t *file, int nargs, int isva, jl_value_t *env, int do_compile)
 {
@@ -127,7 +120,7 @@ JL_DLLEXPORT jl_opaque_closure_t *jl_new_opaque_closure_from_code_info(jl_tuplet
     sigtype = prepend_type(jl_typeof(env), argt);
     jl_method_instance_t *mi = jl_specializations_get_linfo((jl_method_t*)root, sigtype, jl_emptysvec);
     inst = jl_new_codeinst(mi, rt_ub, NULL, (jl_value_t*)ci,
-        0, ((jl_method_t*)root)->primary_world, -1, 0, 0, jl_nothing, 0);
+        0, ((jl_method_t*)root)->primary_world, -1, 0, 0, jl_nothing, jl_nothing, 0);
     jl_mi_cache_insert(mi, inst);
 
     jl_opaque_closure_t *oc = new_opaque_closure(argt, rt_lb, rt_ub, root, env, do_compile);

test/core.jl

@@ -14,7 +14,7 @@ include("testenv.jl")
 # sanity tests that our built-in types are marked correctly for const fields
 for (T, c) in (
         (Core.CodeInfo, []),
-        (Core.CodeInstance, [:def, :rettype, :rettype_const, :ipo_purity_bits, :argescapes]),
+        (Core.CodeInstance, [:def, :rettype, :rettype_const, :ipo_purity_bits, :overrrides, :argescapes]),
         (Core.Method, [#=:name, :module, :file, :line, :primary_world, :sig, :slot_syms, :external_mt, :nargs, :called, :nospecialize, :nkw, :isva, :pure, :is_for_opaque_closure, :constprop=#]),
         (Core.MethodInstance, [#=:def, :specTypes, :sparam_vals=#]),
         (Core.MethodTable, [:module]),