implement comprehensions by calling collect on a Generator

fix #4470 Generalized comprehension syntax
fix #7258 type-inference-independent comprehensions

[ci skip]

base/abstractarray.jl

@@ -1296,21 +1296,23 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray)
     return dest
 end
 
-function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray)
+function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A::AbstractArray)
     # map to dest array, checking the type of each result. if a result does not
     # match, widen the result type and re-dispatch.
-    for i = offs:length(A)
-        @inbounds Ai = A[i]
+    i = offs
+    while !done(A, st)
+        @inbounds Ai, st = next(A, st)
         el = f(Ai)
         S = typeof(el)
         if S === T || S <: T
             @inbounds dest[i] = el::T
+            i += 1
         else
             R = typejoin(T, S)
             new = similar(dest, R)
             copy!(new,1, dest,1, i-1)
             @inbounds new[i] = el
-            return map_to!(f, i+1, new, A)
+            return map_to!(f, i+1, st, new, A)
         end
     end
     return dest
@@ -1320,10 +1322,12 @@ function map(f, A::AbstractArray)
     if isempty(A)
         return isa(f,Type) ? similar(A,f) : similar(A)
     end
-    first = f(A[1])
+    st = start(A)
+    A1, st = next(A, st)
+    first = f(A1)
     dest = similar(A, typeof(first))
     dest[1] = first
-    return map_to!(f, 2, dest, A)
+    return map_to!(f, 2, st, dest, A)
 end
 
 ## 2 argument

base/essentials.jl

@@ -193,3 +193,17 @@ const (:) = Colon()
 # For passing constants through type inference
 immutable Val{T}
 end
+
+immutable Generator{F,I}
+    f::F
+    iter::I
+end
+
+start(g::Generator) = start(g.iter)
+done(g::Generator, s) = done(g.iter, s)
+function next(g::Generator, s)
+    v, s2 = next(g.iter, s)
+    g.f(v), s2
+end
+
+collect(g::Generator) = map(g.f, g.iter)

base/inference.jl

@@ -25,7 +25,6 @@ type StaticVarInfo
     gensym_types::Array{Any,1} # types of the GenSym's in this function
     vinfo::Array{Any,1}  # variable properties
     label_counter::Int   # index of the current highest label for this function
-    fedbackvars::ObjectIdDict
     mod::Module
 end
 
@@ -53,7 +52,7 @@ function StaticVarInfo(linfo::LambdaStaticData, ast=linfo.ast)
     else
         sp = svec()
     end
-    StaticVarInfo(sp, vars, gensym_types, vinflist, nl, ObjectIdDict(), linfo.module)
+    StaticVarInfo(sp, vars, gensym_types, vinflist, nl, linfo.module)
 end
 
 type VarState
@@ -1070,8 +1069,7 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
         return abstract_eval_constant(e)
     end
     e = e::Expr
-    # handle:
-    # call  null  new  &  static_typeof
+    # handle: call  null  new  &
     if is(e.head,:call)
         t = abstract_eval_call(e, vtypes, sv)
     elseif is(e.head,:null)
@@ -1089,40 +1087,6 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
     elseif is(e.head,:&)
         abstract_eval(e.args[1], vtypes, sv)
         t = Any
-    elseif is(e.head,:static_typeof)
-        var = e.args[1]
-        t = abstract_eval(var, vtypes, sv)
-        if isa(t,DataType) && typeseq(t,t.name.primary)
-            # remove unnecessary typevars
-            t = t.name.primary
-        end
-        if is(t,Bottom)
-            # if we haven't gotten fed-back type info yet, return Bottom. otherwise
-            # Bottom is the actual type of the variable, so return Type{Bottom}.
-            if haskey(sv.fedbackvars, var)
-                t = Type{Bottom}
-            end
-        elseif isleaftype(t)
-            t = Type{t}
-        elseif isleaftype(inference_stack.types)
-            if isa(t,TypeVar)
-                t = Type{t.ub}
-            else
-                t = Type{t}
-            end
-        else
-            # if there is any type uncertainty in the arguments, we are
-            # effectively predicting what static_typeof will say when
-            # the function is compiled with actual arguments. in that case
-            # abstract types yield Type{<:T} instead of Type{T}.
-            # this doesn't really model the situation perfectly, but
-            # "isleaftype(inference_stack.types)" should be good enough.
-            if isa(t,TypeVar)
-                t = Type{t}
-            else
-                t = Type{TypeVar(:_,t)}
-            end
-        end
     elseif is(e.head,:method)
         t = (length(e.args) == 1) ? Any : Void
     elseif is(e.head,:copyast)
@@ -1678,10 +1642,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
     recpts = IntSet()  # statements that depend recursively on our value
     W = IntSet()
 
-    @label typeinf_top
-
-    typegotoredo = false
-
     # exception handlers
     cur_hand = ()
     handler_at = Any[ () for i=1:n ]
@@ -1753,24 +1713,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
                             s[l] = stupdate(s[l], changes, vars)
                         end
                     end
-                elseif is(hd,:type_goto)
-                    for i = 2:length(stmt.args)
-                        var = stmt.args[i]::GenSym
-                        # Store types that need to be fed back via type_goto
-                        # in gensym_init. After finishing inference, if any
-                        # of these types changed, start over with the fed-back
-                        # types known from the beginning.
-                        # See issue #3821 (using !typeseq instead of !subtype),
-                        # and issue #7810.
-                        id = var.id+1
-                        vt = gensym_types[id]
-                        ot = gensym_init[id]
-                        if ot===NF || !typeseq(vt,ot)
-                            gensym_init[id] = vt
-                            typegotoredo = true
-                        end
-                        sv.fedbackvars[var] = true
-                    end
                 elseif is(hd,:return)
                     pc´ = n+1
                     rt = abstract_eval(stmt.args[1], s[pc], sv)
@@ -1835,16 +1777,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
         end
     end
 
-    if typegotoredo
-        # if any type_gotos changed, clear state and restart.
-        for ll = 2:length(s)
-            s[ll] = ()
-        end
-        empty!(W)
-        gensym_types[:] = gensym_init
-        frame.result = curtype
-        @goto typeinf_top
-    end
     for i = 1:length(gensym_types)
         if gensym_types[i] === NF
             gensym_types[i] = Union{}
@@ -1931,7 +1863,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, undefs)
 
     e = e::Expr
     head = e.head
-    if is(head,:static_typeof) || is(head,:line) || is(head,:const)
+    if is(head,:line) || is(head,:const)
         return e
     #elseif is(head,:gotoifnot) || is(head,:return)
     #    e.typ = Any
@@ -2168,9 +2100,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool)
     end
     if isa(e,Expr)
         e = e::Expr
-        if e.head === :static_typeof
-            return true
-        end
         ea = e.args
         if e.head === :call
             if is_known_call_p(e, is_pure_builtin, sv)

src/alloc.c

@@ -87,12 +87,11 @@ jl_sym_t *null_sym;    jl_sym_t *body_sym;
 jl_sym_t *method_sym;
 jl_sym_t *enter_sym;   jl_sym_t *leave_sym;
 jl_sym_t *exc_sym;     jl_sym_t *error_sym;
-jl_sym_t *static_typeof_sym;
 jl_sym_t *new_sym;     jl_sym_t *using_sym;
 jl_sym_t *const_sym;   jl_sym_t *thunk_sym;
 jl_sym_t *anonymous_sym;  jl_sym_t *underscore_sym;
 jl_sym_t *abstracttype_sym; jl_sym_t *bitstype_sym;
-jl_sym_t *compositetype_sym; jl_sym_t *type_goto_sym;
+jl_sym_t *compositetype_sym;
 jl_sym_t *global_sym; jl_sym_t *list_sym;
 jl_sym_t *dot_sym;    jl_sym_t *newvar_sym;
 jl_sym_t *boundscheck_sym; jl_sym_t *inbounds_sym;

src/builtins.c

@@ -431,7 +431,7 @@ JL_CALLABLE(jl_f__apply)
         else if (jl_is_tuple(args[i])) {
             n += jl_nfields(args[i]);
         }
-        else if (jl_is_array(args[i]) && ((jl_array_t*)args[i])->ptrarray) {
+        else if (jl_is_array(args[i])) {
             n += jl_array_len(args[i]);
         }
         else {
@@ -474,32 +474,42 @@ JL_CALLABLE(jl_f__apply)
     newargs[0] = f;
     n = 1;
     for(i=1; i < nargs; i++) {
-        if (jl_is_svec(args[i])) {
-            jl_svec_t *t = (jl_svec_t*)args[i];
+        jl_value_t *ai = args[i];
+        if (jl_is_svec(ai)) {
+            jl_svec_t *t = (jl_svec_t*)ai;
             size_t al = jl_svec_len(t);
             for(j=0; j < al; j++)
                 newargs[n++] = jl_svecref(t, j);
         }
-        else if (jl_is_tuple(args[i])) {
-            size_t al = jl_nfields(args[i]);
+        else if (jl_is_tuple(ai)) {
+            size_t al = jl_nfields(ai);
             for(j=0; j < al; j++) {
                 // jl_fieldref may allocate.
-                newargs[n++] = jl_fieldref(args[i], j);
-                if (arg_heap) {
+                newargs[n++] = jl_fieldref(ai, j);
+                if (arg_heap)
                     jl_gc_wb(arg_heap, newargs[n - 1]);
-                }
             }
         }
         else {
-            size_t al = jl_array_len(args[i]);
-            for (j = 0;j < al;j++) {
-                jl_value_t *arg = jl_cellref(args[i], j);
-                // apply with array splatting may have embedded NULL value
-                // #11772
-                if (__unlikely(arg == NULL)) {
-                    jl_throw(jl_undefref_exception);
+            assert(jl_is_array(ai));
+            jl_array_t *aai = (jl_array_t*)ai;
+            size_t al = jl_array_len(aai);
+            if (aai->ptrarray) {
+                for (j = 0; j < al; j++) {
+                    jl_value_t *arg = jl_cellref(aai, j);
+                    // apply with array splatting may have embedded NULL value
+                    // #11772
+                    if (__unlikely(arg == NULL))
+                        jl_throw(jl_undefref_exception);
+                    newargs[n++] = arg;
+                }
+            }
+            else {
+                for (j = 0; j < al; j++) {
+                    newargs[n++] = jl_arrayref(aai, j);
+                    if (arg_heap)
+                        jl_gc_wb(arg_heap, newargs[n - 1]);
                 }
-                newargs[n++] = arg;
             }
         }
     }

src/codegen.cpp

@@ -3552,20 +3552,6 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b
     else if (head == null_sym) {
         return ghostValue(jl_void_type);
     }
-    else if (head == static_typeof_sym) {
-        jl_value_t *extype = expr_type((jl_value_t*)ex, ctx);
-        if (jl_is_type_type(extype)) {
-            extype = jl_tparam0(extype);
-            if (jl_is_typevar(extype))
-                extype = ((jl_tvar_t*)extype)->ub;
-        }
-        else {
-            extype = (jl_value_t*)jl_any_type;
-        }
-        if (jl_is_tuple_type(extype))
-            jl_add_linfo_root(ctx->linfo, extype);
-        return mark_julia_const(extype);
-    }
     else if (head == new_sym) {
         jl_value_t *ty = expr_type(args[0], ctx);
         size_t nargs = jl_array_len(ex->args);
@@ -3705,7 +3691,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b
             return ghostValue(jl_void_type);
         }
         // some expression types are metadata and can be ignored
-        if (valuepos || !(head == line_sym || head == type_goto_sym)) {
+        if (valuepos || !(head == line_sym)) {
             if (head == abstracttype_sym || head == compositetype_sym ||
                 head == bitstype_sym) {
                 jl_errorf("type definition not allowed inside a local scope");

src/interpreter.c

@@ -219,9 +219,6 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng
     else if (ex->head == exc_sym) {
         return jl_exception_in_transit;
     }
-    else if (ex->head == static_typeof_sym) {
-        return (jl_value_t*)jl_any_type;
-    }
     else if (ex->head == method_sym) {
         jl_sym_t *fname = (jl_sym_t*)args[0];
         assert(jl_expr_nargs(ex) != 1 || jl_is_symbol(fname));

src/jltypes.c

@@ -3590,7 +3590,6 @@ void jl_init_types(void)
     exc_sym = jl_symbol("the_exception");
     enter_sym = jl_symbol("enter");
     leave_sym = jl_symbol("leave");
-    static_typeof_sym = jl_symbol("static_typeof");
     new_sym = jl_symbol("new");
     const_sym = jl_symbol("const");
     global_sym = jl_symbol("global");
@@ -3601,7 +3600,6 @@ void jl_init_types(void)
     abstracttype_sym = jl_symbol("abstract_type");
     bitstype_sym = jl_symbol("bits_type");
     compositetype_sym = jl_symbol("composite_type");
-    type_goto_sym = jl_symbol("type_goto");
     toplevel_sym = jl_symbol("toplevel");
     dot_sym = jl_symbol(".");
     boundscheck_sym = jl_symbol("boundscheck");