implement comprehensions by calling collect on a Generator

fix #4470 Generalized comprehension syntax
fix #7258 type-inference-independent comprehensions

[ci skip]

base/essentials.jl

@@ -193,3 +193,17 @@ const (:) = Colon()
 # For passing constants through type inference
 immutable Val{T}
 end
+
+immutable Generator{F,I}
+    f::F
+    iter::I
+end
+
+start(g::Generator) = start(g.iter)
+done(g::Generator, s) = done(g.iter, s)
+function next(g::Generator, s)
+    v, s2 = next(g.iter, s)
+    g.f(v), s2
+end
+
+collect(g::Generator) = map(g.f, g.iter)

base/inference.jl

@@ -25,7 +25,6 @@ type StaticVarInfo
     gensym_types::Array{Any,1} # types of the GenSym's in this function
     vinfo::Array{Any,1}  # variable properties
     label_counter::Int   # index of the current highest label for this function
-    fedbackvars::ObjectIdDict
     mod::Module
 end
 
@@ -53,7 +52,7 @@ function StaticVarInfo(linfo::LambdaStaticData, ast=linfo.ast)
     else
         sp = svec()
     end
-    StaticVarInfo(sp, vars, gensym_types, vinflist, nl, ObjectIdDict(), linfo.module)
+    StaticVarInfo(sp, vars, gensym_types, vinflist, nl, linfo.module)
 end
 
 type VarState
@@ -1070,8 +1069,7 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
         return abstract_eval_constant(e)
     end
     e = e::Expr
-    # handle:
-    # call  null  new  &  static_typeof
+    # handle: call  null  new  &
     if is(e.head,:call)
         t = abstract_eval_call(e, vtypes, sv)
     elseif is(e.head,:null)
@@ -1089,40 +1087,6 @@ function abstract_eval(e::ANY, vtypes, sv::StaticVarInfo)
     elseif is(e.head,:&)
         abstract_eval(e.args[1], vtypes, sv)
         t = Any
-    elseif is(e.head,:static_typeof)
-        var = e.args[1]
-        t = abstract_eval(var, vtypes, sv)
-        if isa(t,DataType) && typeseq(t,t.name.primary)
-            # remove unnecessary typevars
-            t = t.name.primary
-        end
-        if is(t,Bottom)
-            # if we haven't gotten fed-back type info yet, return Bottom. otherwise
-            # Bottom is the actual type of the variable, so return Type{Bottom}.
-            if haskey(sv.fedbackvars, var)
-                t = Type{Bottom}
-            end
-        elseif isleaftype(t)
-            t = Type{t}
-        elseif isleaftype(inference_stack.types)
-            if isa(t,TypeVar)
-                t = Type{t.ub}
-            else
-                t = Type{t}
-            end
-        else
-            # if there is any type uncertainty in the arguments, we are
-            # effectively predicting what static_typeof will say when
-            # the function is compiled with actual arguments. in that case
-            # abstract types yield Type{<:T} instead of Type{T}.
-            # this doesn't really model the situation perfectly, but
-            # "isleaftype(inference_stack.types)" should be good enough.
-            if isa(t,TypeVar)
-                t = Type{t}
-            else
-                t = Type{TypeVar(:_,t)}
-            end
-        end
     elseif is(e.head,:method)
         t = (length(e.args) == 1) ? Any : Void
     elseif is(e.head,:copyast)
@@ -1678,10 +1642,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
     recpts = IntSet()  # statements that depend recursively on our value
     W = IntSet()
 
-    @label typeinf_top
-
-    typegotoredo = false
-
     # exception handlers
     cur_hand = ()
     handler_at = Any[ () for i=1:n ]
@@ -1753,24 +1713,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
                             s[l] = stupdate(s[l], changes, vars)
                         end
                     end
-                elseif is(hd,:type_goto)
-                    for i = 2:length(stmt.args)
-                        var = stmt.args[i]::GenSym
-                        # Store types that need to be fed back via type_goto
-                        # in gensym_init. After finishing inference, if any
-                        # of these types changed, start over with the fed-back
-                        # types known from the beginning.
-                        # See issue #3821 (using !typeseq instead of !subtype),
-                        # and issue #7810.
-                        id = var.id+1
-                        vt = gensym_types[id]
-                        ot = gensym_init[id]
-                        if ot===NF || !typeseq(vt,ot)
-                            gensym_init[id] = vt
-                            typegotoredo = true
-                        end
-                        sv.fedbackvars[var] = true
-                    end
                 elseif is(hd,:return)
                     pc´ = n+1
                     rt = abstract_eval(stmt.args[1], s[pc], sv)
@@ -1835,16 +1777,6 @@ function typeinf_uncached(linfo::LambdaStaticData, atypes::ANY, sparams::SimpleV
         end
     end
 
-    if typegotoredo
-        # if any type_gotos changed, clear state and restart.
-        for ll = 2:length(s)
-            s[ll] = ()
-        end
-        empty!(W)
-        gensym_types[:] = gensym_init
-        frame.result = curtype
-        @goto typeinf_top
-    end
     for i = 1:length(gensym_types)
         if gensym_types[i] === NF
             gensym_types[i] = Union{}
@@ -1931,7 +1863,7 @@ function eval_annotate(e::ANY, vtypes::ANY, sv::StaticVarInfo, decls, undefs)
 
     e = e::Expr
     head = e.head
-    if is(head,:static_typeof) || is(head,:line) || is(head,:const)
+    if is(head,:line) || is(head,:const)
         return e
     #elseif is(head,:gotoifnot) || is(head,:return)
     #    e.typ = Any
@@ -2168,9 +2100,6 @@ function effect_free(e::ANY, sv, allow_volatile::Bool)
     end
     if isa(e,Expr)
         e = e::Expr
-        if e.head === :static_typeof
-            return true
-        end
         ea = e.args
         if e.head === :call
             if is_known_call_p(e, is_pure_builtin, sv)

src/alloc.c

@@ -87,12 +87,11 @@ jl_sym_t *null_sym;    jl_sym_t *body_sym;
 jl_sym_t *method_sym;
 jl_sym_t *enter_sym;   jl_sym_t *leave_sym;
 jl_sym_t *exc_sym;     jl_sym_t *error_sym;
-jl_sym_t *static_typeof_sym;
 jl_sym_t *new_sym;     jl_sym_t *using_sym;
 jl_sym_t *const_sym;   jl_sym_t *thunk_sym;
 jl_sym_t *anonymous_sym;  jl_sym_t *underscore_sym;
 jl_sym_t *abstracttype_sym; jl_sym_t *bitstype_sym;
-jl_sym_t *compositetype_sym; jl_sym_t *type_goto_sym;
+jl_sym_t *compositetype_sym;
 jl_sym_t *global_sym; jl_sym_t *list_sym;
 jl_sym_t *dot_sym;    jl_sym_t *newvar_sym;
 jl_sym_t *boundscheck_sym; jl_sym_t *inbounds_sym;

src/codegen.cpp

@@ -3552,20 +3552,6 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b
     else if (head == null_sym) {
         return ghostValue(jl_void_type);
     }
-    else if (head == static_typeof_sym) {
-        jl_value_t *extype = expr_type((jl_value_t*)ex, ctx);
-        if (jl_is_type_type(extype)) {
-            extype = jl_tparam0(extype);
-            if (jl_is_typevar(extype))
-                extype = ((jl_tvar_t*)extype)->ub;
-        }
-        else {
-            extype = (jl_value_t*)jl_any_type;
-        }
-        if (jl_is_tuple_type(extype))
-            jl_add_linfo_root(ctx->linfo, extype);
-        return mark_julia_const(extype);
-    }
     else if (head == new_sym) {
         jl_value_t *ty = expr_type(args[0], ctx);
         size_t nargs = jl_array_len(ex->args);
@@ -3705,7 +3691,7 @@ static jl_cgval_t emit_expr(jl_value_t *expr, jl_codectx_t *ctx, bool isboxed, b
             return ghostValue(jl_void_type);
         }
         // some expression types are metadata and can be ignored
-        if (valuepos || !(head == line_sym || head == type_goto_sym)) {
+        if (valuepos || !(head == line_sym)) {
             if (head == abstracttype_sym || head == compositetype_sym ||
                 head == bitstype_sym) {
                 jl_errorf("type definition not allowed inside a local scope");

src/interpreter.c

@@ -219,9 +219,6 @@ static jl_value_t *eval(jl_value_t *e, jl_value_t **locals, size_t nl, size_t ng
     else if (ex->head == exc_sym) {
         return jl_exception_in_transit;
     }
-    else if (ex->head == static_typeof_sym) {
-        return (jl_value_t*)jl_any_type;
-    }
     else if (ex->head == method_sym) {
         jl_sym_t *fname = (jl_sym_t*)args[0];
         assert(jl_expr_nargs(ex) != 1 || jl_is_symbol(fname));

src/jltypes.c

@@ -3590,7 +3590,6 @@ void jl_init_types(void)
     exc_sym = jl_symbol("the_exception");
     enter_sym = jl_symbol("enter");
     leave_sym = jl_symbol("leave");
-    static_typeof_sym = jl_symbol("static_typeof");
     new_sym = jl_symbol("new");
     const_sym = jl_symbol("const");
     global_sym = jl_symbol("global");
@@ -3601,7 +3600,6 @@ void jl_init_types(void)
     abstracttype_sym = jl_symbol("abstract_type");
     bitstype_sym = jl_symbol("bits_type");
     compositetype_sym = jl_symbol("composite_type");
-    type_goto_sym = jl_symbol("type_goto");
     toplevel_sym = jl_symbol("toplevel");
     dot_sym = jl_symbol(".");
     boundscheck_sym = jl_symbol("boundscheck");

src/julia-parser.scm

@@ -1397,22 +1397,23 @@
 (define (parse-comma-separated-assignments s)
   (parse-comma-separated s parse-eq*))
 
+(define (parse-iteration-spec s)
+  (let ((r (parse-eq* s)))
+    (cond ((and (pair? r) (eq? (car r) '=))  r)
+          ((eq? r ':)  r)
+          ((and (length= r 4) (eq? (car r) 'comparison)
+                (or (eq? (caddr r) 'in) (eq? (caddr r) '∈)))
+           `(= ,(cadr r) ,(cadddr r)))
+          (else
+           (error "invalid iteration specification")))))
+
 ; as above, but allows both "i=r" and "i in r"
 (define (parse-comma-separated-iters s)
   (let loop ((ranges '()))
-    (let ((r (parse-eq* s)))
-      (let ((r (cond ((and (pair? r) (eq? (car r) '=))
-                      r)
-                     ((eq? r ':)
-                      r)
-                     ((and (length= r 4) (eq? (car r) 'comparison)
-                           (or (eq? (caddr r) 'in) (eq? (caddr r) '∈)))
-                      `(= ,(cadr r) ,(cadddr r)))
-                     (else
-                      (error "invalid iteration specification")))))
-        (case (peek-token s)
-          ((#\,)  (take-token s) (loop (cons r ranges)))
-          (else   (reverse! (cons r ranges))))))))
+    (let ((r (parse-iteration-spec s)))
+      (case (peek-token s)
+        ((#\,)  (take-token s) (loop (cons r ranges)))
+        (else   (reverse! (cons r ranges)))))))
 
 (define (parse-space-separated-exprs s)
   (with-space-sensitive
@@ -1469,7 +1470,12 @@
                        (begin (take-token s) (loop (cons nxt lst))))
                       ((eqv? c #\;)          (loop (cons nxt lst)))
                       ((equal? c closer)     (loop (cons nxt lst)))
-                      ((eq? c 'for)          (take-token s) (parse-generator s t closer))
+                      ((eq? c 'for)
+                       (take-token s)
+                       (let ((gen (parse-generator s nxt #f)))
+                         (if (eqv? (require-token s) #\,)
+                             (take-token s))
+                         (loop (cons gen lst))))
                       ;; newline character isn't detectable here
                       #;((eqv? c #\newline)
                        (error "unexpected line break in argument list"))
@@ -1514,7 +1520,7 @@
 (define (parse-comprehension s first closer)
   (let ((r (parse-comma-separated-iters s)))
     (if (not (eqv? (require-token s) closer))
-        (error (string "expected " closer))
+        (error (string "expected \"" closer "\""))
         (take-token s))
     `(comprehension ,first ,@r)))
 
@@ -1524,12 +1530,11 @@
         `(dict_comprehension ,@(cdr c))
         (error "invalid dict comprehension"))))
 
-(define (parse-generator s first closer)
-  (let ((r (parse-comma-separated-iters s)))
-    (if (not (eqv? (require-token s) closer))
-	(error (string "expected " closer))
-        (take-token s))
-    `(macrocall @generator ,first ,@r)))
+(define (parse-generator s first allow-comma)
+  (let ((r (if allow-comma
+               (parse-comma-separated-iters s)
+               (list (parse-iteration-spec s)))))
+    `(generator ,first ,@r)))
 
 (define (parse-matrix s first closer gotnewline)
   (define (fix head v) (cons head (reverse v)))
@@ -1959,6 +1964,13 @@
                           `(tuple ,ex)
                           ;; value in parentheses (x)
                           ex))
+                     ((eq? t 'for)
+                      (take-token s)
+                      (let ((gen (parse-generator s ex #t)))
+                        (if (eqv? (require-token s) #\) )
+                            (take-token s)
+                            (error "expected \")\""))
+                        gen))
                      (else
                       ;; tuple (x,) (x,y) (x...) etc.
                       (if (eqv? t #\, )