From f5bc2559ad9aa83ef3e2478d9fa4fa0f2e652553 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 30 May 2019 17:45:03 +1000 Subject: [PATCH 01/56] Test tools and some tests for syntax desugaring Start building up a test suite for lowering so that we can port the lowering code to julia with efficiency and confidence. --- src/ast.c | 55 +++++--- test/choosetests.jl | 2 +- test/compiler/lowering.jl | 284 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 321 insertions(+), 20 deletions(-) create mode 100644 test/compiler/lowering.jl diff --git a/src/ast.c b/src/ast.c index fe7615f540a80..d5adfb54cc83e 100644 --- a/src/ast.c +++ b/src/ast.c @@ -154,7 +154,7 @@ value_t fl_julia_scalar(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) return fl_ctx->F; } -static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *mod); +static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *mod, int formonly); value_t fl_julia_logmsg(fl_context_t *fl_ctx, value_t *args, uint32_t nargs) { @@ -426,7 +426,7 @@ static jl_value_t *scm_to_julia(fl_context_t *fl_ctx, value_t e, jl_module_t *mo jl_value_t *v = NULL; JL_GC_PUSH1(&v); JL_TRY { - v = scm_to_julia_(fl_ctx, e, mod); + v = scm_to_julia_(fl_ctx, e, mod, 0); } JL_CATCH { // if expression cannot be converted, replace with error expr @@ -440,7 +440,7 @@ static jl_value_t *scm_to_julia(fl_context_t *fl_ctx, value_t e, jl_module_t *mo extern int64_t conv_to_int64(void *data, numerictype_t tag); -static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *mod) +static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *mod, int formonly) { if (fl_isnumber(fl_ctx, e)) { int64_t i64; @@ -514,48 +514,48 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m // nodes with special representations jl_value_t *ex = NULL, *temp = NULL; if (sym == line_sym && (n == 1 || n == 2)) { - jl_value_t *linenum = scm_to_julia_(fl_ctx, car_(e), mod); + jl_value_t *linenum = scm_to_julia_(fl_ctx, car_(e), mod, formonly); jl_value_t *file = jl_nothing; JL_GC_PUSH2(&linenum, &file); if (n == 2) - file = scm_to_julia_(fl_ctx, car_(cdr_(e)), mod); + file = scm_to_julia_(fl_ctx, car_(cdr_(e)), mod, formonly); temp = jl_new_struct(jl_linenumbernode_type, linenum, file); JL_GC_POP(); return temp; } JL_GC_PUSH1(&ex); if (sym == goto_sym) { - ex = scm_to_julia_(fl_ctx, car_(e), mod); + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); temp = jl_new_struct(jl_gotonode_type, ex); } else if (sym == newvar_sym) { - ex = scm_to_julia_(fl_ctx, car_(e), mod); + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); temp = jl_new_struct(jl_newvarnode_type, ex); } - else if (sym == globalref_sym) { - ex = scm_to_julia_(fl_ctx, car_(e), mod); - temp = scm_to_julia_(fl_ctx, car_(cdr_(e)), mod); + else if (sym == globalref_sym && !formonly) { + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); + temp = scm_to_julia_(fl_ctx, car_(cdr_(e)), mod, formonly); assert(jl_is_module(ex)); assert(jl_is_symbol(temp)); temp = jl_module_globalref((jl_module_t*)ex, (jl_sym_t*)temp); } - else if (sym == top_sym) { + else if (sym == top_sym && !formonly) { assert(mod && "top should not be generated by the parser"); - ex = scm_to_julia_(fl_ctx, car_(e), mod); + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); assert(jl_is_symbol(ex)); temp = jl_module_globalref(jl_base_relative_to(mod), (jl_sym_t*)ex); } - else if (sym == core_sym) { - ex = scm_to_julia_(fl_ctx, car_(e), mod); + else if (sym == core_sym && !formonly) { + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); assert(jl_is_symbol(ex)); temp = jl_module_globalref(jl_core_module, (jl_sym_t*)ex); } else if (sym == inert_sym || (sym == quote_sym && (!iscons(car_(e))))) { - ex = scm_to_julia_(fl_ctx, car_(e), mod); + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); temp = jl_new_struct(jl_quotenode_type, ex); } - else if (sym == thunk_sym) { - ex = scm_to_julia_(fl_ctx, car_(e), mod); + else if (sym == thunk_sym && !formonly) { + ex = scm_to_julia_(fl_ctx, car_(e), mod, formonly); assert(jl_is_code_info(ex)); jl_linenumber_to_lineinfo((jl_code_info_t*)ex, (jl_value_t*)jl_symbol("top-level scope")); temp = (jl_value_t*)jl_exprn(sym, 1); @@ -569,10 +569,10 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m size_t i; for (i = 0; i < n; i++) { assert(iscons(e)); - jl_array_ptr_set(((jl_expr_t*)ex)->args, i, scm_to_julia_(fl_ctx, car_(e), mod)); + jl_array_ptr_set(((jl_expr_t*)ex)->args, i, scm_to_julia_(fl_ctx, car_(e), mod, formonly)); e = cdr_(e); } - if (sym == lambda_sym) + if (sym == lambda_sym && !formonly) ex = (jl_value_t*)jl_new_code_info_from_ast((jl_expr_t*)ex); JL_GC_POP(); if (sym == list_sym) @@ -919,6 +919,23 @@ jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module return result; } +// Call given flisp function on an Expr, converting result back to an Expr. +// Does not expand lambdas and thunks to code info objects. +JL_DLLEXPORT jl_value_t *jl_call_scm_on_ast_formonly(const char *funcname, jl_value_t *expr, jl_module_t *inmodule) +{ + jl_ast_context_t *ctx = jl_ast_ctx_enter(); + fl_context_t *fl_ctx = &ctx->fl; + JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule); + value_t arg = julia_to_scm(fl_ctx, expr); + value_t e = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, funcname)), arg); + jl_value_t *result = scm_to_julia_(fl_ctx, e, inmodule, 1); + JL_GC_PUSH1(&result); + JL_AST_PRESERVE_POP(ctx, old_roots); + jl_ast_ctx_leave(ctx); + JL_GC_POP(); + return result; +} + jl_value_t *jl_call_scm_on_ast_and_loc(const char *funcname, jl_value_t *expr, jl_module_t *inmodule, const char *file, int line) { diff --git a/test/choosetests.jl b/test/choosetests.jl index 50cd3d39311e6..a4d1671fbdcf7 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -108,7 +108,7 @@ function choosetests(choices = []) end compilertests = ["compiler/inference", "compiler/validation", "compiler/ssair", "compiler/irpasses", - "compiler/codegen", "compiler/inline", "compiler/contextual"] + "compiler/codegen", "compiler/inline", "compiler/contextual", "compiler/lowering"] if "compiler" in skip_tests filter!(x -> (x != "compiler" && !(x in compilertests)), tests) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl new file mode 100644 index 0000000000000..8501fca274c1b --- /dev/null +++ b/test/compiler/lowering.jl @@ -0,0 +1,284 @@ +using Core: SSAValue + +# Call into lowering stage 1; syntax desugaring +function fl_expand_forms(ex) + ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "expand-forms", ex, Main) +end + +function lift_lowered_expr!(ex, nextid, valmap, lift_full) + if ex isa SSAValue + # Rename SSAValues into renumbered symbols + return get!(valmap, ex) do + newid = nextid[] + nextid[] = newid+1 + Symbol("ssa$newid") + end + end + if ex isa Expr + filter!(e->!(e isa LineNumberNode), ex.args) + if ex.head == :block && length(ex.args) == 1 + # Remove trivial blocks + return lift_lowered_expr!(ex.args[1], nextid, valmap, lift_full) + end + map!(ex.args, ex.args) do e + lift_lowered_expr!(e, nextid, valmap, lift_full) + end + if lift_full && ex.head == :top + return Expr(:(.), :Top, QuoteNode(ex.args[1])) + end + if lift_full && ex.head == :core + return Expr(:(.), :Core, QuoteNode(ex.args[1])) + end + if lift_full && ex.head == :unnecessary + # Rename to something we can actually type + return Expr(:call, :maybe_unused, ex.args...) + end + end + return ex +end + +""" +Clean up an `Expr` into an equivalent form which can be easily entered by +hand + +* Replacing `SSAValue(id)` with consecutively numbered symbols :ssa\$i +* Remove trivial blocks +""" +function lift_lowered_expr(ex; lift_full=false) + valmap = Dict{SSAValue,Symbol}() + nextid = Ref(0) + lift_lowered_expr!(deepcopy(ex), nextid, valmap, lift_full) +end + +""" +Very slight lowering of reference expressions to allow comparison with +desugared forms. + +* Remove trivial blocks +* Translate psuedo-module expressions Top.x and Core.x to Expr(:top) and + Expr(:core) +""" +function lower_ref_expr!(ex) + if ex isa Expr + filter!(e->!(e isa LineNumberNode), ex.args) + if ex.head == :block && length(ex.args) == 1 + # Remove trivial blocks + return lower_ref_expr!(ex.args[1]) + end + if ex.head == :(.) && length(ex.args) >= 1 && (ex.args[1] == :Top || + ex.args[1] == :Core) + (length(ex.args) == 2 && ex.args[2] isa QuoteNode) || throw("Unexpected top/core expression $(sprint(dump, ex))") + return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) + end + if ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused + return Expr(:unnecessary, ex.args[2:end]...) + end + map!(lower_ref_expr!, ex.args, ex.args) + end + return ex +end +lower_ref_expr(ex) = lower_ref_expr!(deepcopy(ex)) + + +function diffdump(io::IOContext, ex1, ex2, n, prefix, indent) + if ex1 == ex2 + isempty(prefix) || print(io, prefix) + dump(io, ex1, n, indent) + else + if ex1 isa Expr && ex2 isa Expr && ex1.head == ex2.head && length(ex1.args) == length(ex2.args) + isempty(prefix) || print(io, prefix) + println(io, "Expr") + println(io, indent, " head: ", ex1.head) + println(io, indent, " args: Array{Any}(", size(ex1.args), ")") + for i in 1:length(ex1.args) + prefix = string(indent, " ", i, ": ") + diffdump(io, ex1.args[i], ex2.args[i], n - 1, prefix, string(" ", indent)) + i < length(ex1.args) && println(io) + end + else + printstyled(io, string(prefix, sprint(dump, ex1, n, indent; context=io)), color=:red) + println() + printstyled(io, string(prefix, sprint(dump, ex2, n, indent; context=io)), color=:green) + end + end +end + +""" +Display colored differences between two expressions `ex1` and `ex2` using the +`dump` format. +""" +function diffdump(ex1, ex2; maxdepth=20) + mod = get(stdout, :module, Main) + diffdump(IOContext(stdout, :limit => true, :module => mod), ex1, ex2, maxdepth, "", "") + println(stdout) +end + +# For interactive convenience in constructing test cases with flisp based lowering +desugar(ex) = lift_lowered_expr(fl_expand_forms(ex); lift_full=true) + +macro desugar(ex) + quote + desugar($(QuoteNode(ex))) + end +end + +""" +Test that syntax desugaring of `input` produces an expression equivalent to the +reference expression `ref`. +""" +macro test_desugar(input, ref) + ex = quote + input = lift_lowered_expr(fl_expand_forms($(QuoteNode(input)))) + ref = lower_ref_expr($(QuoteNode(ref))) + @test input == ref + if input != ref + println("Diff dump:") + diffdump(input, ref) + end + end + # Attribute the test to the correct line number + @assert ex.args[6].args[1] == Symbol("@test") + ex.args[6].args[2] = __source__ + ex +end + +@testset "Property notation" begin + @test_desugar(a.b, Top.getproperty(a, :b)) + @test_desugar(a.b.c, Top.getproperty(Top.getproperty(a, :b), :c)) + + @test_desugar( + a.b = c, + begin + Top.setproperty!(a, :b, c) + maybe_unused(c) + end + ) + @test_desugar( + a.b.c = d, + begin + ssa0 = Top.getproperty(a, :b) + Top.setproperty!(ssa0, :c, d) + maybe_unused(d) + end + ) +end + +@testset "Index notation" begin + @testset "getindex" begin + # Indexing + @test_desugar a[i] Top.getindex(a, i) + @test_desugar a[i,j] Top.getindex(a, i, j) + # Indexing with `end` + @test_desugar a[end] Top.getindex(a, Top.lastindex(a)) + @test_desugar a[i,end] Top.getindex(a, i, Top.lastindex(a,2)) + # Nesting of `end` + @test_desugar a[b[end] + end] Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) + @test_desugar a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) + # Interaction of end with splatting + @test_desugar(a[I..., end], + Core._apply(Top.getindex, Core.tuple(a), I, + Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(I))))) + ) + end + + @testset "setindex!" begin + # setindex! + @test_desugar( + a[i] = b, + begin + Top.setindex!(a, b, i) + maybe_unused(b) + end + ) + @test_desugar( + a[i,end] = b+c, + begin + ssa0 = b+c + Top.setindex!(a, ssa0, i, Top.lastindex(a,2)) + maybe_unused(ssa0) + end + ) + end +end + +@testset "Array notation" begin + @testset "Literals" begin + @test_desugar [a,b] Top.vect(a,b) + @test_desugar T[a,b] Top.getindex(T, a,b) + end + + @testset "Concatenation" begin + @test_desugar [a b] Top.hcat(a,b) + @test_desugar [a; b] Top.vcat(a,b) + @test_desugar T[a b] Top.typed_hcat(T, a,b) + @test_desugar T[a; b] Top.typed_vcat(T, a,b) + @test_desugar [a b; c] Top.hvcat(Core.tuple(2,1), a, b, c) + @test_desugar T[a b; c] Top.typed_hvcat(T, Core.tuple(2,1), a, b, c) + end +end + +@testset "Splatting" begin + @test_desugar( + f(i, j, v..., k), + Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) + ) +end + +@testset "Comparison chains" begin + @test_desugar( + a < b < c, + if a < b + b < c + else + false + end + ) + # Nested + @test_desugar( + a < b > d <= e, + if a < b + if b > d + d <= e + else + false + end + else + false + end + ) + # Subexpressions + @test_desugar( + a < b+c < d, + if (ssa0 = b+c; a < ssa0) + ssa0 < d + else + false + end + ) + + # Interaction with broadcast syntax + @test_desugar( + a < b .< c, + Top.materialize(Top.broadcasted(&, a < b, Top.broadcasted(<, b, c))) + ) + @test_desugar( + a .< b+c < d, + Top.materialize(Top.broadcasted(&, + begin + ssa0 = b+c + # Is this a bug? + Top.materialize(Top.broadcasted(<, a, ssa0)) + end, + ssa0 < d)) + ) + @test_desugar( + a < b+c .< d, + Top.materialize(Top.broadcasted(&, + begin + ssa0 = b+c + a < ssa0 + end, + Top.broadcasted(<, ssa0, d))) + ) +end + From 332fa1ba4250bf43db9deec2f3227d5d5d87d1cc Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 2 Jun 2019 16:17:12 +1000 Subject: [PATCH 02/56] Lowering tests Tools: * Renumber gensyms * macro to test desugaring errors * catch flisp exceptions Tests: * short circuit * ternary * adjoint * broadcast * keyword args * inplace update operators * destructuring assignment --- src/ast.c | 24 ++- src/jlfrontend.scm | 6 + src/julia-syntax.scm | 2 +- test/compiler/lowering.jl | 315 ++++++++++++++++++++++++++++++-------- 4 files changed, 280 insertions(+), 67 deletions(-) diff --git a/src/ast.c b/src/ast.c index d5adfb54cc83e..e713123e7ebd6 100644 --- a/src/ast.c +++ b/src/ast.c @@ -919,20 +919,34 @@ jl_value_t *jl_call_scm_on_ast(const char *funcname, jl_value_t *expr, jl_module return result; } -// Call given flisp function on an Expr, converting result back to an Expr. -// Does not expand lambdas and thunks to code info objects. -JL_DLLEXPORT jl_value_t *jl_call_scm_on_ast_formonly(const char *funcname, jl_value_t *expr, jl_module_t *inmodule) +// Debug tool: Call given flisp function on an Expr, converting result back to +// an Expr. Does not expand lambdas and thunks to code info objects. +JL_DLLEXPORT jl_value_t *jl_call_scm_on_ast_formonly(const char *funcname, jl_value_t *expr, + jl_module_t *inmodule) { jl_ast_context_t *ctx = jl_ast_ctx_enter(); fl_context_t *fl_ctx = &ctx->fl; JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule); value_t arg = julia_to_scm(fl_ctx, expr); - value_t e = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, funcname)), arg); - jl_value_t *result = scm_to_julia_(fl_ctx, e, inmodule, 1); + value_t e = fl_applyn(fl_ctx, 2, + symbol_value(symbol(fl_ctx, "jl-apply-with-error-wrap")), + symbol_value(symbol(fl_ctx, funcname)), arg); + jl_value_t *result = NULL; JL_GC_PUSH1(&result); + int err = 0; + JL_TRY { + result = scm_to_julia_(fl_ctx, e, inmodule, 1); + } + JL_CATCH { + err = 1; + goto finally; // skip pop_exception + } +finally: JL_AST_PRESERVE_POP(ctx, old_roots); jl_ast_ctx_leave(ctx); JL_GC_POP(); + if (err) + jl_rethrow(); return result; } diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index 5fb59411d465c..5d363f323c80a 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -142,6 +142,12 @@ (error-wrap (lambda () (julia-expand-macroscope expr)))) +;; Debug tool: call named frontend function, translating flisp errors to AST +;; to avoid crashing +(define (jl-apply-with-error-wrap f . args) + (error-wrap (lambda () + (apply f args)))) + ;; construct default definitions of `eval` for non-bare modules ;; called by jl_eval_module_expr (define (module-default-defs e) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index c7270734123be..9ff54847c45a2 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -2210,7 +2210,7 @@ (if (any assignment? a) (error (string "misplaced assignment statement in \"" (deparse e) "\""))) (if (has-parameters? a) - (error "unexpected semicolon in array expression") + (error "unexpected semicolon in array expression") ;; Obsolete? (expand-forms (if (any (lambda (x) (and (pair? x) (eq? (car x) 'row))) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 8501fca274c1b..ce722ccef2d49 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -5,33 +5,46 @@ function fl_expand_forms(ex) ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "expand-forms", ex, Main) end -function lift_lowered_expr!(ex, nextid, valmap, lift_full) +function lift_lowered_expr!(ex, nextids, valmap, lift_full) if ex isa SSAValue # Rename SSAValues into renumbered symbols return get!(valmap, ex) do - newid = nextid[] - nextid[] = newid+1 + newid = nextids[1] + nextids[1] = newid+1 Symbol("ssa$newid") end end + if ex isa Symbol + name = string(ex) + if startswith(name, "#s") + return get!(valmap, ex) do + newid = nextids[2] + nextids[2] = newid+1 + Symbol("gsym$newid") + end + end + end if ex isa Expr filter!(e->!(e isa LineNumberNode), ex.args) if ex.head == :block && length(ex.args) == 1 # Remove trivial blocks - return lift_lowered_expr!(ex.args[1], nextid, valmap, lift_full) + return lift_lowered_expr!(ex.args[1], nextids, valmap, lift_full) end map!(ex.args, ex.args) do e - lift_lowered_expr!(e, nextid, valmap, lift_full) + lift_lowered_expr!(e, nextids, valmap, lift_full) end - if lift_full && ex.head == :top - return Expr(:(.), :Top, QuoteNode(ex.args[1])) - end - if lift_full && ex.head == :core - return Expr(:(.), :Core, QuoteNode(ex.args[1])) - end - if lift_full && ex.head == :unnecessary - # Rename to something we can actually type - return Expr(:call, :maybe_unused, ex.args...) + if lift_full + # Lift exotic Expr heads into standard julia syntax for ease in + # writing test case expressions. + if ex.head == :top || ex.head == :core + # Special global refs renamed to look like modules + newhead = ex.head == :top ? :Top : :Core + return Expr(:(.), newhead, QuoteNode(ex.args[1])) + elseif ex.head == :unnecessary + # `unnecessary` marks expressions generated by lowering that + # do not need to be evaluated if their value is unused. + return Expr(:call, :maybe_unused, ex.args...) + end end end return ex @@ -45,9 +58,8 @@ hand * Remove trivial blocks """ function lift_lowered_expr(ex; lift_full=false) - valmap = Dict{SSAValue,Symbol}() - nextid = Ref(0) - lift_lowered_expr!(deepcopy(ex), nextid, valmap, lift_full) + valmap = Dict{Union{Symbol,SSAValue},Symbol}() + lift_lowered_expr!(deepcopy(ex), ones(Int,2), valmap, lift_full) end """ @@ -61,9 +73,10 @@ desugared forms. function lower_ref_expr!(ex) if ex isa Expr filter!(e->!(e isa LineNumberNode), ex.args) + map!(lower_ref_expr!, ex.args, ex.args) if ex.head == :block && length(ex.args) == 1 # Remove trivial blocks - return lower_ref_expr!(ex.args[1]) + return ex.args[1] end if ex.head == :(.) && length(ex.args) >= 1 && (ex.args[1] == :Top || ex.args[1] == :Core) @@ -73,7 +86,6 @@ function lower_ref_expr!(ex) if ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused return Expr(:unnecessary, ex.args[2:end]...) end - map!(lower_ref_expr!, ex.args, ex.args) end return ex end @@ -132,6 +144,8 @@ macro test_desugar(input, ref) ref = lower_ref_expr($(QuoteNode(ref))) @test input == ref if input != ref + # Kinda crude. Would be better if Test supported custom/more + # capable diffing for failed tests. println("Diff dump:") diffdump(input, ref) end @@ -142,28 +156,41 @@ macro test_desugar(input, ref) ex end +macro test_desugar_error(input, msg) + ex = quote + input = lift_lowered_expr(fl_expand_forms($(QuoteNode(input)))) + @test input == Expr(:error, $msg) + end + # Attribute the test to the correct line number + @assert ex.args[4].args[1] == Symbol("@test") + ex.args[4].args[2] = __source__ + ex +end + +#------------------------------------------------------------------------------- +# Tests + @testset "Property notation" begin - @test_desugar(a.b, Top.getproperty(a, :b)) - @test_desugar(a.b.c, Top.getproperty(Top.getproperty(a, :b), :c)) + @test_desugar a.b Top.getproperty(a, :b) + @test_desugar a.b.c Top.getproperty(Top.getproperty(a, :b), :c) - @test_desugar( - a.b = c, + @test_desugar(a.b = c, begin Top.setproperty!(a, :b, c) maybe_unused(c) end ) - @test_desugar( - a.b.c = d, + @test_desugar(a.b.c = d, begin - ssa0 = Top.getproperty(a, :b) - Top.setproperty!(ssa0, :c, d) + ssa1 = Top.getproperty(a, :b) + Top.setproperty!(ssa1, :c, d) maybe_unused(d) end ) end @testset "Index notation" begin + # (process-indices) (partially-expand-ref) @testset "getindex" begin # Indexing @test_desugar a[i] Top.getindex(a, i) @@ -172,30 +199,31 @@ end @test_desugar a[end] Top.getindex(a, Top.lastindex(a)) @test_desugar a[i,end] Top.getindex(a, i, Top.lastindex(a,2)) # Nesting of `end` + @test_desugar a[[end]] Top.getindex(a, Top.vect(Top.lastindex(a))) @test_desugar a[b[end] + end] Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) @test_desugar a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) - # Interaction of end with splatting + # Interaction of `end` with splatting @test_desugar(a[I..., end], Core._apply(Top.getindex, Core.tuple(a), I, Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(I))))) ) + + @test_desugar_error a[i,j;k] "unexpected semicolon in array expression" end @testset "setindex!" begin - # setindex! - @test_desugar( - a[i] = b, + # (lambda in expand-table) + @test_desugar(a[i] = b, begin Top.setindex!(a, b, i) maybe_unused(b) end ) - @test_desugar( - a[i,end] = b+c, + @test_desugar(a[i,end] = b+c, begin - ssa0 = b+c - Top.setindex!(a, ssa0, i, Top.lastindex(a,2)) - maybe_unused(ssa0) + ssa1 = b+c + Top.setindex!(a, ssa1, i, Top.lastindex(a,2)) + maybe_unused(ssa1) end ) end @@ -204,29 +232,34 @@ end @testset "Array notation" begin @testset "Literals" begin @test_desugar [a,b] Top.vect(a,b) - @test_desugar T[a,b] Top.getindex(T, a,b) + @test_desugar T[a,b] Top.getindex(T, a,b) # Only so much syntax to go round :-/ + @test_desugar_error [a,b;c] "unexpected semicolon in array expression" + @test_desugar_error [a=b,c] "misplaced assignment statement in \"[a = b, c]\"" end @testset "Concatenation" begin + # (lambda in expand-table) @test_desugar [a b] Top.hcat(a,b) @test_desugar [a; b] Top.vcat(a,b) @test_desugar T[a b] Top.typed_hcat(T, a,b) @test_desugar T[a; b] Top.typed_vcat(T, a,b) @test_desugar [a b; c] Top.hvcat(Core.tuple(2,1), a, b, c) @test_desugar T[a b; c] Top.typed_hvcat(T, Core.tuple(2,1), a, b, c) + + @test_desugar_error [a b=c] "misplaced assignment statement in \"[a b = c]\"" + @test_desugar_error [a; b=c] "misplaced assignment statement in \"[a; b = c]\"" + @test_desugar_error T[a b=c] "misplaced assignment statement in \"T[a b = c]\"" + @test_desugar_error T[a; b=c] "misplaced assignment statement in \"T[a; b = c]\"" end end @testset "Splatting" begin - @test_desugar( - f(i, j, v..., k), - Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) - ) + @test_desugar f(i,j,v...,k) Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) end @testset "Comparison chains" begin - @test_desugar( - a < b < c, + # (expand-compare-chain) + @test_desugar(a < b < c, if a < b b < c else @@ -234,8 +267,7 @@ end end ) # Nested - @test_desugar( - a < b > d <= e, + @test_desugar(a < b > d <= e, if a < b if b > d d <= e @@ -247,38 +279,199 @@ end end ) # Subexpressions - @test_desugar( - a < b+c < d, - if (ssa0 = b+c; a < ssa0) - ssa0 < d + @test_desugar(a < b+c < d, + if (ssa1 = b+c; a < ssa1) + ssa1 < d else false end ) # Interaction with broadcast syntax - @test_desugar( - a < b .< c, + @test_desugar(a < b .< c, Top.materialize(Top.broadcasted(&, a < b, Top.broadcasted(<, b, c))) ) - @test_desugar( - a .< b+c < d, + @test_desugar(a .< b+c < d, Top.materialize(Top.broadcasted(&, begin - ssa0 = b+c + ssa1 = b+c # Is this a bug? - Top.materialize(Top.broadcasted(<, a, ssa0)) + Top.materialize(Top.broadcasted(<, a, ssa1)) end, - ssa0 < d)) + ssa1 < d)) ) - @test_desugar( - a < b+c .< d, + @test_desugar(a < b+c .< d, Top.materialize(Top.broadcasted(&, begin - ssa0 = b+c - a < ssa0 + ssa1 = b+c + a < ssa1 end, - Top.broadcasted(<, ssa0, d))) + Top.broadcasted(<, ssa1, d))) + ) +end + +@testset "Short circuit , ternary" begin + # (expand-or) (expand-and) + @test_desugar a || b if a; a else b end + @test_desugar a && b if a; b else false end + @test_desugar a ? x : y if a; x else y end +end + +@testset "Adjoint" begin + @test_desugar a' Top.adjoint(a) +end + +@testset "Broadcast" begin + # Basic + @test_desugar x .+ y Top.materialize(Top.broadcasted(+, x, y)) + @test_desugar f.(x) Top.materialize(Top.broadcasted(f, x)) + # Fusing + @test_desugar f.(x) .+ g.(y) Top.materialize(Top.broadcasted(+, Top.broadcasted(f, x), + Top.broadcasted(g, y))) + # Keywords don't participate + @test_desugar(f.(x, a=1), + Top.materialize( + begin + ssa1 = Top.broadcasted_kwsyntax + ssa2 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) + Core.kwfunc(ssa1)(ssa2, ssa1, f, x) + end + ) ) + # Nesting + @test_desugar f.(g(x)) Top.materialize(Top.broadcasted(f, g(x))) + @test_desugar f.(g(h.(x))) Top.materialize(Top.broadcasted(f, + g(Top.materialize(Top.broadcasted(h, x))))) + + # In place + @test_desugar x .= a Top.materialize!(x, Top.broadcasted(Top.identity, a)) + @test_desugar x .= f.(a) Top.materialize!(x, Top.broadcasted(f, a)) + @test_desugar x .+= a Top.materialize!(x, Top.broadcasted(+, x, a)) +end + +@testset "Keyword arguments" begin + @test_desugar( + f(x,a=1), + begin + ssa1 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) + Core.kwfunc(f)(ssa1, f, x) + end + ) +end + +@testset "In place update operators" begin + # (lower-update-op) + @test_desugar x += a x = x+a + @test_desugar x::Int += a x = x::Int + a + @test_desugar(x[end] += a, + begin + ssa1 = Top.lastindex(x) + begin + ssa2 = Top.getindex(x, ssa1) + a + Top.setindex!(x, ssa2, ssa1) + maybe_unused(ssa2) + end + end + ) + @test_desugar(x[f(y)] += a, + begin + ssa1 = f(y) + begin + ssa2 = Top.getindex(x, ssa1) + a + Top.setindex!(x, ssa2, ssa1) + maybe_unused(ssa2) + end + end + ) + @test_desugar((x,y) .+= a, + begin + ssa1 = Core.tuple(x, y) + Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) + end + ) + @test_desugar([x y] .+= a, + begin + ssa1 = Top.hcat(x, y) + Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) + end + ) + # TODO @test_desugar (x+y) += 1 Error +end + +@testset "Assignment" begin + # (lambda in expand-table) + + # Assignment chain; nontrivial rhs + @test_desugar(x = y = f(a), + begin + ssa1 = f(a) + y = ssa1 + x = ssa1 + maybe_unused(ssa1) + end + ) + + @testset "Multiple Assignemnt" begin + # Simple multiple assignment exact match + @test_desugar((x,y) = (a,b), + begin + x = a + y = b + maybe_unused(Core.tuple(a,b)) + end + ) + # Destructuring + @test_desugar((x,y) = a, + begin + begin + ssa1 = Top.indexed_iterate(a, 1) + x = Core.getfield(ssa1, 1) + gsym1 = Core.getfield(ssa1, 2) + ssa1 + end + begin + ssa2 = Top.indexed_iterate(a, 2, gsym1) + y = Core.getfield(ssa2, 1) + ssa2 + end + maybe_unused(a) + end + ) + # Nested destructuring + @test_desugar((x,(y,z)) = a, + begin + begin + ssa1 = Top.indexed_iterate(a, 1) + x = Core.getfield(ssa1, 1) + gsym1 = Core.getfield(ssa1, 2) + ssa1 + end + begin + ssa2 = Top.indexed_iterate(a, 2, gsym1) + begin + ssa3 = Core.getfield(ssa2, 1) + begin + ssa4 = Top.indexed_iterate(ssa3, 1) + y = Core.getfield(ssa4, 1) + gsym2 = Core.getfield(ssa4, 2) + ssa4 + end + begin + ssa5 = Top.indexed_iterate(ssa3, 2, gsym2) + z = Core.getfield(ssa5, 1) + ssa5 + end + maybe_unused(ssa3) + end + ssa2 + end + maybe_unused(a) + end + ) + end + + @test_desugar_error 1=a "invalid assignment location \"1\"" + @test_desugar_error true=a "invalid assignment location \"true\"" + @test_desugar_error "str"=a "invalid assignment location \"\"str\"\"" end From 45eee71393d659640b72959fb666a17892e0f3d7 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 2 Jun 2019 19:55:21 +1000 Subject: [PATCH 03/56] Support interpolating aribtrary Exprs as reference ASTs --- test/compiler/lowering.jl | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index ce722ccef2d49..4da5f0621e951 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -76,15 +76,22 @@ function lower_ref_expr!(ex) map!(lower_ref_expr!, ex.args, ex.args) if ex.head == :block && length(ex.args) == 1 # Remove trivial blocks - return ex.args[1] + return lower_ref_expr!(ex.args[1]) end + # Translate a selection of special expressions into the exotic Expr + # heads used in lowered code. if ex.head == :(.) && length(ex.args) >= 1 && (ex.args[1] == :Top || ex.args[1] == :Core) (length(ex.args) == 2 && ex.args[2] isa QuoteNode) || throw("Unexpected top/core expression $(sprint(dump, ex))") return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) - end - if ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused + elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused return Expr(:unnecessary, ex.args[2:end]...) + elseif ex.head == :$ && length(ex.args) == 1 && ex.args[1] isa Expr && + ex.args[1].head == :call && ex.args[1].args[1] == :Expr + # Expand exprs of form $(Expr(head, ...)) + return Expr(map(q->q.value, ex.args[1].args[2:end])...) + elseif ex.head == :macrocall + # TODO heads for blocks end end return ex @@ -470,6 +477,13 @@ end ) end + @test_desugar(x::T = a, + begin + $(Expr(:decl, :x, :T)) + x = a + end + ) + @test_desugar_error 1=a "invalid assignment location \"1\"" @test_desugar_error true=a "invalid assignment location \"true\"" @test_desugar_error "str"=a "invalid assignment location \"\"str\"\"" From 34e9a56de86ed4e8dcfac5007695d687705a5c48 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Mon, 3 Jun 2019 08:15:03 +1000 Subject: [PATCH 04/56] Start testing loop structures + Test assignment errors --- src/julia-syntax.scm | 2 +- test/compiler/lowering.jl | 53 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 9ff54847c45a2..083059ebee694 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1910,7 +1910,7 @@ (rr (if (or (symbol-like? rhs) (atom? rhs)) rhs (make-ssavalue)))) `(block ,.(if (eq? aa a) '() `((= ,aa ,(expand-forms a)))) - ,.(if (eq? bb b) '() `((= ,bb ,(expand-forms b)))) + ,.(if (eq? bb b) '() `((= ,bb ,(expand-forms b)))) ;; Obsolete ? ,.(if (eq? rr rhs) '() `((= ,rr ,(expand-forms rhs)))) (call (top setproperty!) ,aa ,bb ,rr) (unnecessary ,rr))))) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 4da5f0621e951..9f961c8065b4c 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -15,8 +15,9 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) end end if ex isa Symbol + # Rename gensyms name = string(ex) - if startswith(name, "#s") + if startswith(name, "#") return get!(valmap, ex) do newid = nextids[2] nextids[2] = newid+1 @@ -324,8 +325,12 @@ end @test_desugar a ? x : y if a; x else y end end -@testset "Adjoint" begin - @test_desugar a' Top.adjoint(a) +@testset "Misc operators" begin + @test_desugar a' Top.adjoint(a) + # <: and >: are special Expr heads which need to be turned into call when + # used as operators + @test_desugar a <: b $(Expr(:call, :(<:), :a, :b)) + @test_desugar a >: b $(Expr(:call, :(>:), :a, :b)) end @testset "Broadcast" begin @@ -484,8 +489,50 @@ end end ) + # Invalid assignments @test_desugar_error 1=a "invalid assignment location \"1\"" @test_desugar_error true=a "invalid assignment location \"true\"" @test_desugar_error "str"=a "invalid assignment location \"\"str\"\"" + @test_desugar_error [x y]=c "invalid assignment location \"[x y]\"" + @test_desugar_error a[x y]=c "invalid spacing in left side of indexed assignment" + @test_desugar_error a[x;y]=c "unexpected \";\" in left side of indexed assignment" + @test_desugar_error [x;y]=c "use \"(a, b) = ...\" to assign multiple values" + + # Old deprecation (6575e12ba46) + @test_desugar_error x.(y)=c "invalid syntax \"x.(y) = ...\"" end +@testset "For loops" begin + #= + @test_desugar( + for i in a + body + end, + begin + @break_block :loop_exit begin + ssa1 = a + gsym1 = Top.iterate(ssa1) + if Top.not_int(Core.:(===)(gsym1, nothing)) + @_do_while begin + @break_block :loop_cont begin + @scope_block begin + local i + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + body + end + end + gsym1 = Top.iterate(ssa1, ssa3) + end begin + Top.not_int(Core.:(===)(gsym1, nothing)) + end + end + end + end + ) + =# +end From 4e8ece3fc3ab83f9bf9841a42c7d45e6f4dbfe09 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 4 Jun 2019 08:15:28 +1000 Subject: [PATCH 05/56] Start testing loops & functions --- test/compiler/lowering.jl | 160 ++++++++++++++++++++++++++++---------- 1 file changed, 117 insertions(+), 43 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 9f961c8065b4c..8dfedd5616f64 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -87,12 +87,6 @@ function lower_ref_expr!(ex) return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused return Expr(:unnecessary, ex.args[2:end]...) - elseif ex.head == :$ && length(ex.args) == 1 && ex.args[1] isa Expr && - ex.args[1].head == :call && ex.args[1].args[1] == :Expr - # Expand exprs of form $(Expr(head, ...)) - return Expr(map(q->q.value, ex.args[1].args[2:end])...) - elseif ex.head == :macrocall - # TODO heads for blocks end end return ex @@ -134,7 +128,7 @@ function diffdump(ex1, ex2; maxdepth=20) end # For interactive convenience in constructing test cases with flisp based lowering -desugar(ex) = lift_lowered_expr(fl_expand_forms(ex); lift_full=true) +desugar(ex; lift_full=true) = lift_lowered_expr(fl_expand_forms(ex); lift_full=lift_full) macro desugar(ex) quote @@ -148,8 +142,8 @@ reference expression `ref`. """ macro test_desugar(input, ref) ex = quote - input = lift_lowered_expr(fl_expand_forms($(QuoteNode(input)))) - ref = lower_ref_expr($(QuoteNode(ref))) + input = lift_lowered_expr(fl_expand_forms($(Expr(:quote, input)))) + ref = lower_ref_expr($(Expr(:quote, ref))) @test input == ref if input != ref # Kinda crude. Would be better if Test supported custom/more @@ -166,7 +160,7 @@ end macro test_desugar_error(input, msg) ex = quote - input = lift_lowered_expr(fl_expand_forms($(QuoteNode(input)))) + input = lift_lowered_expr(fl_expand_forms($(Expr(:quote, input)))) @test input == Expr(:error, $msg) end # Attribute the test to the correct line number @@ -327,8 +321,8 @@ end @testset "Misc operators" begin @test_desugar a' Top.adjoint(a) - # <: and >: are special Expr heads which need to be turned into call when - # used as operators + # <: and >: are special Expr heads which need to be turned into Expr(:call) + # when used as operators @test_desugar a <: b $(Expr(:call, :(<:), :a, :b)) @test_desugar a >: b $(Expr(:call, :(>:), :a, :b)) end @@ -407,7 +401,7 @@ end Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) end ) - # TODO @test_desugar (x+y) += 1 Error + @test_desugar_error (x+y) += 1 "invalid assignment location \"(x + y)\"" end @testset "Assignment" begin @@ -502,37 +496,117 @@ end @test_desugar_error x.(y)=c "invalid syntax \"x.(y) = ...\"" end -@testset "For loops" begin - #= - @test_desugar( - for i in a - body - end, +@testset "Loops" begin + @test_desugar(while cond + body1 + continue + body2 + break + body3 + end, + $(Expr(Symbol("break-block"), Symbol("loop-exit"), + Expr(:_while, :cond, + Expr(Symbol("break-block"), Symbol("loop-cont"), + Expr(Symbol("scope-block"), quote + body1 + $(Expr(:break, Symbol("loop-cont"))) + body2 + $(Expr(:break, Symbol("loop-exit"))) + body3 + end + ) + ) + ) + )) + ) + + @test_desugar(for i = a + body1 + continue + body2 + break + end, + $(Expr(Symbol("break-block"), Symbol("loop-exit"), + quote + ssa1 = a + gsym1 = Top.iterate(ssa1) + if Top.not_int(Core.:(===)(gsym1, $nothing)) + $(Expr(:_do_while, + quote + $(Expr(Symbol("break-block"), Symbol("loop-cont"), + Expr(Symbol("scope-block"), + quote + local i + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + begin + body1 + $(Expr(:break, Symbol("loop-cont"))) + body2 + $(Expr(:break, Symbol("loop-exit"))) + end + end))) + gsym1 = Top.iterate(ssa1, ssa3) + end, + :(Top.not_int(Core.:(===)(gsym1, $nothing))))) + end + end)) + ) +end + +@testset "Functions" begin + # Short form + @test_desugar(f(x) = x, begin - @break_block :loop_exit begin - ssa1 = a - gsym1 = Top.iterate(ssa1) - if Top.not_int(Core.:(===)(gsym1, nothing)) - @_do_while begin - @break_block :loop_cont begin - @scope_block begin - local i - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - body - end - end - gsym1 = Top.iterate(ssa1, ssa3) - end begin - Top.not_int(Core.:(===)(gsym1, nothing)) - end - end - end + $(Expr(:method, :f)) + $(Expr(:method, + :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, + Expr(Symbol("#self#"), :x), + Any[], + Expr(Symbol("scope-block"), + :x)))) + maybe_unused(f) + end + ) + # Long form with argument annotations + @test_desugar(function f(x,y) + body + end, + begin + $(Expr(:method, :f)) + $(Expr(:method, + :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec())), + Expr(:lambda, + Expr(Symbol("#self#"), :x, :y), + Any[], + Expr(Symbol("scope-block"), + :body)))) + maybe_unused(f) + end + ) + # Return type + @test_desugar(f(x)::T = x, + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, + Expr(Symbol("#self#"), :x), + Any[], + Expr(Symbol("scope-block"), + quote + ssa1 = T + $(Expr(:meta, Symbol("ret-type"), :ssa1)) + x + end)))) + maybe_unused(f) end ) - =# end From 827bfb135a3a6b13f99ca5c244842f996c7f7f60 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 4 Jun 2019 15:35:05 +1000 Subject: [PATCH 06/56] More test for lowering of functions, declarations, tuples --- src/julia-syntax.scm | 2 +- test/compiler/lowering.jl | 285 +++++++++++++++++++++++++++++++++----- 2 files changed, 248 insertions(+), 39 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 083059ebee694..3acae00674a38 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1279,7 +1279,7 @@ (assigned-name (cadr e))) (else e))) -;; local x, y=2, z => local x;local y;local z;y = 2 +;; local x, (y=2), z => local x;local y;local z;y = 2 (define (expand-decls what binds const?) (if (not (list? binds)) (error (string "invalid \"" what "\" declaration"))) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 8dfedd5616f64..9ef77c2a50d69 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -5,6 +5,9 @@ function fl_expand_forms(ex) ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "expand-forms", ex, Main) end +# Make it easy to replace fl_expand_forms with a julia version in the future. +expand_forms = ex->fl_expand_forms(ex) + function lift_lowered_expr!(ex, nextids, valmap, lift_full) if ex isa SSAValue # Rename SSAValues into renumbered symbols @@ -15,6 +18,10 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) end end if ex isa Symbol + # Rename special names + if ex == Symbol("#self#") + return :_self_ + end # Rename gensyms name = string(ex) if startswith(name, "#") @@ -34,6 +41,10 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) map!(ex.args, ex.args) do e lift_lowered_expr!(e, nextids, valmap, lift_full) end + # FIXME: This translation of _self_ is an error in converting lamba exprs. + if ex.head == Symbol("#self#") + ex = Expr(:_self_, ex.args...) + end if lift_full # Lift exotic Expr heads into standard julia syntax for ease in # writing test case expressions. @@ -83,7 +94,9 @@ function lower_ref_expr!(ex) # heads used in lowered code. if ex.head == :(.) && length(ex.args) >= 1 && (ex.args[1] == :Top || ex.args[1] == :Core) - (length(ex.args) == 2 && ex.args[2] isa QuoteNode) || throw("Unexpected top/core expression $(sprint(dump, ex))") + if !(length(ex.args) == 2 && ex.args[2] isa QuoteNode) + throw("Unexpected top/core expression $(sprint(dump, ex))") + end return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused return Expr(:unnecessary, ex.args[2:end]...) @@ -128,11 +141,16 @@ function diffdump(ex1, ex2; maxdepth=20) end # For interactive convenience in constructing test cases with flisp based lowering -desugar(ex; lift_full=true) = lift_lowered_expr(fl_expand_forms(ex); lift_full=lift_full) +desugar(ex; lift_full=true) = lift_lowered_expr(expand_forms(ex); lift_full=lift_full) -macro desugar(ex) +""" + @desugar ex [kws...] + +Convenience macro, equivalent to `desugar(:(ex), kws...)`. +""" +macro desugar(ex, kws...) quote - desugar($(QuoteNode(ex))) + desugar($(Expr(:quote, ex)); $(map(esc, kws)...)) end end @@ -142,11 +160,11 @@ reference expression `ref`. """ macro test_desugar(input, ref) ex = quote - input = lift_lowered_expr(fl_expand_forms($(Expr(:quote, input)))) + input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) ref = lower_ref_expr($(Expr(:quote, ref))) @test input == ref if input != ref - # Kinda crude. Would be better if Test supported custom/more + # Kinda crude. Would be much neater if Test supported custom/more # capable diffing for failed tests. println("Diff dump:") diffdump(input, ref) @@ -160,7 +178,7 @@ end macro test_desugar_error(input, msg) ex = quote - input = lift_lowered_expr(fl_expand_forms($(Expr(:quote, input)))) + input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) @test input == Expr(:error, $msg) end # Attribute the test to the correct line number @@ -173,6 +191,7 @@ end # Tests @testset "Property notation" begin + # flisp: (expand-fuse-broadcast) @test_desugar a.b Top.getproperty(a, :b) @test_desugar a.b.c Top.getproperty(Top.getproperty(a, :b), :c) @@ -192,18 +211,18 @@ end end @testset "Index notation" begin - # (process-indices) (partially-expand-ref) + # flisp: (process-indices) (partially-expand-ref) @testset "getindex" begin # Indexing - @test_desugar a[i] Top.getindex(a, i) - @test_desugar a[i,j] Top.getindex(a, i, j) + @test_desugar a[i] Top.getindex(a, i) + @test_desugar a[i,j] Top.getindex(a, i, j) # Indexing with `end` - @test_desugar a[end] Top.getindex(a, Top.lastindex(a)) - @test_desugar a[i,end] Top.getindex(a, i, Top.lastindex(a,2)) + @test_desugar a[end] Top.getindex(a, Top.lastindex(a)) + @test_desugar a[i,end] Top.getindex(a, i, Top.lastindex(a,2)) # Nesting of `end` @test_desugar a[[end]] Top.getindex(a, Top.vect(Top.lastindex(a))) - @test_desugar a[b[end] + end] Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) - @test_desugar a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) + @test_desugar a[b[end] + end] Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) + @test_desugar a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) # Interaction of `end` with splatting @test_desugar(a[I..., end], Core._apply(Top.getindex, Core.tuple(a), I, @@ -214,7 +233,7 @@ end end @testset "setindex!" begin - # (lambda in expand-table) + # flisp: (lambda in expand-table) @test_desugar(a[i] = b, begin Top.setindex!(a, b, i) @@ -240,7 +259,7 @@ end end @testset "Concatenation" begin - # (lambda in expand-table) + # flisp: (lambda in expand-table) @test_desugar [a b] Top.hcat(a,b) @test_desugar [a; b] Top.vcat(a,b) @test_desugar T[a b] Top.typed_hcat(T, a,b) @@ -255,12 +274,17 @@ end end end +@testset "Tuples" begin + @test_desugar (x,y) Core.tuple(x,y) + @test_desugar (x=a,y=b) Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) +end + @testset "Splatting" begin @test_desugar f(i,j,v...,k) Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) end @testset "Comparison chains" begin - # (expand-compare-chain) + # flisp: (expand-compare-chain) @test_desugar(a < b < c, if a < b b < c @@ -313,7 +337,7 @@ end end @testset "Short circuit , ternary" begin - # (expand-or) (expand-and) + # flisp: (expand-or) (expand-and) @test_desugar a || b if a; a else b end @test_desugar a && b if a; b else false end @test_desugar a ? x : y if a; x else y end @@ -366,7 +390,7 @@ end end @testset "In place update operators" begin - # (lower-update-op) + # flisp: (lower-update-op) @test_desugar x += a x = x+a @test_desugar x::Int += a x = x::Int + a @test_desugar(x[end] += a, @@ -405,7 +429,7 @@ end end @testset "Assignment" begin - # (lambda in expand-table) + # flisp: (lambda in expand-table) # Assignment chain; nontrivial rhs @test_desugar(x = y = f(a), @@ -476,13 +500,6 @@ end ) end - @test_desugar(x::T = a, - begin - $(Expr(:decl, :x, :T)) - x = a - end - ) - # Invalid assignments @test_desugar_error 1=a "invalid assignment location \"1\"" @test_desugar_error true=a "invalid assignment location \"true\"" @@ -496,6 +513,92 @@ end @test_desugar_error x.(y)=c "invalid syntax \"x.(y) = ...\"" end +@testset "Declarations" begin + # flisp: (expand-decls) (expand-local-or-global-decl) (expand-const-decl) + + # const + @test_desugar((const x=a), + begin + $(Expr(:const, :x)) # `const x` is invalid surface syntax + x = a + end + ) + @test_desugar((const x,y = a,b), + begin + $(Expr(:const, :x)) + $(Expr(:const, :y)) + begin + x = a + y = b + maybe_unused(Core.tuple(a,b)) + end + end + ) + + # local + @test_desugar((local x, y), + begin + local y + local x + end + ) + # Locals with initialization. Note parentheses are needed for this to parse + # as individual assignments rather than multiple assignment. + @test_desugar((local (x=a), (y=b), z), + begin + local z + local y + local x + x = a + y = b + end + ) + # Multiple assignment form + @test_desugar(begin + local x,y = a,b + end, + begin + local x + local y + begin + x = a + y = b + maybe_unused(Core.tuple(a,b)) + end + end + ) + + # global + @test_desugar((global x, (y=a)), + begin + global y + global x + y = a + end + ) + + # type decl + @test_desugar(x::T = a, + begin + $(Expr(:decl, :x, :T)) + x = a + end + ) + + # type aliases + @test_desugar(A{T} = B{T}, + begin + $(Expr(Symbol("const-if-global"), :A)) + A = $(Expr(Symbol("scope-block"), + quote + $(Expr(Symbol("local-def"), :T)) + T = Core.TypeVar(:T) + Core.UnionAll(T, Core.apply_type(B, T)) + end)) + end + ) +end + @testset "Loops" begin @test_desugar(while cond body1 @@ -507,7 +610,8 @@ end $(Expr(Symbol("break-block"), Symbol("loop-exit"), Expr(:_while, :cond, Expr(Symbol("break-block"), Symbol("loop-cont"), - Expr(Symbol("scope-block"), quote + Expr(Symbol("scope-block"), + quote body1 $(Expr(:break, Symbol("loop-cont"))) body2 @@ -567,46 +671,151 @@ end :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, - Expr(Symbol("#self#"), :x), + Expr(:_self_, :x), Any[], Expr(Symbol("scope-block"), :x)))) maybe_unused(f) end ) + # Long form with argument annotations - @test_desugar(function f(x,y) - body + @test_desugar(function f(x::T, y) + body(x) end, begin $(Expr(:method, :f)) $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec())), + :(Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec())), Expr(:lambda, - Expr(Symbol("#self#"), :x, :y), + Expr(:_self_, :x, :y), Any[], Expr(Symbol("scope-block"), - :body)))) + :(body(x)))))) maybe_unused(f) end ) - # Return type - @test_desugar(f(x)::T = x, + + # Default arguments + @test_desugar(function f(x=a, y=b) + body(x) + end, + begin + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f)), Core.svec())), + Expr(:lambda, + Expr(:_self_), + Any[], + Expr(Symbol("scope-block"), + :(_self_(a, b)))))) + maybe_unused(f) + end + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, + Expr(:_self_, :x), + Any[], + Expr(Symbol("scope-block"), + :(_self_(x, b)))))) + maybe_unused(f) + end + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec())), + Expr(:lambda, + Expr(:_self_, :x, :y), + Any[], + Expr(Symbol("scope-block"), + :(body(x)))))) + maybe_unused(f) + end + end + ) + + # Varargs + @test_desugar(function f(x, args...) + body + end, + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.apply_type(Vararg, Core.Any)), Core.svec())), + Expr(:lambda, + Expr(:_self_, :x, :args), + Any[], + Expr(Symbol("scope-block"), :body)))) + maybe_unused(f) + end + ) + + # Keyword args + # TODO + #= @desugar(function f(x; k1=v1, k2=v2) =# + #= body =# + #= end, =# + #= ) =# + + # Return type declaration + @test_desugar(function f(x)::T + body(x) + end, begin $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, - Expr(Symbol("#self#"), :x), + Expr(:_self_, :x), Any[], Expr(Symbol("scope-block"), quote ssa1 = T $(Expr(:meta, Symbol("ret-type"), :ssa1)) - x + body(x) end)))) maybe_unused(f) end ) + + # Anon functions + @test_desugar((x,y)->body(x,y), + begin + local gsym1 + begin + $(Expr(:method, :gsym1)) + $(Expr(:method, :gsym1, + :(Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec())), + Expr(:lambda, + Expr(:_self_, :x, :y), + Any[], + Expr(Symbol("scope-block"), + :(body(x, y)))))) + maybe_unused(gsym1) + end + end + ) + + # Invalid names + @test_desugar_error ccall(x)=body "invalid function name \"ccall\"" + @test_desugar_error cglobal(x)=body "invalid function name \"cglobal\"" + @test_desugar_error true(x)=body "invalid function name \"true\"" + @test_desugar_error false(x)=body "invalid function name \"false\"" +end + +@testset "Forms without desugaring" begin + # (expand-forms) + # The following Expr heads are currently not touched by desugaring + for head in [:quote, :top, :core, :globalref, :outerref, :module, :toplevel, :null, :meta, :using, :import, :export] + ex = Expr(head, Expr(:foobar, :junk, nothing, 42)) + @test expand_forms(ex) == ex + end + # flisp: inert,line have special representations on the julia side + @test expand_forms(QuoteNode(Expr(:$, :x))) == QuoteNode(Expr(:$, :x)) # flisp: `(inert ,expr) + @test expand_forms(LineNumberNode(1, :foo)) == LineNumberNode(1, :foo) # flisp: `(line ,line ,file) end + From f169b0c361ea36bf00480bf529f774fa6e19e1cc Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 4 Jun 2019 16:41:27 +1000 Subject: [PATCH 07/56] Fix translation of partially lowered lambdas --- src/ast.c | 22 ++++++++++++-- test/compiler/lowering.jl | 60 ++++++++++++++------------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/src/ast.c b/src/ast.c index e713123e7ebd6..1462e55f2f075 100644 --- a/src/ast.c +++ b/src/ast.c @@ -572,8 +572,26 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m jl_array_ptr_set(((jl_expr_t*)ex)->args, i, scm_to_julia_(fl_ctx, car_(e), mod, formonly)); e = cdr_(e); } - if (sym == lambda_sym && !formonly) - ex = (jl_value_t*)jl_new_code_info_from_ast((jl_expr_t*)ex); + if (sym == lambda_sym) { + if (formonly) { + if (jl_array_len(((jl_expr_t*)ex)->args) >= 1 && + jl_is_expr(jl_array_ptr_ref(((jl_expr_t*)ex)->args, 0))) { + // Hack: fixup translation of lambda arg names (only used in testing) + jl_expr_t *argexpr = (jl_expr_t*)jl_array_ptr_ref(((jl_expr_t*)ex)->args, 0); + size_t nargs = jl_array_len(argexpr->args) + 1; + jl_array_t *args = jl_alloc_vec_any(nargs); + JL_GC_PUSH1(&args); + jl_array_ptr_set(args, 0, argexpr->head); + for (size_t i = 1; i < nargs; ++i) + jl_array_ptr_set(args, i, jl_array_ptr_ref(argexpr->args, i-1)); + jl_array_ptr_set(((jl_expr_t*)ex)->args, 0, args); + JL_GC_POP(); + } + } + else { + ex = (jl_value_t*)jl_new_code_info_from_ast((jl_expr_t*)ex); + } + } JL_GC_POP(); if (sym == list_sym) return (jl_value_t*)((jl_expr_t*)ex)->args; diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 9ef77c2a50d69..5e9d9b4432252 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -18,7 +18,6 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) end end if ex isa Symbol - # Rename special names if ex == Symbol("#self#") return :_self_ end @@ -41,10 +40,6 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) map!(ex.args, ex.args) do e lift_lowered_expr!(e, nextids, valmap, lift_full) end - # FIXME: This translation of _self_ is an error in converting lamba exprs. - if ex.head == Symbol("#self#") - ex = Expr(:_self_, ex.args...) - end if lift_full # Lift exotic Expr heads into standard julia syntax for ease in # writing test case expressions. @@ -58,6 +53,10 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) return Expr(:call, :maybe_unused, ex.args...) end end + elseif ex isa Vector # Occasional case of lambdas + map!(ex, ex) do e + lift_lowered_expr!(e, nextids, valmap, lift_full) + end end return ex end @@ -664,17 +663,14 @@ end @testset "Functions" begin # Short form - @test_desugar(f(x) = x, + @test_desugar(f(x) = body(x), begin $(Expr(:method, :f)) - $(Expr(:method, - :f, + $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x), - Any[], + Expr(:lambda, [:_self_, :x], [], Expr(Symbol("scope-block"), - :x)))) + :(body(x)))))) maybe_unused(f) end ) @@ -685,12 +681,9 @@ end end, begin $(Expr(:method, :f)) - $(Expr(:method, - :f, + $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x, :y), - Any[], + Expr(:lambda, [:_self_, :x, :y], [], Expr(Symbol("scope-block"), :(body(x)))))) maybe_unused(f) @@ -699,16 +692,14 @@ end # Default arguments @test_desugar(function f(x=a, y=b) - body(x) + body(x,y) end, begin begin $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f)), Core.svec())), - Expr(:lambda, - Expr(:_self_), - Any[], + Expr(:lambda, [:_self_], [], Expr(Symbol("scope-block"), :(_self_(a, b)))))) maybe_unused(f) @@ -717,9 +708,7 @@ end $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x), - Any[], + Expr(:lambda, [:_self_, :x], [], Expr(Symbol("scope-block"), :(_self_(x, b)))))) maybe_unused(f) @@ -728,11 +717,9 @@ end $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x, :y), - Any[], + Expr(:lambda, [:_self_, :x, :y], [], Expr(Symbol("scope-block"), - :(body(x)))))) + :(body(x,y)))))) maybe_unused(f) end end @@ -740,16 +727,15 @@ end # Varargs @test_desugar(function f(x, args...) - body + body(x, args) end, begin $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.apply_type(Vararg, Core.Any)), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x, :args), - Any[], - Expr(Symbol("scope-block"), :body)))) + Expr(:lambda, [:_self_, :x, :args], [], + Expr(Symbol("scope-block"), + :(body(x, args)))))) maybe_unused(f) end ) @@ -769,9 +755,7 @@ end $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x), - Any[], + Expr(:lambda, [:_self_, :x], [], Expr(Symbol("scope-block"), quote ssa1 = T @@ -790,9 +774,7 @@ end $(Expr(:method, :gsym1)) $(Expr(:method, :gsym1, :(Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec())), - Expr(:lambda, - Expr(:_self_, :x, :y), - Any[], + Expr(:lambda, [:_self_, :x, :y], [], Expr(Symbol("scope-block"), :(body(x, y)))))) maybe_unused(gsym1) From cd57d4c5c19fca98078fb30d680fece0a17c07c0 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 4 Jun 2019 17:23:53 +1000 Subject: [PATCH 08/56] Test case: `for outer i = iter` --- test/compiler/lowering.jl | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 5e9d9b4432252..67d30c0078136 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -659,6 +659,38 @@ end end end)) ) + + # For loops with `outer` + @test_desugar(for outer i = a + body + end, + $(Expr(Symbol("break-block"), Symbol("loop-exit"), + quote + ssa1 = a + gsym1 = Top.iterate(ssa1) + $(Expr(Symbol("require-existing-local"), :i)) # Cf above. + if Top.not_int(Core.:(===)(gsym1, $nothing)) + $(Expr(:_do_while, + quote + $(Expr(Symbol("break-block"), Symbol("loop-cont"), + Expr(Symbol("scope-block"), + quote + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + begin + body + end + end))) + gsym1 = Top.iterate(ssa1, ssa3) + end, + :(Top.not_int(Core.:(===)(gsym1, $nothing))))) + end + end)) + ) end @testset "Functions" begin From 3c320c2a55d4f4eae6b5212a60c4b57d5bf1653f Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 4 Jun 2019 17:32:16 +1000 Subject: [PATCH 09/56] Test case: Function keyword args --- test/compiler/lowering.jl | 70 +++++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 67d30c0078136..ea0937231d87b 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -772,12 +772,70 @@ end end ) - # Keyword args - # TODO - #= @desugar(function f(x; k1=v1, k2=v2) =# - #= body =# - #= end, =# - #= ) =# + # Keyword arguments + @test_desugar(function f(x; k1=v1, k2=v2) + body + end, + begin + Core.ifelse(false, false, + begin + $(Expr(:method, :f)) + begin + $(Expr(:method, :gsym1)) + $(Expr(:method, :gsym1, + :(Core.svec(Core.svec(Core.typeof(gsym1), Core.Any, Core.Any, Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, Any[:gsym1, :k1, :k2, :_self_, :x], Any[], + Expr(Symbol("scope-block"), :body)))) + maybe_unused(gsym1) + end + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, Any[:_self_, :x], Any[], + Expr(Symbol("scope-block"), + :(return gsym1(v1, v2, _self_, x)))))) + maybe_unused(f) + end + begin + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.kwftype(Core.Typeof(f)), Core.Any, Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, Any[:gsym2, :gsym3, :_self_, :x], Any[], + Expr(Symbol("scope-block"), + Expr(Symbol("scope-block"), + quote + $(Expr(Symbol("local-def"), :k1)) + k1 = if Top.haskey(gsym3, :k1) + Top.getindex(gsym3, :k1) + else + v1 + end + $(Expr(Symbol("scope-block"), + quote + $(Expr(Symbol("local-def"), :k2)) + k2 = if Top.haskey(gsym3, :k2) + Top.getindex(gsym3, :k2) + else + v2 + end + begin + ssa1 = Top.pairs(Top.structdiff(gsym3, Core.apply_type(Core.NamedTuple, Core.tuple(:k1, :k2)))) + if Top.isempty(ssa1) + $nothing + else + Top.kwerr(gsym3, _self_, x) + end + return gsym1(k1, k2, _self_, x) + end + end)) + end))))) + maybe_unused(f) + end + f + end) + end + ) # Return type declaration @test_desugar(function f(x)::T From 714ed8702b0e22f40315443d41555d805d6149ef Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 4 Jun 2019 17:42:45 +1000 Subject: [PATCH 10/56] WIP: Tests for let blocks --- test/compiler/lowering.jl | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index ea0937231d87b..1ef6c32af7625 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -596,9 +596,49 @@ end end)) end ) + +end + +@testset "let blocks" begin + # flisp: (expand-let) + @test_desugar(let x,y + body + end, + begin + $(Expr(Symbol("scope-block"), + quote + local x + $(Expr(Symbol("scope-block"), + quote + local y + body + end)) + end)) + end + ) + # Let with assignment + @test_desugar(let x=a,y=b + body + end, + begin + $(Expr(Symbol("scope-block"), + quote + $(Expr(Symbol("local-def"), :x)) + x = a + $(Expr(Symbol("scope-block"), + quote + $(Expr(Symbol("local-def"), :y)) + y = b + body + end)) + end)) + end + ) + # TODO: More coverage. Internals look complex. end @testset "Loops" begin + # flisp: (expand-for) (lambda in expand-forms) @test_desugar(while cond body1 continue From 81df808945221834292e73f4de562ec67142d014 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 5 Jun 2019 19:45:37 +1000 Subject: [PATCH 11/56] `let` with empty bindings as syntax for scope-block --- test/compiler/lowering.jl | 230 +++++++++++++++++++------------------- 1 file changed, 115 insertions(+), 115 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 1ef6c32af7625..e177df0248b33 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -51,6 +51,8 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) # `unnecessary` marks expressions generated by lowering that # do not need to be evaluated if their value is unused. return Expr(:call, :maybe_unused, ex.args...) + elseif ex.head == Symbol("scope-block") + return Expr(:let, Expr(:block), ex.args[1]) end end elseif ex isa Vector # Occasional case of lambdas @@ -99,6 +101,8 @@ function lower_ref_expr!(ex) return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused return Expr(:unnecessary, ex.args[2:end]...) + elseif ex.head == :let && isempty(ex.args[1].args) + return Expr(Symbol("scope-block"), ex.args[2]) end end return ex @@ -588,12 +592,11 @@ end @test_desugar(A{T} = B{T}, begin $(Expr(Symbol("const-if-global"), :A)) - A = $(Expr(Symbol("scope-block"), - quote - $(Expr(Symbol("local-def"), :T)) - T = Core.TypeVar(:T) - Core.UnionAll(T, Core.apply_type(B, T)) - end)) + A = let + $(Expr(Symbol("local-def"), :T)) + T = Core.TypeVar(:T) + Core.UnionAll(T, Core.apply_type(B, T)) + end end ) @@ -605,15 +608,13 @@ end body end, begin - $(Expr(Symbol("scope-block"), - quote - local x - $(Expr(Symbol("scope-block"), - quote - local y - body - end)) - end)) + let + local x + let + local y + body + end + end end ) # Let with assignment @@ -621,17 +622,15 @@ end body end, begin - $(Expr(Symbol("scope-block"), - quote - $(Expr(Symbol("local-def"), :x)) - x = a - $(Expr(Symbol("scope-block"), - quote - $(Expr(Symbol("local-def"), :y)) - y = b - body - end)) - end)) + let + $(Expr(Symbol("local-def"), :x)) + x = a + let + $(Expr(Symbol("local-def"), :y)) + y = b + body + end + end end ) # TODO: More coverage. Internals look complex. @@ -649,18 +648,13 @@ end $(Expr(Symbol("break-block"), Symbol("loop-exit"), Expr(:_while, :cond, Expr(Symbol("break-block"), Symbol("loop-cont"), - Expr(Symbol("scope-block"), - quote - body1 - $(Expr(:break, Symbol("loop-cont"))) - body2 - $(Expr(:break, Symbol("loop-exit"))) - body3 - end - ) - ) - ) - )) + :(let + body1 + $(Expr(:break, Symbol("loop-cont"))) + body2 + $(Expr(:break, Symbol("loop-exit"))) + body3 + end))))) ) @test_desugar(for i = a @@ -677,22 +671,21 @@ end $(Expr(:_do_while, quote $(Expr(Symbol("break-block"), Symbol("loop-cont"), - Expr(Symbol("scope-block"), - quote - local i - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - begin - body1 - $(Expr(:break, Symbol("loop-cont"))) - body2 - $(Expr(:break, Symbol("loop-exit"))) - end - end))) + :(let + local i + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + begin + body1 + $(Expr(:break, Symbol("loop-cont"))) + body2 + $(Expr(:break, Symbol("loop-exit"))) + end + end))) gsym1 = Top.iterate(ssa1, ssa3) end, :(Top.not_int(Core.:(===)(gsym1, $nothing))))) @@ -713,18 +706,17 @@ end $(Expr(:_do_while, quote $(Expr(Symbol("break-block"), Symbol("loop-cont"), - Expr(Symbol("scope-block"), - quote - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - begin - body - end - end))) + :(let + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + begin + body + end + end))) gsym1 = Top.iterate(ssa1, ssa3) end, :(Top.not_int(Core.:(===)(gsym1, $nothing))))) @@ -741,8 +733,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, [:_self_, :x], [], - Expr(Symbol("scope-block"), - :(body(x)))))) + :(let + body(x) + end)))) maybe_unused(f) end ) @@ -756,8 +749,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec())), Expr(:lambda, [:_self_, :x, :y], [], - Expr(Symbol("scope-block"), - :(body(x)))))) + :(let + body(x) + end)))) maybe_unused(f) end ) @@ -772,8 +766,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f)), Core.svec())), Expr(:lambda, [:_self_], [], - Expr(Symbol("scope-block"), - :(_self_(a, b)))))) + :(let + _self_(a, b) + end)))) maybe_unused(f) end begin @@ -781,8 +776,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, [:_self_, :x], [], - Expr(Symbol("scope-block"), - :(_self_(x, b)))))) + :(let + _self_(x, b) + end)))) maybe_unused(f) end begin @@ -790,8 +786,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec())), Expr(:lambda, [:_self_, :x, :y], [], - Expr(Symbol("scope-block"), - :(body(x,y)))))) + :(let + body(x,y) + end)))) maybe_unused(f) end end @@ -806,8 +803,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.apply_type(Vararg, Core.Any)), Core.svec())), Expr(:lambda, [:_self_, :x, :args], [], - Expr(Symbol("scope-block"), - :(body(x, args)))))) + :(let + body(x, args) + end)))) maybe_unused(f) end ) @@ -825,7 +823,9 @@ end $(Expr(:method, :gsym1, :(Core.svec(Core.svec(Core.typeof(gsym1), Core.Any, Core.Any, Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, Any[:gsym1, :k1, :k2, :_self_, :x], Any[], - Expr(Symbol("scope-block"), :body)))) + :(let + body + end)))) maybe_unused(gsym1) end begin @@ -833,8 +833,9 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, Any[:_self_, :x], Any[], - Expr(Symbol("scope-block"), - :(return gsym1(v1, v2, _self_, x)))))) + :(let + return gsym1(v1, v2, _self_, x) + end)))) maybe_unused(f) end begin @@ -842,34 +843,33 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.kwftype(Core.Typeof(f)), Core.Any, Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, Any[:gsym2, :gsym3, :_self_, :x], Any[], - Expr(Symbol("scope-block"), - Expr(Symbol("scope-block"), - quote - $(Expr(Symbol("local-def"), :k1)) - k1 = if Top.haskey(gsym3, :k1) - Top.getindex(gsym3, :k1) + :(let + let + $(Expr(Symbol("local-def"), :k1)) + k1 = if Top.haskey(gsym3, :k1) + Top.getindex(gsym3, :k1) + else + v1 + end + let + $(Expr(Symbol("local-def"), :k2)) + k2 = if Top.haskey(gsym3, :k2) + Top.getindex(gsym3, :k2) else - v1 + v2 + end + begin + ssa1 = Top.pairs(Top.structdiff(gsym3, Core.apply_type(Core.NamedTuple, Core.tuple(:k1, :k2)))) + if Top.isempty(ssa1) + $nothing + else + Top.kwerr(gsym3, _self_, x) + end + return gsym1(k1, k2, _self_, x) end - $(Expr(Symbol("scope-block"), - quote - $(Expr(Symbol("local-def"), :k2)) - k2 = if Top.haskey(gsym3, :k2) - Top.getindex(gsym3, :k2) - else - v2 - end - begin - ssa1 = Top.pairs(Top.structdiff(gsym3, Core.apply_type(Core.NamedTuple, Core.tuple(:k1, :k2)))) - if Top.isempty(ssa1) - $nothing - else - Top.kwerr(gsym3, _self_, x) - end - return gsym1(k1, k2, _self_, x) - end - end)) - end))))) + end + end + end)))) maybe_unused(f) end f @@ -886,12 +886,11 @@ end $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), Expr(:lambda, [:_self_, :x], [], - Expr(Symbol("scope-block"), - quote - ssa1 = T - $(Expr(:meta, Symbol("ret-type"), :ssa1)) - body(x) - end)))) + :(let + ssa1 = T + $(Expr(:meta, Symbol("ret-type"), :ssa1)) + body(x) + end)))) maybe_unused(f) end ) @@ -905,8 +904,9 @@ end $(Expr(:method, :gsym1, :(Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec())), Expr(:lambda, [:_self_, :x, :y], [], - Expr(Symbol("scope-block"), - :(body(x, y)))))) + :(let + body(x, y) + end)))) maybe_unused(gsym1) end end From 7745341123e106c6f83e19b7e565ca4532c68465 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 5 Jun 2019 19:46:22 +1000 Subject: [PATCH 12/56] S-Expression parser. Very WIP. --- test/compiler/sexpressions.jl | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 test/compiler/sexpressions.jl diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl new file mode 100644 index 0000000000000..971058893b5b2 --- /dev/null +++ b/test/compiler/sexpressions.jl @@ -0,0 +1,84 @@ +module SExprs + +struct Unquote + name::Symbol +end + +function skipws(io::IO) + while !eof(io) + c = read(io, Char) + if !isspace(c) + return c + end + end + error("S-Expression terminated early by EOF") +end + +function _parse(io::IO, c::Char) + if c == '(' + lst = Any[] # ok ok, this is not a real man's list. + while true + c = skipws(io) + c == ')' && return lst + push!(lst, _parse(io, c)) + end + elseif c == ',' + c = skipws(io) + c != ')' || error("No symbol found for unquote ','. Got unexpected ')' instead") + x = _parse(io, c) + x isa Symbol || error("Unquote only supports symbols as arguments. Got $x") + return Unquote(x) + else + # Anything else is a word... + buf = IOBuffer(sizehint=10) + write(buf, c) + while !eof(io) + c = read(io, Char) + isspace(c) && break + if c == ')' || c == ',' + skip(io, -1) # Assumes seekable :-/ + break + end + write(buf, c) + end + return Symbol(take!(buf)) + end +end + +function parse(io::IO) + c = skipws(io) + c != ')' || error("Closing ')' found before '('") + _parse(io, c) +end + +parse(content::String) = parse(IOBuffer(content)) + +macro sexpr_str(str) + parse(IOBuffer(str)) +end + +end + + +using Test + +@testset "S-Expressions" begin + + parse = SExprs.parse + Unquote = SExprs.Unquote + + @test parse("aa") == :aa + @test parse(",aa") == Unquote(:aa) + @test parse("(aa bb cc)") == [:aa, :bb, :cc] + @test parse("(+ b (* c d))") == [:+, :b, [:*, :c, :d]] + # Whitespace + @test parse(" ( a\nb\n\r(c d)) ") == [:a, :b, [:c, :d]] + + # Errors + @test_throws ErrorException parse(")") + @test_throws ErrorException parse("(") + @test_throws ErrorException parse("(,)") + @test_throws ErrorException parse(",") + @test_throws ErrorException parse(",(") + @test_throws ErrorException parse(",(a)") +end From dfe454734d6e358a8a0ced7ff796a12c4ef65820 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 5 Jun 2019 23:21:35 +1000 Subject: [PATCH 13/56] Test optionally generated functions --- test/compiler/lowering.jl | 61 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index e177df0248b33..43a16daa0aa8f 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -919,6 +919,67 @@ end @test_desugar_error false(x)=body "invalid function name \"false\"" end +@testset "Generated functions" begin + ln = LineNumberNode(@__LINE__()+2, Symbol(@__FILE__)) + @test_desugar(function f(x) + body1(x) + if $(Expr(:generated)) + gen_body1(x) + else + normal_body1(x) + end + body2 + if $(Expr(:generated)) + gen_body2 + else + normal_body2 + end + end, + begin + begin + global gsym1 + begin + $(Expr(:method, :gsym1)) + $(Expr(:method, :gsym1, + :(Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec())), + Expr(:lambda, [:_self_, :gsym2, :x], [], + :(let + $(Expr(:meta, :nospecialize, :gsym2, :x)) + Core._expr(:block, + $(QuoteNode(LineNumberNode(ln.line, ln.file))), + $(Expr(:copyast, QuoteNode(:(body1(x))))), + # FIXME: These line numbers seem buggy? + $(QuoteNode(LineNumberNode(ln.line+1, ln.file))), + gen_body1(x), + $(QuoteNode(LineNumberNode(ln.line+6, ln.file))), + :body2, + $(QuoteNode(LineNumberNode(ln.line+7, ln.file))), + gen_body2 + ) + end)))) + maybe_unused(gsym1) + end + end + $(Expr(:method, :f)) + $(Expr(:method, :f, + :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), + Expr(:lambda, [:_self_, :x], [], + :(let + $(Expr(:meta, :generated, + Expr(:new, :(Core.GeneratedFunctionStub), + :gsym1, [:_self_, :x], + :nothing, ln.line, + QuoteNode(ln.file), false))) + body1(x) + normal_body1(x) + body2 + normal_body2 + end)))) + maybe_unused(f) + end + ) +end + @testset "Forms without desugaring" begin # (expand-forms) # The following Expr heads are currently not touched by desugaring From ac777f51401567c5bc64259595bb9f9051f0a349 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 5 Jun 2019 23:22:05 +1000 Subject: [PATCH 14/56] Minor cleanup --- test/compiler/lowering.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 43a16daa0aa8f..d9592bd2c6d0d 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -822,7 +822,7 @@ end $(Expr(:method, :gsym1)) $(Expr(:method, :gsym1, :(Core.svec(Core.svec(Core.typeof(gsym1), Core.Any, Core.Any, Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, Any[:gsym1, :k1, :k2, :_self_, :x], Any[], + Expr(:lambda, [:gsym1, :k1, :k2, :_self_, :x], [], :(let body end)))) @@ -832,7 +832,7 @@ end $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, Any[:_self_, :x], Any[], + Expr(:lambda, [:_self_, :x], [], :(let return gsym1(v1, v2, _self_, x) end)))) @@ -842,7 +842,7 @@ end $(Expr(:method, :f)) $(Expr(:method, :f, :(Core.svec(Core.svec(Core.kwftype(Core.Typeof(f)), Core.Any, Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, Any[:gsym2, :gsym3, :_self_, :x], Any[], + Expr(:lambda, [:gsym2, :gsym3, :_self_, :x], [], :(let let $(Expr(Symbol("local-def"), :k1)) From 6e61c1027266c2db12619f5654e91463d8e66c0a Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 6 Jun 2019 00:30:45 +1000 Subject: [PATCH 15/56] SExpressions: Parse various atoms, string macro unquote interpolation --- test/compiler/sexpressions.jl | 58 ++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl index 971058893b5b2..90c3755aaffd0 100644 --- a/test/compiler/sexpressions.jl +++ b/test/compiler/sexpressions.jl @@ -1,5 +1,7 @@ module SExprs +export Unquote, @sexpr_str + struct Unquote name::Symbol end @@ -16,7 +18,7 @@ end function _parse(io::IO, c::Char) if c == '(' - lst = Any[] # ok ok, this is not a real man's list. + lst = Any[] while true c = skipws(io) c == ')' && return lst @@ -28,6 +30,32 @@ function _parse(io::IO, c::Char) x = _parse(io, c) x isa Symbol || error("Unquote only supports symbols as arguments. Got $x") return Unquote(x) + elseif isdigit(c) + n = c-'0' + while !eof(io) + c = read(io, Char) + if !isdigit(c) + skip(io, -1) + break + end + n = 10*n + (c-'0') + end + return n + elseif c == '"' + buf = IOBuffer(sizehint=10) + while true + !eof(io) || error("Unterminated string") + c = read(io, Char) + c == '"' && break + write(buf, c) + end + return String(take!(buf)) + elseif c == '#' + !eof(io) || error("Unterminated #") + c = read(io, Char) + return c == 't' ? true : + c == 'f' ? false : + error("Invalid boolean") else # Anything else is a word... buf = IOBuffer(sizehint=10) @@ -53,8 +81,13 @@ end parse(content::String) = parse(IOBuffer(content)) +map_unqoutes(sx) = sx +map_unqoutes(sx::Unquote) = esc(sx.name) +map_unqoutes(sx::Vector) = Expr(:vect, map(map_unqoutes, sx)...) + macro sexpr_str(str) - parse(IOBuffer(str)) + sx = parse(IOBuffer(str)) + map_unqoutes(sx) end end @@ -67,18 +100,33 @@ using Test parse = SExprs.parse Unquote = SExprs.Unquote + # atoms + @test parse("#t") == true + @test parse("#f") == false @test parse("aa") == :aa - @test parse(",aa") == Unquote(:aa) + @test parse("+-*") == Symbol("+-*") + @test parse("12") == 12 + @test parse("\"some str 123\"") == "some str 123" + # lists @test parse("(aa bb cc)") == [:aa, :bb, :cc] @test parse("(+ b (* c d))") == [:+, :b, [:*, :c, :d]] - # Whitespace + # unquote + @test parse(",aa") == Unquote(:aa) + # whitespace @test parse(" ( a\nb\n\r(c d)) ") == [:a, :b, [:c, :d]] - # Errors + # interpolation + x = 10 + y = "str" + @test SExprs.sexpr"(,x ,y ,Int)" == [10, "str", Int] + + # errors @test_throws ErrorException parse(")") @test_throws ErrorException parse("(") @test_throws ErrorException parse("(,)") @test_throws ErrorException parse(",") @test_throws ErrorException parse(",(") @test_throws ErrorException parse(",(a)") + @test_throws ErrorException parse("\"asdf") + @test_throws ErrorException parse("#") end From 21393943759fadc4a19daf6ac9584910a3eed3b5 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 6 Jun 2019 01:25:39 +1000 Subject: [PATCH 16/56] Very basic S-Expression deparse + some fixes --- test/compiler/sexpressions.jl | 49 +++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl index 90c3755aaffd0..f086102bfd58c 100644 --- a/test/compiler/sexpressions.jl +++ b/test/compiler/sexpressions.jl @@ -9,13 +9,13 @@ end function skipws(io::IO) while !eof(io) c = read(io, Char) - if !isspace(c) - return c - end + isspace(c) || return c end error("S-Expression terminated early by EOF") end +putback(io::IO) = skip(io, -1) # assumes seekable :-/ + function _parse(io::IO, c::Char) if c == '(' lst = Any[] @@ -35,7 +35,7 @@ function _parse(io::IO, c::Char) while !eof(io) c = read(io, Char) if !isdigit(c) - skip(io, -1) + putback(io) break end n = 10*n + (c-'0') @@ -64,7 +64,7 @@ function _parse(io::IO, c::Char) c = read(io, Char) isspace(c) && break if c == ')' || c == ',' - skip(io, -1) # Assumes seekable :-/ + putback(io) break end write(buf, c) @@ -81,13 +81,33 @@ end parse(content::String) = parse(IOBuffer(content)) -map_unqoutes(sx) = sx -map_unqoutes(sx::Unquote) = esc(sx.name) -map_unqoutes(sx::Vector) = Expr(:vect, map(map_unqoutes, sx)...) +deparse(io::IO, sx::String) = print(io, '"', sx, '"') +deparse(io::IO, sx::Bool) = print(io, sx ? "#t" : "#f") +deparse(io::IO, sx) = print(io, sx) +deparse(io::IO, sx::Unquote) = print(io, ',', string(sx.name)) +function deparse(io::IO, sx::Vector) + write(io, '(') + for (i,e) in enumerate(sx) + deparse(io, e) + i != length(sx) && write(io, ' ') + end + write(io, ')') +end + +function deparse(sx) + buf = IOBuffer() + deparse(buf, sx) + String(take!(buf)) +end + +fill_unquotes_expr(sx) = sx +fill_unquotes_expr(sx::Symbol) = QuoteNode(sx) +fill_unquotes_expr(sx::Unquote) = esc(sx.name) +fill_unquotes_expr(sx::Vector) = Expr(:vect, map(fill_unquotes_expr, sx)...) macro sexpr_str(str) sx = parse(IOBuffer(str)) - map_unqoutes(sx) + fill_unquotes_expr(sx) end end @@ -96,9 +116,7 @@ end using Test @testset "S-Expressions" begin - parse = SExprs.parse - Unquote = SExprs.Unquote # atoms @test parse("#t") == true @@ -111,16 +129,16 @@ using Test @test parse("(aa bb cc)") == [:aa, :bb, :cc] @test parse("(+ b (* c d))") == [:+, :b, [:*, :c, :d]] # unquote - @test parse(",aa") == Unquote(:aa) + @test parse(",aa") == SExprs.Unquote(:aa) # whitespace @test parse(" ( a\nb\n\r(c d)) ") == [:a, :b, [:c, :d]] # interpolation x = 10 y = "str" - @test SExprs.sexpr"(,x ,y ,Int)" == [10, "str", Int] + @test SExprs.sexpr"(x ,y ,Int)" == [:x, "str", Int] - # errors + # parse errors @test_throws ErrorException parse(")") @test_throws ErrorException parse("(") @test_throws ErrorException parse("(,)") @@ -129,4 +147,7 @@ using Test @test_throws ErrorException parse(",(a)") @test_throws ErrorException parse("\"asdf") @test_throws ErrorException parse("#") + + # printing + @test SExprs.deparse([1,2, [:aa, :bb], SExprs.Unquote(:cc)]) == "(1 2 (aa bb) ,cc)" end From 098fecc1d9afc059520b9ef3351ab4a83f34d009 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 6 Jun 2019 01:26:22 +1000 Subject: [PATCH 17/56] Try out S-Expressions as desugaring test reference --- test/compiler/lowering.jl | 115 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index d9592bd2c6d0d..fef09f4192cfc 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1,3 +1,5 @@ +include("sexpressions.jl") + using Core: SSAValue # Call into lowering stage 1; syntax desugaring @@ -75,6 +77,52 @@ function lift_lowered_expr(ex; lift_full=false) lift_lowered_expr!(deepcopy(ex), ones(Int,2), valmap, lift_full) end +function to_sexpr!(ex, nextids, valmap) + if ex isa SSAValue + # Rename SSAValues into renumbered symbols + return get!(valmap, ex) do + newid = nextids[1] + nextids[1] = newid+1 + Symbol("ssa$newid") + end + elseif ex isa Symbol + if ex == Symbol("#self#") + return :_self_ + end + # Rename gensyms + name = string(ex) + if startswith(name, "#") + return get!(valmap, ex) do + newid = nextids[2] + nextids[2] = newid+1 + Symbol("gsym$newid") + end + end + elseif ex isa Expr + filter!(e->!(e isa LineNumberNode), ex.args) + if ex.head == :block && length(ex.args) == 1 + # Remove trivial blocks + return to_sexpr!(ex.args[1], nextids, valmap) + end + map!(ex.args, ex.args) do e + to_sexpr!(e, nextids, valmap) + end + return [ex.head; ex.args] + elseif ex isa QuoteNode + return [:quote, ex.value] + elseif ex isa Vector # Occasional case of lambdas + map!(ex, ex) do e + to_sexpr!(e, nextids, valmap) + end + end + return ex +end + +function to_sexpr(ex) + valmap = Dict{Union{Symbol,SSAValue},Symbol}() + to_sexpr!(deepcopy(ex), ones(Int,2), valmap) +end + """ Very slight lowering of reference expressions to allow comparison with desugared forms. @@ -157,6 +205,12 @@ macro desugar(ex, kws...) end end +macro desugar_sx(ex) + quote + SExprs.deparse(to_sexpr($(Expr(:quote, ex)))) + end +end + """ Test that syntax desugaring of `input` produces an expression equivalent to the reference expression `ref`. @@ -179,6 +233,25 @@ macro test_desugar(input, ref) ex end +macro test_desugar_sexpr(input, ref) + ex = quote + input = to_sexpr(expand_forms($(Expr(:quote, input)))) + ref = SExprs.parse($(esc(ref))) + @test input == ref + if input != ref + # Kinda crude. Would be much neater if Test supported custom/more + # capable diffing for failed tests. + println("Diff dump:") + println(SExprs.deparse(input)) + println(SExprs.deparse(ref)) + end + end + # Attribute the test to the correct line number + @assert ex.args[6].args[1] == Symbol("@test") + ex.args[6].args[2] = __source__ + ex +end + macro test_desugar_error(input, msg) ex = quote input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) @@ -213,6 +286,28 @@ end ) end +# Example S-Expression version of the above. +@testset "Property notation" begin + # flisp: (expand-fuse-broadcast) + @test_desugar_sexpr a.b "(call (top getproperty) a (quote b))" + @test_desugar_sexpr a.b.c "(call (top getproperty) + (call (top getproperty) a (quote b)) + (quote c))" + + @test_desugar_sexpr(a.b = c, + "(block + (call (top setproperty!) a (quote b) c) + (unnecessary c))" + ) + @test_desugar_sexpr(a.b.c = d, + "(block + (= ssa1 (call (top getproperty) a (quote b))) + (call (top setproperty!) ssa1 (quote c) d) + (unnecessary d))" + ) +end + + @testset "Index notation" begin # flisp: (process-indices) (partially-expand-ref) @testset "getindex" begin @@ -657,6 +752,26 @@ end end))))) ) + # Alternative with S-Expressions + @test_desugar_sexpr(while cond + body1 + continue + body2 + break + body3 + end, + "(break-block loop-exit + (_while cond + (break-block loop-cont + (scope-block + (block + body1 + (break loop-cont) + body2 + (break loop-exit) + body3)))))" + ) + @test_desugar(for i = a body1 continue From c8cd625c34f812cd86a6ee279769fe977ae82c86 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Fri, 7 Jun 2019 16:17:09 +1000 Subject: [PATCH 18/56] Tweak S-Expression test infrastructure --- test/compiler/lowering.jl | 101 +++++++++++++++++----------------- test/compiler/sexpressions.jl | 6 +- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index fef09f4192cfc..b8047c529b869 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1,5 +1,7 @@ include("sexpressions.jl") +using .SExprs + using Core: SSAValue # Call into lowering stage 1; syntax desugaring @@ -216,34 +218,29 @@ Test that syntax desugaring of `input` produces an expression equivalent to the reference expression `ref`. """ macro test_desugar(input, ref) - ex = quote - input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) - ref = lower_ref_expr($(Expr(:quote, ref))) - @test input == ref - if input != ref - # Kinda crude. Would be much neater if Test supported custom/more - # capable diffing for failed tests. - println("Diff dump:") - diffdump(input, ref) + ex = if ref.head == :macrocall && ref.args[1] == Symbol("@sx_str") + # Reference is an S-Expression - test against that directly. + quote + input = to_sexpr(expand_forms($(Expr(:quote, input)))) + ref = $(esc(ref)) + @test input == ref + if input != ref + println("Diff dump:") + println(SExprs.deparse(input)) + println(SExprs.deparse(ref)) + end end - end - # Attribute the test to the correct line number - @assert ex.args[6].args[1] == Symbol("@test") - ex.args[6].args[2] = __source__ - ex -end - -macro test_desugar_sexpr(input, ref) - ex = quote - input = to_sexpr(expand_forms($(Expr(:quote, input)))) - ref = SExprs.parse($(esc(ref))) - @test input == ref - if input != ref - # Kinda crude. Would be much neater if Test supported custom/more - # capable diffing for failed tests. - println("Diff dump:") - println(SExprs.deparse(input)) - println(SExprs.deparse(ref)) + else + quote + input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) + ref = lower_ref_expr($(Expr(:quote, ref))) + @test input == ref + if input != ref + # Kinda crude. Would be much neater if Test supported custom/more + # capable diffing for failed tests. + println("Diff dump:") + diffdump(input, ref) + end end end # Attribute the test to the correct line number @@ -289,21 +286,21 @@ end # Example S-Expression version of the above. @testset "Property notation" begin # flisp: (expand-fuse-broadcast) - @test_desugar_sexpr a.b "(call (top getproperty) a (quote b))" - @test_desugar_sexpr a.b.c "(call (top getproperty) - (call (top getproperty) a (quote b)) - (quote c))" - - @test_desugar_sexpr(a.b = c, - "(block - (call (top setproperty!) a (quote b) c) - (unnecessary c))" + @test_desugar a.b sx"(call (top getproperty) a (quote b))" + @test_desugar a.b.c sx"(call (top getproperty) + (call (top getproperty) a (quote b)) + (quote c))" + + @test_desugar(a.b = c, + sx"(block + (call (top setproperty!) a (quote b) c) + (unnecessary c))" ) - @test_desugar_sexpr(a.b.c = d, - "(block - (= ssa1 (call (top getproperty) a (quote b))) - (call (top setproperty!) ssa1 (quote c) d) - (unnecessary d))" + @test_desugar(a.b.c = d, + sx"(block + (= ssa1 (call (top getproperty) a (quote b))) + (call (top setproperty!) ssa1 (quote c) d) + (unnecessary d))" ) end @@ -753,23 +750,23 @@ end ) # Alternative with S-Expressions - @test_desugar_sexpr(while cond + @test_desugar(while cond body1 continue body2 break body3 end, - "(break-block loop-exit - (_while cond - (break-block loop-cont - (scope-block - (block - body1 - (break loop-cont) - body2 - (break loop-exit) - body3)))))" + sx"(break-block loop-exit + (_while cond + (break-block loop-cont + (scope-block + (block + body1 + (break loop-cont) + body2 + (break loop-exit) + body3)))))" ) @test_desugar(for i = a diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl index f086102bfd58c..ca38f0ba7347b 100644 --- a/test/compiler/sexpressions.jl +++ b/test/compiler/sexpressions.jl @@ -1,6 +1,6 @@ module SExprs -export Unquote, @sexpr_str +export @sx_str struct Unquote name::Symbol @@ -105,7 +105,7 @@ fill_unquotes_expr(sx::Symbol) = QuoteNode(sx) fill_unquotes_expr(sx::Unquote) = esc(sx.name) fill_unquotes_expr(sx::Vector) = Expr(:vect, map(fill_unquotes_expr, sx)...) -macro sexpr_str(str) +macro sx_str(str) sx = parse(IOBuffer(str)) fill_unquotes_expr(sx) end @@ -136,7 +136,7 @@ using Test # interpolation x = 10 y = "str" - @test SExprs.sexpr"(x ,y ,Int)" == [:x, "str", Int] + @test SExprs.sx"(x ,y ,Int)" == [:x, "str", Int] # parse errors @test_throws ErrorException parse(")") From 295a09a57114fa29f20b6dbbf81e4dac66af7fea Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 13 Jun 2019 07:26:54 +1000 Subject: [PATCH 19/56] A few small fixes --- test/compiler/lowering.jl | 6 ++++-- test/compiler/sexpressions.jl | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index b8047c529b869..0313f811f2660 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -111,7 +111,9 @@ function to_sexpr!(ex, nextids, valmap) end return [ex.head; ex.args] elseif ex isa QuoteNode - return [:quote, ex.value] + return [:quote, to_sexpr!(ex.value, nextids, valmap)] + elseif ex isa LineNumberNode + return [:line, ex.line, ex.file] elseif ex isa Vector # Occasional case of lambdas map!(ex, ex) do e to_sexpr!(e, nextids, valmap) @@ -209,7 +211,7 @@ end macro desugar_sx(ex) quote - SExprs.deparse(to_sexpr($(Expr(:quote, ex)))) + SExprs.deparse(to_sexpr(expand_forms($(Expr(:quote, ex))))) end end diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl index ca38f0ba7347b..44310eed08464 100644 --- a/test/compiler/sexpressions.jl +++ b/test/compiler/sexpressions.jl @@ -84,6 +84,7 @@ parse(content::String) = parse(IOBuffer(content)) deparse(io::IO, sx::String) = print(io, '"', sx, '"') deparse(io::IO, sx::Bool) = print(io, sx ? "#t" : "#f") deparse(io::IO, sx) = print(io, sx) +deparse(io::IO, sx::Nothing) = "nothing" deparse(io::IO, sx::Unquote) = print(io, ',', string(sx.name)) function deparse(io::IO, sx::Vector) write(io, '(') From 62602dda2fe196355f9ae9b7a89b83d0adcfceec Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 16 Jun 2019 12:31:26 +1000 Subject: [PATCH 20/56] S-Expression util tweaks --- test/compiler/lowering.jl | 7 ++++++- test/compiler/sexpressions.jl | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 0313f811f2660..c25491bf86b25 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -209,9 +209,14 @@ macro desugar(ex, kws...) end end + +function desugar_sx(ex) + SExprs.prettyprint(to_sexpr(expand_forms(ex))) +end + macro desugar_sx(ex) quote - SExprs.deparse(to_sexpr(expand_forms($(Expr(:quote, ex))))) + println(desugar_sx($(Expr(:quote, ex)))) end end diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl index 44310eed08464..01ec7e275b641 100644 --- a/test/compiler/sexpressions.jl +++ b/test/compiler/sexpressions.jl @@ -101,6 +101,15 @@ function deparse(sx) String(take!(buf)) end +function prettyprint(sx) + buf = IOBuffer() + # Cheat terribly using racket's pretty printer + run(pipeline(`racket -e "(require racket/pretty) (pretty-write (read (current-input-port)))"`, + stdin=IOBuffer(deparse(sx)), + stdout=buf)) + String(take!(buf)) +end + fill_unquotes_expr(sx) = sx fill_unquotes_expr(sx::Symbol) = QuoteNode(sx) fill_unquotes_expr(sx::Unquote) = esc(sx.name) From 57afea5e66bb1c9d9243de9f7343959d752b6cd6 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 16 Jun 2019 12:34:02 +1000 Subject: [PATCH 21/56] Remove S-Expression alternative for now. Remove the S-Expressions parser in the interests of keeping focussed enough to finish testing the desugaring tests. --- test/compiler/lowering.jl | 137 ++-------------------------- test/compiler/sexpressions.jl | 163 ---------------------------------- 2 files changed, 9 insertions(+), 291 deletions(-) delete mode 100644 test/compiler/sexpressions.jl diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index c25491bf86b25..d9592bd2c6d0d 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1,7 +1,3 @@ -include("sexpressions.jl") - -using .SExprs - using Core: SSAValue # Call into lowering stage 1; syntax desugaring @@ -79,54 +75,6 @@ function lift_lowered_expr(ex; lift_full=false) lift_lowered_expr!(deepcopy(ex), ones(Int,2), valmap, lift_full) end -function to_sexpr!(ex, nextids, valmap) - if ex isa SSAValue - # Rename SSAValues into renumbered symbols - return get!(valmap, ex) do - newid = nextids[1] - nextids[1] = newid+1 - Symbol("ssa$newid") - end - elseif ex isa Symbol - if ex == Symbol("#self#") - return :_self_ - end - # Rename gensyms - name = string(ex) - if startswith(name, "#") - return get!(valmap, ex) do - newid = nextids[2] - nextids[2] = newid+1 - Symbol("gsym$newid") - end - end - elseif ex isa Expr - filter!(e->!(e isa LineNumberNode), ex.args) - if ex.head == :block && length(ex.args) == 1 - # Remove trivial blocks - return to_sexpr!(ex.args[1], nextids, valmap) - end - map!(ex.args, ex.args) do e - to_sexpr!(e, nextids, valmap) - end - return [ex.head; ex.args] - elseif ex isa QuoteNode - return [:quote, to_sexpr!(ex.value, nextids, valmap)] - elseif ex isa LineNumberNode - return [:line, ex.line, ex.file] - elseif ex isa Vector # Occasional case of lambdas - map!(ex, ex) do e - to_sexpr!(e, nextids, valmap) - end - end - return ex -end - -function to_sexpr(ex) - valmap = Dict{Union{Symbol,SSAValue},Symbol}() - to_sexpr!(deepcopy(ex), ones(Int,2), valmap) -end - """ Very slight lowering of reference expressions to allow comparison with desugared forms. @@ -209,45 +157,20 @@ macro desugar(ex, kws...) end end - -function desugar_sx(ex) - SExprs.prettyprint(to_sexpr(expand_forms(ex))) -end - -macro desugar_sx(ex) - quote - println(desugar_sx($(Expr(:quote, ex)))) - end -end - """ Test that syntax desugaring of `input` produces an expression equivalent to the reference expression `ref`. """ macro test_desugar(input, ref) - ex = if ref.head == :macrocall && ref.args[1] == Symbol("@sx_str") - # Reference is an S-Expression - test against that directly. - quote - input = to_sexpr(expand_forms($(Expr(:quote, input)))) - ref = $(esc(ref)) - @test input == ref - if input != ref - println("Diff dump:") - println(SExprs.deparse(input)) - println(SExprs.deparse(ref)) - end - end - else - quote - input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) - ref = lower_ref_expr($(Expr(:quote, ref))) - @test input == ref - if input != ref - # Kinda crude. Would be much neater if Test supported custom/more - # capable diffing for failed tests. - println("Diff dump:") - diffdump(input, ref) - end + ex = quote + input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) + ref = lower_ref_expr($(Expr(:quote, ref))) + @test input == ref + if input != ref + # Kinda crude. Would be much neater if Test supported custom/more + # capable diffing for failed tests. + println("Diff dump:") + diffdump(input, ref) end end # Attribute the test to the correct line number @@ -290,28 +213,6 @@ end ) end -# Example S-Expression version of the above. -@testset "Property notation" begin - # flisp: (expand-fuse-broadcast) - @test_desugar a.b sx"(call (top getproperty) a (quote b))" - @test_desugar a.b.c sx"(call (top getproperty) - (call (top getproperty) a (quote b)) - (quote c))" - - @test_desugar(a.b = c, - sx"(block - (call (top setproperty!) a (quote b) c) - (unnecessary c))" - ) - @test_desugar(a.b.c = d, - sx"(block - (= ssa1 (call (top getproperty) a (quote b))) - (call (top setproperty!) ssa1 (quote c) d) - (unnecessary d))" - ) -end - - @testset "Index notation" begin # flisp: (process-indices) (partially-expand-ref) @testset "getindex" begin @@ -756,26 +657,6 @@ end end))))) ) - # Alternative with S-Expressions - @test_desugar(while cond - body1 - continue - body2 - break - body3 - end, - sx"(break-block loop-exit - (_while cond - (break-block loop-cont - (scope-block - (block - body1 - (break loop-cont) - body2 - (break loop-exit) - body3)))))" - ) - @test_desugar(for i = a body1 continue diff --git a/test/compiler/sexpressions.jl b/test/compiler/sexpressions.jl deleted file mode 100644 index 01ec7e275b641..0000000000000 --- a/test/compiler/sexpressions.jl +++ /dev/null @@ -1,163 +0,0 @@ -module SExprs - -export @sx_str - -struct Unquote - name::Symbol -end - -function skipws(io::IO) - while !eof(io) - c = read(io, Char) - isspace(c) || return c - end - error("S-Expression terminated early by EOF") -end - -putback(io::IO) = skip(io, -1) # assumes seekable :-/ - -function _parse(io::IO, c::Char) - if c == '(' - lst = Any[] - while true - c = skipws(io) - c == ')' && return lst - push!(lst, _parse(io, c)) - end - elseif c == ',' - c = skipws(io) - c != ')' || error("No symbol found for unquote ','. Got unexpected ')' instead") - x = _parse(io, c) - x isa Symbol || error("Unquote only supports symbols as arguments. Got $x") - return Unquote(x) - elseif isdigit(c) - n = c-'0' - while !eof(io) - c = read(io, Char) - if !isdigit(c) - putback(io) - break - end - n = 10*n + (c-'0') - end - return n - elseif c == '"' - buf = IOBuffer(sizehint=10) - while true - !eof(io) || error("Unterminated string") - c = read(io, Char) - c == '"' && break - write(buf, c) - end - return String(take!(buf)) - elseif c == '#' - !eof(io) || error("Unterminated #") - c = read(io, Char) - return c == 't' ? true : - c == 'f' ? false : - error("Invalid boolean") - else - # Anything else is a word... - buf = IOBuffer(sizehint=10) - write(buf, c) - while !eof(io) - c = read(io, Char) - isspace(c) && break - if c == ')' || c == ',' - putback(io) - break - end - write(buf, c) - end - return Symbol(take!(buf)) - end -end - -function parse(io::IO) - c = skipws(io) - c != ')' || error("Closing ')' found before '('") - _parse(io, c) -end - -parse(content::String) = parse(IOBuffer(content)) - -deparse(io::IO, sx::String) = print(io, '"', sx, '"') -deparse(io::IO, sx::Bool) = print(io, sx ? "#t" : "#f") -deparse(io::IO, sx) = print(io, sx) -deparse(io::IO, sx::Nothing) = "nothing" -deparse(io::IO, sx::Unquote) = print(io, ',', string(sx.name)) -function deparse(io::IO, sx::Vector) - write(io, '(') - for (i,e) in enumerate(sx) - deparse(io, e) - i != length(sx) && write(io, ' ') - end - write(io, ')') -end - -function deparse(sx) - buf = IOBuffer() - deparse(buf, sx) - String(take!(buf)) -end - -function prettyprint(sx) - buf = IOBuffer() - # Cheat terribly using racket's pretty printer - run(pipeline(`racket -e "(require racket/pretty) (pretty-write (read (current-input-port)))"`, - stdin=IOBuffer(deparse(sx)), - stdout=buf)) - String(take!(buf)) -end - -fill_unquotes_expr(sx) = sx -fill_unquotes_expr(sx::Symbol) = QuoteNode(sx) -fill_unquotes_expr(sx::Unquote) = esc(sx.name) -fill_unquotes_expr(sx::Vector) = Expr(:vect, map(fill_unquotes_expr, sx)...) - -macro sx_str(str) - sx = parse(IOBuffer(str)) - fill_unquotes_expr(sx) -end - -end - - -using Test - -@testset "S-Expressions" begin - parse = SExprs.parse - - # atoms - @test parse("#t") == true - @test parse("#f") == false - @test parse("aa") == :aa - @test parse("+-*") == Symbol("+-*") - @test parse("12") == 12 - @test parse("\"some str 123\"") == "some str 123" - # lists - @test parse("(aa bb cc)") == [:aa, :bb, :cc] - @test parse("(+ b (* c d))") == [:+, :b, [:*, :c, :d]] - # unquote - @test parse(",aa") == SExprs.Unquote(:aa) - # whitespace - @test parse(" ( a\nb\n\r(c d)) ") == [:a, :b, [:c, :d]] - - # interpolation - x = 10 - y = "str" - @test SExprs.sx"(x ,y ,Int)" == [:x, "str", Int] - - # parse errors - @test_throws ErrorException parse(")") - @test_throws ErrorException parse("(") - @test_throws ErrorException parse("(,)") - @test_throws ErrorException parse(",") - @test_throws ErrorException parse(",(") - @test_throws ErrorException parse(",(a)") - @test_throws ErrorException parse("\"asdf") - @test_throws ErrorException parse("#") - - # printing - @test SExprs.deparse([1,2, [:aa, :bb], SExprs.Unquote(:cc)]) == "(1 2 (aa bb) ,cc)" -end From cc6030aa5d051b09c9c6442290cc48d755786b32 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 16 Jun 2019 20:42:26 +1000 Subject: [PATCH 22/56] Getindex lowering: additional special cases --- test/compiler/lowering.jl | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index d9592bd2c6d0d..aed9c00596bab 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -226,13 +226,24 @@ end @test_desugar a[[end]] Top.getindex(a, Top.vect(Top.lastindex(a))) @test_desugar a[b[end] + end] Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) @test_desugar a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) + @test_desugar a[end][b[i]] Top.getindex(Top.getindex(a, Top.lastindex(a)), Top.getindex(b,i)) + # Interaction of `end` with splatting - @test_desugar(a[I..., end], - Core._apply(Top.getindex, Core.tuple(a), I, - Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(I))))) + @test_desugar(a[I..., end, J..., end], + Core._apply(Top.getindex, Core.tuple(a), + I, + Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(I)))), + J, + Core.tuple(Top.lastindex(a, Top.:+(2, Top.length(J), Top.length(I))))) + ) + @test_desugar(a[f(x)..., end], + begin + ssa1 = f(x) + Core._apply(Top.getindex, Core.tuple(a), + ssa1, + Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(ssa1))))) + end ) - - @test_desugar_error a[i,j;k] "unexpected semicolon in array expression" end @testset "setindex!" begin From f31c479dfe2ba1c53ebdc6053d34588fbd354145 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Fri, 21 Jun 2019 23:40:04 +1000 Subject: [PATCH 23/56] List forms from expand-table + minor cleanups --- test/compiler/lowering.jl | 94 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index aed9c00596bab..a7487fa4526ab 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -265,6 +265,7 @@ end end @testset "Array notation" begin + # flisp: (in expand-table) @testset "Literals" begin @test_desugar [a,b] Top.vect(a,b) @test_desugar T[a,b] Top.getindex(T, a,b) # Only so much syntax to go round :-/ @@ -295,6 +296,8 @@ end @testset "Splatting" begin @test_desugar f(i,j,v...,k) Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) + + @test_desugar_error x... "\"...\" expression outside call" end @testset "Comparison chains" begin @@ -350,7 +353,7 @@ end ) end -@testset "Short circuit , ternary" begin +@testset "Short circuit; ternary" begin # flisp: (expand-or) (expand-and) @test_desugar a || b if a; a else b end @test_desugar a && b if a; b else false end @@ -363,9 +366,12 @@ end # when used as operators @test_desugar a <: b $(Expr(:call, :(<:), :a, :b)) @test_desugar a >: b $(Expr(:call, :(>:), :a, :b)) + + @test_desugar_error $(Expr(:$, :x)) "\"\$\" expression outside quote" end @testset "Broadcast" begin + # flisp: (expand-fuse-broadcast) # Basic @test_desugar x .+ y Top.materialize(Top.broadcasted(+, x, y)) @test_desugar f.(x) Top.materialize(Top.broadcasted(f, x)) @@ -393,7 +399,7 @@ end @test_desugar x .+= a Top.materialize!(x, Top.broadcasted(+, x, a)) end -@testset "Keyword arguments" begin +@testset "Call with keyword arguments" begin @test_desugar( f(x,a=1), begin @@ -1003,3 +1009,87 @@ end @test expand_forms(LineNumberNode(1, :foo)) == LineNumberNode(1, :foo) # flisp: `(line ,line ,file) end + +# flisp entry points for desugaring various forms. Many of these are inlined +# into expand-table. +# +# function expand-function-def +# -> expand-arrow +# let expand-let +# macro expand-macro-def +# struct expand-struct-def +# try expand-try +# lambda expand-table +# block expand-table +# |.| expand-fuse-broadcast +# .= expand-fuse-broadcast +# |<:| expand-table +# |>:| expand-table +# where expand-wheres +# const expand-const-decl +# local expand-local-or-global-decl +# global expand-local-or-global-decl +# local-def expand-local-or-global-decl +# = expand-table +# abstract expand-table +# primitive expand-table +# comparison expand-compare-chain +# ref partially-expand-ref +# curly expand-table +# call expand-table +# do expand-table +# tuple lower-named-tuple +# braces expand-table +# bracescat expand-table +# string expand-table +# :: expand-table +# while expand-table +# break expand-table +# continue expand-table +# for expand-for +# && expand-and +# || expand-or +# += lower-update-op +# -= lower-update-op +# *= lower-update-op +# .*= lower-update-op +# /= lower-update-op +# ./= lower-update-op +# //= lower-update-op +# .//= lower-update-op +# |\\=| lower-update-op +# |.\\=| lower-update-op +# |.+=| lower-update-op +# |.-=| lower-update-op +# ^= lower-update-op +# .^= lower-update-op +# ÷= lower-update-op +# .÷= lower-update-op +# %= lower-update-op +# .%= lower-update-op +# |\|=| lower-update-op +# |.\|=| lower-update-op +# &= lower-update-op +# .&= lower-update-op +# $= lower-update-op +# ⊻= lower-update-op +# .⊻= lower-update-op +# <<= lower-update-op +# .<<= lower-update-op +# >>= lower-update-op +# .>>= lower-update-op +# >>>= lower-update-op +# .>>>= lower-update-op +# |...| expand-table +# $ expand-table +# vect expand-table +# hcat expand-table +# vcat expand-table +# typed_hcat expand-table +# typed_vcat expand-table +# |'| expand-table +# generator expand-generator +# flatten expand-generator +# comprehension expand-table +# typed_comprehension lower-comprehension + From 8010fd39858b6a2ec209c59d97ca561f56aabd2e Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Mon, 24 Jun 2019 09:36:55 +1000 Subject: [PATCH 24/56] Rename internal Expr heads to use _ rather than - This makes these Expr heads more consistent and easier to work with from julia. --- base/boot.jl | 4 +- src/jlfrontend.scm | 2 +- src/julia-syntax.scm | 166 +++++++++++++++++++++---------------------- 3 files changed, 86 insertions(+), 86 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index a9f33562ee481..cd37dd5f14070 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -526,7 +526,7 @@ function (g::GeneratedFunctionStub)(@nospecialize args...) return body end lam = Expr(:lambda, g.argnames, - Expr(Symbol("scope-block"), + Expr(:scope_block, Expr(:block, LineNumberNode(g.line, g.file), Expr(:meta, :push_loc, g.file, Symbol("@generated body")), @@ -535,7 +535,7 @@ function (g::GeneratedFunctionStub)(@nospecialize args...) if g.spnames === nothing return lam else - return Expr(Symbol("with-static-parameters"), lam, g.spnames...) + return Expr(:with_static_parameters, lam, g.spnames...) end end diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index 5d363f323c80a..e0baa445dd207 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -92,7 +92,7 @@ (let* ((ex (julia-expand0 ex0)) (th (julia-expand1 `(lambda () () - (scope-block + (scope_block ,(blockify ex))) file line))) (if (and (null? (cdadr (caddr th))) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 3acae00674a38..27ababb57deb6 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -154,7 +154,7 @@ argl)) (body (blockify body))) `(lambda ,argl () - (scope-block + (scope_block ,(if (equal? rett '(core Any)) body (let ((meta (take-while (lambda (x) (and (pair? x) @@ -163,7 +163,7 @@ (R (make-ssavalue))) `(,(car body) ,@meta (= ,R ,rett) - (meta ret-type ,R) + (meta ret_type ,R) ,@(list-tail body (+ 1 (length meta)))))))))) ;; convert x<:T<:y etc. exprs into (name lower-bound upper-bound) @@ -250,7 +250,7 @@ (define (replace-vars e renames) (cond ((symbol? e) (lookup e renames e)) ((or (not (pair? e)) (quoted? e)) e) - ((memq (car e) '(-> function scope-block)) e) + ((memq (car e) '(-> function scope_block)) e) (else (cons (car e) (map (lambda (x) (replace-vars x renames)) @@ -841,7 +841,7 @@ field-names) `(block (global ,name) (const ,name) - (scope-block + (scope_block (block ,@(map (lambda (v) `(local ,v)) params) ,@(map (lambda (n v) (make-assignment n (bounds-to-TypeVar v #t))) params bounds) @@ -849,7 +849,7 @@ (call (core svec) ,@(map quotify field-names)) ,super (call (core svec) ,@field-types) ,mut ,min-initialized))) ;; "inner" constructors - (scope-block + (scope_block (block (global ,name) ,@(map (lambda (c) @@ -869,7 +869,7 @@ (and (expr-contains-eq (car p) (cons 'list root-types)) (loop (append (cdr p) root-types) (cdr sp))))))) - `((scope-block + `((scope_block (block (global ,name) ,(default-outer-ctor name field-names field-types @@ -882,7 +882,7 @@ (params bounds) (sparam-name-bounds params) `(block (global ,name) (const ,name) - (scope-block + (scope_block (block ,@(map (lambda (v) `(local ,v)) params) ,@(map (lambda (n v) (make-assignment n (bounds-to-TypeVar v #t))) params bounds) @@ -893,7 +893,7 @@ (params bounds) (sparam-name-bounds params) `(block (global ,name) (const ,name) - (scope-block + (scope_block (block ,@(map (lambda (v) `(local ,v)) params) ,@(map (lambda (n v) (make-assignment n (bounds-to-TypeVar v #t))) params bounds) @@ -1076,7 +1076,7 @@ (expand-forms (if (null? binds) - `(scope-block (block ,ex)) + `(scope_block (block ,ex)) (let loop ((binds (reverse binds)) (blk ex)) (if (null? binds) @@ -1085,7 +1085,7 @@ ((or (symbol? (car binds)) (decl? (car binds))) ;; just symbol -> add local (loop (cdr binds) - `(scope-block + `(scope_block (block (local ,(car binds)) ,blk)))) @@ -1100,11 +1100,11 @@ (if (not (symbol? name)) (error "invalid let syntax")) (loop (cdr binds) - `(scope-block + `(scope_block (block ,(if (expr-contains-eq name (caddar binds)) `(local ,name) ;; might need a Box for recursive functions - `(local-def ,name)) + `(local_def ,name)) ,asgn ,blk))))) ((or (symbol? (cadar binds)) @@ -1113,16 +1113,16 @@ (loop (cdr binds) (if (expr-contains-eq vname (caddar binds)) (let ((tmp (make-ssavalue))) - `(scope-block + `(scope_block (block (= ,tmp ,(caddar binds)) - (scope-block + (scope_block (block - (local-def ,(cadar binds)) + (local_def ,(cadar binds)) (= ,vname ,tmp) ,blk))))) - `(scope-block + `(scope_block (block - (local-def ,(cadar binds)) + (local_def ,(cadar binds)) (= ,vname ,(caddar binds)) ,blk)))))) ;; (a, b, c, ...) = rhs @@ -1135,14 +1135,14 @@ (let ((temp (make-ssavalue))) `(block (= ,temp ,(caddr (car binds))) - (scope-block + (scope_block (block - ,@(map (lambda (v) `(local-def ,v)) vars) + ,@(map (lambda (v) `(local_def ,v)) vars) (= ,(cadr (car binds)) ,temp) ,blk)))) - `(scope-block + `(scope_block (block - ,@(map (lambda (v) `(local-def ,v)) vars) + ,@(map (lambda (v) `(local_def ,v)) vars) ,(car binds) ,blk)))))) (else (error "invalid let syntax")))) @@ -1223,17 +1223,17 @@ `(tryfinally ,(if (not (eq? catchb 'false)) `(try ,tryb ,var ,catchb) - `(scope-block ,tryb)) - (scope-block ,finalb))))) + `(scope_block ,tryb)) + (scope_block ,finalb))))) ((length= e 4) (expand-forms (if (and (symbol-like? var) (not (eq? var 'false))) - `(trycatch (scope-block ,tryb) - (scope-block + `(trycatch (scope_block ,tryb) + (scope_block (block (= ,var (the_exception)) ,catchb))) - `(trycatch (scope-block ,tryb) - (scope-block ,catchb))))) + `(trycatch (scope_block ,tryb) + (scope_block ,catchb))))) (else (error "invalid \"try\" form"))))) @@ -1245,7 +1245,7 @@ (if (null? params) (error (string "empty type parameter list in \"" (deparse `(= (curly ,name) ,type-ex)) "\""))) `(block - (const-if-global ,name) + (const_if_global ,name) ,(expand-forms `(= ,name (where ,type-ex ,@params))))) (expand-forms @@ -1257,7 +1257,7 @@ (if (atom? arg) e (case (car arg) - ((global local local-def) + ((global local local_def) (for-each (lambda (b) (if (not (assignment? b)) (error "expected assignment after \"const\""))) (cdr arg)) @@ -1543,8 +1543,8 @@ (apply append (map lhs-vars (filter (lambda (x) (not (outer? x))) (butlast lhss)))))))) - `(break-block - loop-exit + `(break_block + loop_exit ,(let nest ((lhss lhss) (itrs itrs)) (if (null? lhss) @@ -1563,16 +1563,16 @@ ,(nest (cdr lhss) (cdr itrs)))) (body (if (null? (cdr lhss)) - `(break-block - loop-cont + `(break_block + loop_cont (let (block ,@(map (lambda (v) `(= ,v ,v)) copied-vars)) ,body)) - `(scope-block ,body)))) + `(scope_block ,body)))) `(block (= ,coll ,(car itrs)) (= ,next (call (top iterate) ,coll)) ;; TODO avoid `local declared twice` error from this ;;,@(if outer `((local ,lhs)) '()) - ,@(if outer `((require-existing-local ,lhs)) '()) + ,@(if outer `((require_existing_local ,lhs)) '()) (if (call (top not_int) (call (core ===) ,next (null))) (_do_while (block ,body @@ -1848,7 +1848,7 @@ 'const expand-const-decl 'local expand-local-or-global-decl 'global expand-local-or-global-decl - 'local-def expand-local-or-global-decl + 'local_def expand-local-or-global-decl '= (lambda (e) @@ -2129,18 +2129,18 @@ 'while (lambda (e) - `(break-block loop-exit + `(break_block loop_exit (_while ,(expand-forms (cadr e)) - (break-block loop-cont - (scope-block ,(blockify (expand-forms (caddr e)))))))) + (break_block loop_cont + (scope_block ,(blockify (expand-forms (caddr e)))))))) 'break (lambda (e) (if (pair? (cdr e)) e - '(break loop-exit))) + '(break loop_exit))) - 'continue (lambda (e) '(break loop-cont)) + 'continue (lambda (e) '(break loop_cont)) 'for (lambda (e) @@ -2326,7 +2326,7 @@ ,(construct-loops (cdr itrs) (cdr iv))))) (let ((overall-itr (if (length= itrs 1) (car iv) prod))) - `(scope-block + `(scope_block (block (local ,result) (local ,idx) ,.(map (lambda (v r) `(= ,v ,(caddr r))) iv itrs) @@ -2369,7 +2369,7 @@ (if (or (not (pair? e)) (quoted? e)) '() (case (car e) - ((lambda scope-block module toplevel) '()) + ((lambda scope_block module toplevel) '()) ((method) (let ((v (decl-var (method-expr-name e)))) (append! @@ -2389,7 +2389,7 @@ (define (find-decls kind e) (if (or (not (pair? e)) (quoted? e)) '() - (cond ((memq (car e) '(lambda scope-block module toplevel)) + (cond ((memq (car e) '(lambda scope_block module toplevel)) '()) ((eq? (car e) kind) (if (underscore-symbol? (cadr e)) @@ -2400,7 +2400,7 @@ e)))))) (define (find-local-decls e) (find-decls 'local e)) -(define (find-local-def-decls e) (find-decls 'local-def e)) +(define (find-local-def-decls e) (find-decls 'local_def e)) (define (find-global-decls e) (find-decls 'global e)) (define (check-valid-name e) @@ -2458,11 +2458,11 @@ ((eq? (car e) 'global) (check-valid-name (cadr e)) e) - ((memq (car e) '(local local-def)) + ((memq (car e) '(local local_def)) (check-valid-name (cadr e)) ;; remove local decls '(null)) - ((eq? (car e) 'require-existing-local) + ((eq? (car e) 'require_existing_local) (if (not (in-scope? (cadr e) scope)) (error "no outer local variable declaration exists for \"for outer\"")) '(null)) @@ -2486,8 +2486,8 @@ (let* ((args (lam:vars e)) (body (resolve-scopes- (lam:body e) (make-scope e args '() '() sp '() scope)))) `(lambda ,(cadr e) ,(caddr e) ,body))) - ((eq? (car e) 'scope-block) - (let* ((blok (cadr e)) ;; body of scope-block expression + ((eq? (car e) 'scope_block) + (let* ((blok (cadr e)) ;; body of scope_block expression (lam (scope:lam scope)) (argnames (lam:vars lam)) (current-locals (caddr lam)) ;; locals created so far in our lambda @@ -2532,7 +2532,7 @@ (if lam (set-car! (cddr lam) (append (caddr lam) newnames newnames-def))) - (insert-after-meta ;; return the new, expanded scope-block + (insert-after-meta ;; return the new, expanded scope_block (blockify (resolve-scopes- blok (make-scope lam @@ -2545,15 +2545,15 @@ (map cons need-rename-def renamed-def)) scope))) (append! (map (lambda (v) `(local ,v)) newnames) - (map (lambda (v) `(local-def ,v)) newnames-def))) + (map (lambda (v) `(local_def ,v)) newnames-def))) )) ((eq? (car e) 'module) (error "\"module\" expression not at top level")) - ((eq? (car e) 'break-block) - `(break-block ,(cadr e) ;; ignore type symbol of break-block expression - ,(resolve-scopes- (caddr e) scope))) ;; body of break-block expression - ((eq? (car e) 'with-static-parameters) - `(with-static-parameters + ((eq? (car e) 'break_block) + `(break_block ,(cadr e) ;; ignore type symbol of break_block expression + ,(resolve-scopes- (caddr e) scope))) ;; body of break_block expression + ((eq? (car e) 'with_static_parameters) + `(with_static_parameters ,(resolve-scopes- (cadr e) scope (cddr e)) ,@(cddr e))) ((and (eq? (car e) 'method) (length> e 2)) @@ -2579,8 +2579,8 @@ (cond ((or (eq? e 'true) (eq? e 'false) (eq? e UNUSED) (underscore-symbol? e)) tab) ((symbol? e) (put! tab e #t)) ((and (pair? e) (eq? (car e) 'outerref)) tab) - ((and (pair? e) (eq? (car e) 'break-block)) (free-vars- (caddr e) tab)) - ((and (pair? e) (eq? (car e) 'with-static-parameters)) (free-vars- (cadr e) tab)) + ((and (pair? e) (eq? (car e) 'break_block)) (free-vars- (caddr e) tab)) + ((and (pair? e) (eq? (car e) 'with_static_parameters)) (free-vars- (cadr e) tab)) ((or (atom? e) (quoted? e)) tab) ((eq? (car e) 'lambda) (let ((bound (lambda-all-vars e))) @@ -2654,7 +2654,7 @@ (vinfo:set-read! vi #t)))) e) (case (car e) - ((local-def) ;; a local that we know has an assignment that dominates all usages + ((local_def) ;; a local that we know has an assignment that dominates all usages (let ((vi (var-info-for (cadr e) env))) (vinfo:set-never-undef! vi #t))) ((=) @@ -2685,8 +2685,8 @@ (vinfo:set-type! vi (caddr e)))))) ((lambda) (analyze-vars-lambda e env captvars sp '())) - ((with-static-parameters) - ;; (with-static-parameters func_expr sp_1 sp_2 ...) + ((with_static_parameters) + ;; (with_static_parameters func_expr sp_1 sp_2 ...) (assert (eq? (car (cadr e)) 'lambda)) (analyze-vars-lambda (cadr e) env captvars sp (cddr e))) @@ -2889,7 +2889,7 @@ f(x) = yt(x) `(call (core svec) (call (core svec) ,@newtypes) (call (core svec) ,@(append (cddr (cadddr te)) type-sp))))) -;; collect all toplevel-butfirst expressions inside `e`, and return +;; collect all toplevel_butfirst expressions inside `e`, and return ;; (ex . stmts), where `ex` is the expression to evaluated and ;; `stmts` is a list of statements to move to the top level. (define (lift-toplevel e) @@ -2898,7 +2898,7 @@ f(x) = yt(x) (if (or (atom? e) (quoted? e)) e (let ((e (cons (car e) (map lift- (cdr e))))) - (if (eq? (car e) 'toplevel-butfirst) + (if (eq? (car e) 'toplevel_butfirst) (begin (set! top (cons (cddr e) top)) (cadr e)) e)))) @@ -2956,10 +2956,10 @@ f(x) = yt(x) (lambda (x) (and (pair? x) (not (eq? (car x) 'lambda))))))) (define lambda-opt-ignored-exprs - (Set '(quote top core line inert local local-def unnecessary copyast + (Set '(quote top core line inert local local_def unnecessary copyast meta inbounds boundscheck loopinfo decl aliasscope popaliasscope - struct_type abstract_type primitive_type thunk with-static-parameters - global globalref outerref const-if-global + struct_type abstract_type primitive_type thunk with_static_parameters + global globalref outerref const_if_global const null ssavalue isdefined toplevel module lambda error gc_preserve_begin gc_preserve_end import using export))) @@ -3018,11 +3018,11 @@ f(x) = yt(x) #f) ((lambda-opt-ignored-exprs (car e)) #f) - ((eq? (car e) 'scope-block) + ((eq? (car e) 'scope_block) (visit (cadr e))) ((memq (car e) '(block call new splatnew _do_while)) (eager-any visit (cdr e))) - ((eq? (car e) 'break-block) + ((eq? (car e) 'break_block) (visit (caddr e))) ((eq? (car e) 'return) (begin0 (visit (cadr e)) @@ -3152,7 +3152,7 @@ f(x) = yt(x) (let ((var (cadr e)) (rhs (cl-convert (caddr e) fname lam namemap defined toplevel interp))) (convert-assignment var rhs fname lam interp))) - ((local-def) ;; make new Box for local declaration of defined variable + ((local_def) ;; make new Box for local declaration of defined variable (let ((vi (assq (cadr e) (car (lam:vinfo lam))))) (if (and vi (vinfo:asgn vi) (vinfo:capt vi)) `(= ,(cadr e) (call (core Box))) @@ -3165,7 +3165,7 @@ f(x) = yt(x) '(null) `(newvar ,(cadr e)))))) ((const) e) - ((const-if-global) + ((const_if_global) (if (local-in? (cadr e) lam) '(null) `(const ,(cadr e)))) @@ -3220,8 +3220,8 @@ f(x) = yt(x) e (begin (put! defined (cadr e) #t) - `(toplevel-butfirst - ;; wrap in toplevel-butfirst so it gets moved higher along with + `(toplevel_butfirst + ;; wrap in toplevel_butfirst so it gets moved higher along with ;; closure type definitions ,e (thunk (lambda () (() () 0 ()) (block (return ,e)))))))) @@ -3244,7 +3244,7 @@ f(x) = yt(x) (let* ((exprs (lift-toplevel (convert-lambda lam2 '|#anon| #t '()))) (top-stmts (cdr exprs)) (newlam (compact-and-renumber (linearize (car exprs)) 'none 0))) - `(toplevel-butfirst + `(toplevel_butfirst (block ,@sp-inits (method ,name ,(cl-convert sig fname lam namemap defined toplevel interp) ,(julia-bq-macro newlam))) @@ -3378,16 +3378,16 @@ f(x) = yt(x) (not (memq (car vi) moved-vars))) (car (lam:vinfo lam))))) (if (or exists (and short (pair? alldefs))) - `(toplevel-butfirst + `(toplevel_butfirst (null) ,@sp-inits ,@mk-method) (begin (put! defined name #t) - `(toplevel-butfirst + `(toplevel_butfirst ,(convert-assignment name mk-closure fname lam interp) ,@typedef - ,@(map (lambda (v) `(moved-local ,v)) moved-vars) + ,@(map (lambda (v) `(moved_local ,v)) moved-vars) ,@sp-inits ,@mk-method)))))))) ((lambda) ;; happens inside (thunk ...) and generated function bodies @@ -3416,8 +3416,8 @@ f(x) = yt(x) (if (or (symbol? (cadr e)) (and (pair? (cadr e)) (eq? (caadr e) 'outerref))) (error "type declarations on global variables are not yet supported")) (cl-convert `(call (core typeassert) ,@(cdr e)) fname lam namemap defined toplevel interp)))) - ;; `with-static-parameters` expressions can be removed now; used only by analyze-vars - ((with-static-parameters) + ;; `with_static_parameters` expressions can be removed now; used only by analyze-vars + ((with_static_parameters) (cl-convert (cadr e) fname lam namemap defined toplevel interp)) (else (if (eq? (car e) 'struct_type) @@ -3619,7 +3619,7 @@ f(x) = yt(x) (emit `(= ,lhs ,rr))))) #f) ;; the interpreter loop. `break-labels` keeps track of the labels to jump to - ;; for all currently closing break-blocks. + ;; for all currently closing break_blocks. ;; `value` means we are in a context where a value is required; a meaningful ;; value must be returned. ;; `tail` means we are in tail position, where a value needs to be `return`ed @@ -3776,7 +3776,7 @@ f(x) = yt(x) (emit `(goto ,topl)) (mark-label endl)) (if value (compile '(null) break-labels value tail))) - ((break-block) + ((break_block) (let ((endl (make-label))) (compile (caddr e) (cons (list (cadr e) endl handler-level catch-token-stack) @@ -3888,9 +3888,9 @@ f(x) = yt(x) (if (var-info-for vname vi) ;; issue #7264 (error (string "`global " vname "`: " vname " is a local variable in its enclosing scope")) (emit e)))) - ((local-def) #f) + ((local_def) #f) ((local) #f) - ((moved-local) + ((moved_local) (set-car! (lam:vinfo lam) (append (car (lam:vinfo lam)) `((,(cadr e) Any 2)))) #f) ((const) @@ -3985,7 +3985,7 @@ f(x) = yt(x) (emit e)) ;; strip filenames out of non-initial line nodes (emit `(line ,(cadr e))))) - ((and (eq? (car e) 'meta) (length> e 2) (eq? (cadr e) 'ret-type)) + ((and (eq? (car e) 'meta) (length> e 2) (eq? (cadr e) 'ret_type)) (assert (or (not value) tail)) (assert (not rett)) (set! rett (caddr e))) From 9f4c547312730d54376f221924fa584fcb52517d Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Mon, 24 Jun 2019 09:40:11 +1000 Subject: [PATCH 25/56] Hack: Change show() for exotic Expr heads to use @ Expr macro Useful for writing the tests. May also be useful in general, though expanding it involves macro expansion rather than unquote expansion. --- base/show.jl | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/base/show.jl b/base/show.jl index 037c66ae95f50..3d3088e3a10ec 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1477,13 +1477,9 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int) unhandled = true end if unhandled - print(io, "\$(Expr(") - show(io, ex.head) - for arg in args - print(io, ", ") - show(io, arg) - end - print(io, "))") + # Always shown this macro call with parens for clarity + show_call(io, :call, Symbol("@Expr"), + [QuoteNode(ex.head), ex.args...], indent) end nothing end From d1bfad9c86259734e4c655e9ff59447dc993f0e5 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Mon, 24 Jun 2019 09:38:54 +1000 Subject: [PATCH 26/56] Use macro @ Expr(...) to replace `$(Expr(...))` A more readable alternative to writing $(Expr(head, ...)). This is often more readable because the entire expression can be quoted code, as opposed to being a mixture of quoted and unquoted parts. --- test/compiler/lowering.jl | 666 +++++++++++++++++++++----------------- 1 file changed, 364 insertions(+), 302 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index a7487fa4526ab..4703267481574 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1,4 +1,5 @@ using Core: SSAValue +using Base: remove_linenums! # Call into lowering stage 1; syntax desugaring function fl_expand_forms(ex) @@ -32,7 +33,6 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) end end if ex isa Expr - filter!(e->!(e isa LineNumberNode), ex.args) if ex.head == :block && length(ex.args) == 1 # Remove trivial blocks return lift_lowered_expr!(ex.args[1], nextids, valmap, lift_full) @@ -51,8 +51,12 @@ function lift_lowered_expr!(ex, nextids, valmap, lift_full) # `unnecessary` marks expressions generated by lowering that # do not need to be evaluated if their value is unused. return Expr(:call, :maybe_unused, ex.args...) - elseif ex.head == Symbol("scope-block") - return Expr(:let, Expr(:block), ex.args[1]) + #= + elseif ex.head in [:_while, :_do_while, :scope_block, :break_block, + :break, :local_def, :require_existing_local] + return Expr(:macrocall, Symbol("@Expr"), nothing, + QuoteNode(ex.head), ex.args...) + =# end end elseif ex isa Vector # Occasional case of lambdas @@ -72,7 +76,7 @@ hand """ function lift_lowered_expr(ex; lift_full=false) valmap = Dict{Union{Symbol,SSAValue},Symbol}() - lift_lowered_expr!(deepcopy(ex), ones(Int,2), valmap, lift_full) + lift_lowered_expr!(remove_linenums!(deepcopy(ex)), ones(Int,2), valmap, lift_full) end """ @@ -85,7 +89,6 @@ desugared forms. """ function lower_ref_expr!(ex) if ex isa Expr - filter!(e->!(e isa LineNumberNode), ex.args) map!(lower_ref_expr!, ex.args, ex.args) if ex.head == :block && length(ex.args) == 1 # Remove trivial blocks @@ -101,13 +104,15 @@ function lower_ref_expr!(ex) return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused return Expr(:unnecessary, ex.args[2:end]...) - elseif ex.head == :let && isempty(ex.args[1].args) - return Expr(Symbol("scope-block"), ex.args[2]) + elseif ex.head == :macrocall && ex.args[1] == Symbol("@Expr") + head = ex.args[3] + head = head isa QuoteNode ? head.value : Symbol(head) + return Expr(head, ex.args[4:end]...) end end return ex end -lower_ref_expr(ex) = lower_ref_expr!(deepcopy(ex)) +lower_ref_expr(ex) = lower_ref_expr!(remove_linenums!(deepcopy(ex))) function diffdump(io::IOContext, ex1, ex2, n, prefix, indent) @@ -144,7 +149,23 @@ function diffdump(ex1, ex2; maxdepth=20) end # For interactive convenience in constructing test cases with flisp based lowering -desugar(ex; lift_full=true) = lift_lowered_expr(expand_forms(ex); lift_full=lift_full) +function desugar(ex; lift=:full) + expanded = expand_forms(ex) + if lift == :full || lift == :partial + lift_lowered_expr(expanded; lift_full=(lift == :full)) + else + expanded + end +end + +# Macro for producing exotic `Expr`s found in lowered ASTs +# +# Note that this is provided for convenience/reference but in practice we +# expand it "manually" inside lower_ref_expr! +macro Ex(head, args...) + head = head isa QuoteNode ? head.value : Symbol(head) + esc(Expr(head, args...)) +end """ @desugar ex [kws...] @@ -539,14 +560,14 @@ end # const @test_desugar((const x=a), begin - $(Expr(:const, :x)) # `const x` is invalid surface syntax + @Expr :const x # `const x` is invalid surface syntax x = a end ) @test_desugar((const x,y = a,b), begin - $(Expr(:const, :x)) - $(Expr(:const, :y)) + @Expr :const x + @Expr :const y begin x = a y = b @@ -600,7 +621,7 @@ end # type decl @test_desugar(x::T = a, begin - $(Expr(:decl, :x, :T)) + @Expr :decl x T x = a end ) @@ -608,12 +629,13 @@ end # type aliases @test_desugar(A{T} = B{T}, begin - $(Expr(Symbol("const-if-global"), :A)) - A = let - $(Expr(Symbol("local-def"), :T)) - T = Core.TypeVar(:T) - Core.UnionAll(T, Core.apply_type(B, T)) - end + @Expr :const_if_global A + A = @Expr(:scope_block, + begin + @Expr :local_def T + T = Core.TypeVar(:T) + Core.UnionAll(T, Core.apply_type(B, T)) + end) end ) @@ -624,121 +646,123 @@ end @test_desugar(let x,y body end, - begin - let - local x - let - local y - body - end - end - end + @Expr(:scope_block, + begin + local x + @Expr(:scope_block, + begin + local y + body + end) + end) ) # Let with assignment @test_desugar(let x=a,y=b body end, - begin - let - $(Expr(Symbol("local-def"), :x)) - x = a - let - $(Expr(Symbol("local-def"), :y)) - y = b - body - end - end - end + @Expr(:scope_block, + begin + @Expr :local_def x + x = a + @Expr(:scope_block, + begin + @Expr :local_def y + y = b + body + end) + end) ) # TODO: More coverage. Internals look complex. end @testset "Loops" begin # flisp: (expand-for) (lambda in expand-forms) - @test_desugar(while cond - body1 - continue - body2 - break - body3 - end, - $(Expr(Symbol("break-block"), Symbol("loop-exit"), - Expr(:_while, :cond, - Expr(Symbol("break-block"), Symbol("loop-cont"), - :(let - body1 - $(Expr(:break, Symbol("loop-cont"))) - body2 - $(Expr(:break, Symbol("loop-exit"))) - body3 - end))))) + @test_desugar( + while cond + body1 + continue + body2 + break + body3 + end, + @Expr(:break_block, loop_exit, + @Expr(:_while, cond, + @Expr(:break_block, loop_cont, + @Expr(:scope_block, begin + body1 + @Expr :break loop_cont + body2 + @Expr :break loop_exit + body3 + end)))) ) - @test_desugar(for i = a - body1 - continue - body2 - break - end, - $(Expr(Symbol("break-block"), Symbol("loop-exit"), - quote - ssa1 = a - gsym1 = Top.iterate(ssa1) - if Top.not_int(Core.:(===)(gsym1, $nothing)) - $(Expr(:_do_while, - quote - $(Expr(Symbol("break-block"), Symbol("loop-cont"), - :(let - local i - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - begin - body1 - $(Expr(:break, Symbol("loop-cont"))) - body2 - $(Expr(:break, Symbol("loop-exit"))) - end - end))) - gsym1 = Top.iterate(ssa1, ssa3) - end, - :(Top.not_int(Core.:(===)(gsym1, $nothing))))) - end - end)) + @test_desugar( + for i = a + body1 + continue + body2 + break + end, + @Expr(:break_block, loop_exit, + begin + ssa1 = a + gsym1 = Top.iterate(ssa1) + if Top.not_int(Core.:(===)(gsym1, $nothing)) + @Expr(:_do_while, + begin + @Expr(:break_block, loop_cont, + @Expr(:scope_block, + begin + local i + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + begin + body1 + @Expr :break loop_cont + body2 + @Expr :break loop_exit + end + end)) + gsym1 = Top.iterate(ssa1, ssa3) + end, + Top.not_int(Core.:(===)(gsym1, $nothing))) + end + end) ) # For loops with `outer` @test_desugar(for outer i = a body end, - $(Expr(Symbol("break-block"), Symbol("loop-exit"), - quote - ssa1 = a - gsym1 = Top.iterate(ssa1) - $(Expr(Symbol("require-existing-local"), :i)) # Cf above. - if Top.not_int(Core.:(===)(gsym1, $nothing)) - $(Expr(:_do_while, - quote - $(Expr(Symbol("break-block"), Symbol("loop-cont"), - :(let - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - begin - body - end - end))) - gsym1 = Top.iterate(ssa1, ssa3) - end, - :(Top.not_int(Core.:(===)(gsym1, $nothing))))) - end - end)) + @Expr(:break_block, loop_exit, + begin + ssa1 = a + gsym1 = Top.iterate(ssa1) + @Expr(:require_existing_local, i) + if Top.not_int(Core.:(===)(gsym1, $nothing)) + @Expr(:_do_while, + begin + @Expr(:break_block, loop_cont, + @Expr(:scope_block, + begin + begin + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + body + end)) + gsym1 = Top.iterate(ssa1, ssa3) + end, + Top.not_int(Core.:(===)(gsym1, $nothing))) + end + end) ) end @@ -746,13 +770,12 @@ end # Short form @test_desugar(f(x) = body(x), begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x], [], - :(let - body(x) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, + body(x)))) maybe_unused(f) end ) @@ -762,13 +785,12 @@ end body(x) end, begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x, :y], [], - :(let - body(x) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:scope_block, + body(x)))) maybe_unused(f) end ) @@ -779,33 +801,27 @@ end end, begin begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f)), Core.svec())), - Expr(:lambda, [:_self_], [], - :(let - _self_(a, b) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f)), Core.svec()), + @Expr(:lambda, $([:_self_]), $([]), + @Expr(:scope_block, _self_(a, b)))) maybe_unused(f) end begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x], [], - :(let - _self_(x, b) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, _self_(x, b)))) maybe_unused(f) end begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x, :y], [], - :(let - body(x,y) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:scope_block, body(x, y)))) maybe_unused(f) end end @@ -816,13 +832,12 @@ end body(x, args) end, begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.apply_type(Vararg, Core.Any)), Core.svec())), - Expr(:lambda, [:_self_, :x, :args], [], - :(let - body(x, args) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any, + Core.apply_type(Vararg, Core.Any)), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :args]), $([]), + @Expr(:scope_block, body(x, args)))) maybe_unused(f) end ) @@ -831,67 +846,61 @@ end @test_desugar(function f(x; k1=v1, k2=v2) body end, + Core.ifelse(false, false, begin - Core.ifelse(false, false, + @Expr(:method, f) begin - $(Expr(:method, :f)) - begin - $(Expr(:method, :gsym1)) - $(Expr(:method, :gsym1, - :(Core.svec(Core.svec(Core.typeof(gsym1), Core.Any, Core.Any, Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:gsym1, :k1, :k2, :_self_, :x], [], - :(let - body - end)))) - maybe_unused(gsym1) - end - begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x], [], - :(let - return gsym1(v1, v2, _self_, x) - end)))) - maybe_unused(f) - end - begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.kwftype(Core.Typeof(f)), Core.Any, Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:gsym2, :gsym3, :_self_, :x], [], - :(let - let - $(Expr(Symbol("local-def"), :k1)) - k1 = if Top.haskey(gsym3, :k1) - Top.getindex(gsym3, :k1) - else - v1 - end - let - $(Expr(Symbol("local-def"), :k2)) - k2 = if Top.haskey(gsym3, :k2) - Top.getindex(gsym3, :k2) + @Expr(:method, gsym1) + @Expr(:method, gsym1, + Core.svec(Core.svec(Core.typeof(gsym1), Core.Any, Core.Any, Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:gsym1, :k1, :k2, :_self_, :x]), $([]), + @Expr(:scope_block, body))) + maybe_unused(gsym1) + end + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, return gsym1(v1, v2, _self_, x)))) + maybe_unused(f) + end + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.kwftype(Core.Typeof(f)), Core.Any, Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:gsym2, :gsym3, :_self_, :x]), $([]), + @Expr(:scope_block, + @Expr(:scope_block, + begin + @Expr(:local_def, k1) + k1 = if Top.haskey(gsym3, :k1) + Top.getindex(gsym3, :k1) else - v2 + v1 end - begin - ssa1 = Top.pairs(Top.structdiff(gsym3, Core.apply_type(Core.NamedTuple, Core.tuple(:k1, :k2)))) - if Top.isempty(ssa1) - $nothing - else - Top.kwerr(gsym3, _self_, x) - end - return gsym1(k1, k2, _self_, x) - end - end - end - end)))) - maybe_unused(f) - end - f - end) - end + @Expr(:scope_block, begin + @Expr(:local_def, k2) + k2 = if Top.haskey(gsym3, :k2) + Top.getindex(gsym3, :k2) + else + v2 + end + begin + ssa1 = Top.pairs(Top.structdiff(gsym3, Core.apply_type(Core.NamedTuple, Core.tuple(:k1, :k2)))) + if Top.isempty(ssa1) + $nothing + else + Top.kwerr(gsym3, _self_, x) + end + return gsym1(k1, k2, _self_, x) + end + end) + end)))) + maybe_unused(f) + end + f + end) ) # Return type declaration @@ -899,15 +908,16 @@ end body(x) end, begin - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x], [], - :(let - ssa1 = T - $(Expr(:meta, Symbol("ret-type"), :ssa1)) - body(x) - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, + begin + ssa1 = T + @Expr(:meta, ret_type, ssa1) + body(x) + end))) maybe_unused(f) end ) @@ -917,13 +927,11 @@ end begin local gsym1 begin - $(Expr(:method, :gsym1)) - $(Expr(:method, :gsym1, - :(Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x, :y], [], - :(let - body(x, y) - end)))) + @Expr(:method, gsym1) + @Expr(:method, gsym1, + Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:scope_block, body(x, y)))) maybe_unused(gsym1) end end @@ -956,42 +964,46 @@ end begin global gsym1 begin - $(Expr(:method, :gsym1)) - $(Expr(:method, :gsym1, - :(Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :gsym2, :x], [], - :(let - $(Expr(:meta, :nospecialize, :gsym2, :x)) - Core._expr(:block, - $(QuoteNode(LineNumberNode(ln.line, ln.file))), - $(Expr(:copyast, QuoteNode(:(body1(x))))), - # FIXME: These line numbers seem buggy? - $(QuoteNode(LineNumberNode(ln.line+1, ln.file))), - gen_body1(x), - $(QuoteNode(LineNumberNode(ln.line+6, ln.file))), - :body2, - $(QuoteNode(LineNumberNode(ln.line+7, ln.file))), - gen_body2 - ) - end)))) + @Expr(:method, gsym1) + @Expr(:method, gsym1, + Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :gsym2, :x]), $([]), + @Expr(:scope_block, + begin + @Expr(:meta, nospecialize, gsym2, x) + Core._expr(:block, + $(QuoteNode(LineNumberNode(ln.line, ln.file))), + @Expr(:copyast, $(QuoteNode(:(body1(x))))), + # FIXME: These line numbers seem buggy? + $(QuoteNode(LineNumberNode(ln.line+1, ln.file))), + gen_body1(x), + $(QuoteNode(LineNumberNode(ln.line+6, ln.file))), + :body2, + $(QuoteNode(LineNumberNode(ln.line+7, ln.file))), + gen_body2) + end))) maybe_unused(gsym1) end end - $(Expr(:method, :f)) - $(Expr(:method, :f, - :(Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec())), - Expr(:lambda, [:_self_, :x], [], - :(let - $(Expr(:meta, :generated, - Expr(:new, :(Core.GeneratedFunctionStub), - :gsym1, [:_self_, :x], - :nothing, ln.line, - QuoteNode(ln.file), false))) - body1(x) - normal_body1(x) - body2 - normal_body2 - end)))) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, + begin + @Expr(:meta, generated, + @Expr(:new, + Core.GeneratedFunctionStub, + gsym1, $([:_self_, :x]), + nothing, + $(ln.line), + $(QuoteNode(ln.file)), + false)) + body1(x) + normal_body1(x) + body2 + normal_body2 + end))) maybe_unused(f) end ) @@ -1010,8 +1022,17 @@ end end -# flisp entry points for desugaring various forms. Many of these are inlined -# into expand-table. +#------------------------------------------------------------------------------- +# Julia AST Notes +# +# Broadly speaking there's three categories of `Expr` expression heads: +# * Forms which represent normal julia surface syntax +# * Special forms which are emitted by macros in the public API, but which +# have no normal syntax. +# * Forms which are used internally as part of lowering + +# Here's the forms which are transformed as part of the desugaring pass in +# expand-table: # # function expand-function-def # -> expand-arrow @@ -1021,15 +1042,15 @@ end # try expand-try # lambda expand-table # block expand-table -# |.| expand-fuse-broadcast +# . expand-fuse-broadcast # .= expand-fuse-broadcast -# |<:| expand-table -# |>:| expand-table +# <: expand-table +# >: expand-table # where expand-wheres # const expand-const-decl # local expand-local-or-global-decl # global expand-local-or-global-decl -# local-def expand-local-or-global-decl +# local_def expand-local-or-global-decl # = expand-table # abstract expand-table # primitive expand-table @@ -1049,47 +1070,88 @@ end # for expand-for # && expand-and # || expand-or -# += lower-update-op -# -= lower-update-op -# *= lower-update-op -# .*= lower-update-op -# /= lower-update-op -# ./= lower-update-op -# //= lower-update-op -# .//= lower-update-op -# |\\=| lower-update-op -# |.\\=| lower-update-op -# |.+=| lower-update-op -# |.-=| lower-update-op -# ^= lower-update-op -# .^= lower-update-op -# ÷= lower-update-op -# .÷= lower-update-op -# %= lower-update-op -# .%= lower-update-op -# |\|=| lower-update-op -# |.\|=| lower-update-op -# &= lower-update-op -# .&= lower-update-op -# $= lower-update-op -# ⊻= lower-update-op -# .⊻= lower-update-op -# <<= lower-update-op -# .<<= lower-update-op -# >>= lower-update-op -# .>>= lower-update-op -# >>>= lower-update-op -# .>>>= lower-update-op -# |...| expand-table +# += -= *= .*= /= ./= lower-update-op +# //= .//= \\= .\\= +# .+= .-= ^= .^= ÷= +# .÷= %= .%= |= .|= +# &= .&= $= ⊻= .⊻= +# <<= .<<= >>= .>>= +# >>>= .>>>= +# ... expand-table # $ expand-table # vect expand-table # hcat expand-table # vcat expand-table # typed_hcat expand-table # typed_vcat expand-table -# |'| expand-table +# ' expand-table # generator expand-generator # flatten expand-generator # comprehension expand-table # typed_comprehension lower-comprehension +# Heads of internal AST forms (incomplete) +# +# Emitted by public macros: +# inbounds @inbounds +# boundscheck @boundscheck +# isdefined @isdefined +# generated @generated +# locals Base.@locals +# meta @inline, @noinline, ... +# symbolicgoto @goto +# symboliclabel @label +# gc_preserve_begin GC.@preserve +# gc_preserve_end GC.@preserve +# foreigncall ccall +# loopinfo @simd +# +# Scoping and variables: +# scope_block +# toplevel_butfirst +# toplevel +# aliasscope +# popaliasscope +# require_existing_local +# local_def +# const_if_global +# top +# core +# globalref +# outerref +# +# Looping: +# _while +# _do_while +# break_block +# +# Types: +# new +# splatnew +# +# Functions: +# lambda +# method +# ret_type +# +# Errors: +# error +# incomplete +# +# Other (TODO) +# with_static_parameters + +# IR: +# +# Exceptions: +# enter +# leave +# pop_exception +# gotoifnot +# +# SSAIR: +# throw_undef_if_not +# unreachable +# undefcheck +# invoke + From 0cc233a3b46e2aa596a0b49e180794060efdd05f Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 25 Jun 2019 18:21:59 +1000 Subject: [PATCH 27/56] Fix @ Expr macro --- test/compiler/lowering.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 4703267481574..00828938cd5ba 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -106,8 +106,8 @@ function lower_ref_expr!(ex) return Expr(:unnecessary, ex.args[2:end]...) elseif ex.head == :macrocall && ex.args[1] == Symbol("@Expr") head = ex.args[3] - head = head isa QuoteNode ? head.value : Symbol(head) - return Expr(head, ex.args[4:end]...) + head isa QuoteNode || throw(ArgumentError("`head` argument to @Expr should be quoted")) + return Expr(head.value, ex.args[4:end]...) end end return ex @@ -162,9 +162,9 @@ end # # Note that this is provided for convenience/reference but in practice we # expand it "manually" inside lower_ref_expr! -macro Ex(head, args...) - head = head isa QuoteNode ? head.value : Symbol(head) - esc(Expr(head, args...)) +macro Expr(head, args...) + head isa QuoteNode || throw(ArgumentError("`head` argument to @Expr should be quoted")) + esc(Expr(head.value, args...)) end """ From 34a44b94a49a77304bceea9132593ed3b7a2be06 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 26 Jun 2019 17:30:39 +1000 Subject: [PATCH 28/56] A few more tests for expand-let --- test/compiler/lowering.jl | 41 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 00828938cd5ba..24b399eaf3b7e 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -178,6 +178,16 @@ macro desugar(ex, kws...) end end +macro maketest(ex) + Base.remove_linenums!(ex) + quote + print("@test_desugar(") + print($(QuoteNode(ex)), ",\n") + print(desugar($(Expr(:quote, ex))), "\n") + print(")") + end +end + """ Test that syntax desugaring of `input` produces an expression equivalent to the reference expression `ref`. @@ -643,6 +653,17 @@ end @testset "let blocks" begin # flisp: (expand-let) + @test_desugar(let x::Int + body + end, + @Expr(:scope_block, begin + begin + local x + @Expr(:decl, x, Int) + end + body + end) + ) @test_desugar(let x,y body end, @@ -672,7 +693,25 @@ end end) end) ) - # TODO: More coverage. Internals look complex. + + @test_desugar(let f(x) = 1 + body + end, + @Expr(:scope_block, + begin + @Expr(:local_def, f) + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), @Expr(:scope_block, 1))) + end + body + end) + ) + + # Other things in the variable list should produce an error + @test_desugar_error let f(x); body end "invalid let syntax" end @testset "Loops" begin From 3533d159bb8c54c034b0400cf373a953e2ebfd08 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 26 Jun 2019 18:32:32 +1000 Subject: [PATCH 29/56] Test set macro specifically for desugaring testsets This allows for a much lighter weight way of writing a big list of test cases as a flat list, cutting down on a lot of repetition and improving the indentation. The grouping is perhaps a little less clear but that seems like a reasonable price to pay for improved formatting. --- test/compiler/lowering.jl | 1413 ++++++++++++++++++++----------------- 1 file changed, 770 insertions(+), 643 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 24b399eaf3b7e..b127a66043095 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -178,36 +178,84 @@ macro desugar(ex, kws...) end end +# Convenience macro to print code for a test case macro maketest(ex) Base.remove_linenums!(ex) quote - print("@test_desugar(") - print($(QuoteNode(ex)), ",\n") + print($(QuoteNode(ex)), "\n") print(desugar($(Expr(:quote, ex))), "\n") - print(")") end end """ -Test that syntax desugaring of `input` produces an expression equivalent to the -reference expression `ref`. + @testset_desugar(name, exprs) + +Test that a set of expressions lower correctly to desugared AST form. This +creates a new `@testset` with the given `name`. The statements in the block +`exprs` are interpreted as a flat list of any number of `input_expr`, +`ref_expr` pairs, where `input_expr` should be transformed by lowering into +`ref_expr` by the desugaring pass. For example, + +``` +@testset_desugar "Property notation" begin + # flisp: (expand-fuse-broadcast) + a.b + Top.getproperty(a, :b) + + a.b.c + Top.getproperty(Top.getproperty(a, :b), :c) +end +``` """ -macro test_desugar(input, ref) - ex = quote - input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) - ref = lower_ref_expr($(Expr(:quote, ref))) - @test input == ref - if input != ref - # Kinda crude. Would be much neater if Test supported custom/more - # capable diffing for failed tests. - println("Diff dump:") - diffdump(input, ref) +macro testset_desugar(name, block) + if !(block isa Expr && block.head == :block) + throw(ArgumentError("@testset_desugar requires a block as the second argument")) + end + loc = nothing + tests = [] + i = 1 + while i <= length(block.args) + if block.args[i] isa LineNumberNode + loc = block.args[i] + i += 1 + continue + end + exs = [] + while i <= length(block.args) && length(exs) < 2 + if !(block.args[i] isa LineNumberNode) + push!(exs, block.args[i]) + end + i += 1 + end + if length(exs) == 0 + break + end + if length(exs) == 1 + throw(ArgumentError("List of expressions to @testset_desugar must consist of input,ref pairs")) + end + input = exs[1] + ref = exs[2] + ex = quote + input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) + ref = lower_ref_expr($(Expr(:quote, ref))) + @test input == ref + if input != ref + # Kinda crude. Would be much neater if Test supported custom/more + # capable diffing for failed tests. + println("Diff dump:") + diffdump(input, ref) + end + end + # Attribute the test to the correct line number + @assert ex.args[6].args[1] == Symbol("@test") + ex.args[6].args[2] = loc + push!(tests, ex) + end + quote + @testset $name begin + $(tests...) end end - # Attribute the test to the correct line number - @assert ex.args[6].args[1] == Symbol("@test") - ex.args[6].args[2] = __source__ - ex end macro test_desugar_error(input, msg) @@ -224,668 +272,744 @@ end #------------------------------------------------------------------------------- # Tests -@testset "Property notation" begin +@testset_desugar "Property notation" begin # flisp: (expand-fuse-broadcast) - @test_desugar a.b Top.getproperty(a, :b) - @test_desugar a.b.c Top.getproperty(Top.getproperty(a, :b), :c) + a.b + Top.getproperty(a, :b) - @test_desugar(a.b = c, - begin - Top.setproperty!(a, :b, c) - maybe_unused(c) - end - ) - @test_desugar(a.b.c = d, - begin - ssa1 = Top.getproperty(a, :b) - Top.setproperty!(ssa1, :c, d) - maybe_unused(d) - end - ) -end + a.b.c + Top.getproperty(Top.getproperty(a, :b), :c) -@testset "Index notation" begin - # flisp: (process-indices) (partially-expand-ref) - @testset "getindex" begin - # Indexing - @test_desugar a[i] Top.getindex(a, i) - @test_desugar a[i,j] Top.getindex(a, i, j) - # Indexing with `end` - @test_desugar a[end] Top.getindex(a, Top.lastindex(a)) - @test_desugar a[i,end] Top.getindex(a, i, Top.lastindex(a,2)) - # Nesting of `end` - @test_desugar a[[end]] Top.getindex(a, Top.vect(Top.lastindex(a))) - @test_desugar a[b[end] + end] Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) - @test_desugar a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) - @test_desugar a[end][b[i]] Top.getindex(Top.getindex(a, Top.lastindex(a)), Top.getindex(b,i)) - - # Interaction of `end` with splatting - @test_desugar(a[I..., end, J..., end], - Core._apply(Top.getindex, Core.tuple(a), - I, - Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(I)))), - J, - Core.tuple(Top.lastindex(a, Top.:+(2, Top.length(J), Top.length(I))))) - ) - @test_desugar(a[f(x)..., end], - begin - ssa1 = f(x) - Core._apply(Top.getindex, Core.tuple(a), - ssa1, - Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(ssa1))))) - end - ) + a.b=c + begin + Top.setproperty!(a, :b, c) + maybe_unused(c) end - @testset "setindex!" begin - # flisp: (lambda in expand-table) - @test_desugar(a[i] = b, - begin - Top.setindex!(a, b, i) - maybe_unused(b) - end - ) - @test_desugar(a[i,end] = b+c, - begin - ssa1 = b+c - Top.setindex!(a, ssa1, i, Top.lastindex(a,2)) - maybe_unused(ssa1) - end - ) + a.b.c=d + begin + ssa1 = Top.getproperty(a, :b) + Top.setproperty!(ssa1, :c, d) + maybe_unused(d) end end -@testset "Array notation" begin - # flisp: (in expand-table) - @testset "Literals" begin - @test_desugar [a,b] Top.vect(a,b) - @test_desugar T[a,b] Top.getindex(T, a,b) # Only so much syntax to go round :-/ - @test_desugar_error [a,b;c] "unexpected semicolon in array expression" - @test_desugar_error [a=b,c] "misplaced assignment statement in \"[a = b, c]\"" +@testset_desugar "Index notation; getindex" begin + # Indexing + a[i] + Top.getindex(a, i) + + a[i,j] + Top.getindex(a, i, j) + + # Indexing with `end` + a[end] + Top.getindex(a, Top.lastindex(a)) + + a[i,end] + Top.getindex(a, i, Top.lastindex(a,2)) + + # Nesting of `end` + a[[end]] + Top.getindex(a, Top.vect(Top.lastindex(a))) + + a[b[end] + end] + Top.getindex(a, Top.getindex(b, Top.lastindex(b)) + Top.lastindex(a)) + + a[f(end) + 1] + Top.getindex(a, f(Top.lastindex(a)) + 1) + + a[end][b[i]] + Top.getindex(Top.getindex(a, Top.lastindex(a)), Top.getindex(b,i)) + + # Interaction of `end` with splatting + a[I..., end, J..., end] + Core._apply(Top.getindex, Core.tuple(a), + I, + Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(I)))), + J, + Core.tuple(Top.lastindex(a, Top.:+(2, Top.length(J), Top.length(I))))) + + a[f(x)..., end] + begin + ssa1 = f(x) + Core._apply(Top.getindex, Core.tuple(a), + ssa1, + Core.tuple(Top.lastindex(a, Top.:+(1, Top.length(ssa1))))) + end +end + +@testset_desugar "Index notation; setindex!" begin + # flisp: (lambda in expand-table) + a[i] = b + begin + Top.setindex!(a, b, i) + maybe_unused(b) end - @testset "Concatenation" begin - # flisp: (lambda in expand-table) - @test_desugar [a b] Top.hcat(a,b) - @test_desugar [a; b] Top.vcat(a,b) - @test_desugar T[a b] Top.typed_hcat(T, a,b) - @test_desugar T[a; b] Top.typed_vcat(T, a,b) - @test_desugar [a b; c] Top.hvcat(Core.tuple(2,1), a, b, c) - @test_desugar T[a b; c] Top.typed_hvcat(T, Core.tuple(2,1), a, b, c) - - @test_desugar_error [a b=c] "misplaced assignment statement in \"[a b = c]\"" - @test_desugar_error [a; b=c] "misplaced assignment statement in \"[a; b = c]\"" - @test_desugar_error T[a b=c] "misplaced assignment statement in \"T[a b = c]\"" - @test_desugar_error T[a; b=c] "misplaced assignment statement in \"T[a; b = c]\"" + a[i,end] = b+c + begin + ssa1 = b+c + Top.setindex!(a, ssa1, i, Top.lastindex(a,2)) + maybe_unused(ssa1) end end -@testset "Tuples" begin - @test_desugar (x,y) Core.tuple(x,y) - @test_desugar (x=a,y=b) Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) +@testset_desugar "Array Literals" begin + # flisp: (in expand-table) + [a,b] + Top.vect(a,b) + + T[a,b] + Top.getindex(T, a,b) # Only so much syntax to go round :-/ + + [a,b;c] + @Expr(:error, "unexpected semicolon in array expression") + + [a=b,c] + @Expr(:error, "misplaced assignment statement in \"[a = b, c]\"") +end + +@testset_desugar "Array Concatenation" begin + # flisp: (lambda in expand-table) + [a b] + Top.hcat(a,b) + + [a; b] + Top.vcat(a,b) + + T[a b] + Top.typed_hcat(T, a,b) + + T[a; b] + Top.typed_vcat(T, a,b) + + [a b; c] + Top.hvcat(Core.tuple(2,1), a, b, c) + + T[a b; c] + Top.typed_hvcat(T, Core.tuple(2,1), a, b, c) + + [a b=c] + @Expr(:error, "misplaced assignment statement in \"[a b = c]\"") + + [a; b=c] + @Expr(:error, "misplaced assignment statement in \"[a; b = c]\"") + + T[a b=c] + @Expr(:error, "misplaced assignment statement in \"T[a b = c]\"") + + T[a; b=c] + @Expr(:error, "misplaced assignment statement in \"T[a; b = c]\"") end -@testset "Splatting" begin - @test_desugar f(i,j,v...,k) Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) +@testset_desugar "Tuples" begin + (x,y) + Core.tuple(x,y) - @test_desugar_error x... "\"...\" expression outside call" + (x=a,y=b) + Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) end -@testset "Comparison chains" begin +@testset_desugar "Splatting" begin + f(i,j,v...,k) + Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) + + x... + @Expr(:error, "\"...\" expression outside call") +end + +@testset_desugar "Comparison chains" begin # flisp: (expand-compare-chain) - @test_desugar(a < b < c, - if a < b - b < c - else - false - end - ) + a < b < c + if a < b + b < c + else + false + end + # Nested - @test_desugar(a < b > d <= e, - if a < b - if b > d - d <= e - else - false - end + a < b > d <= e + if a < b + if b > d + d <= e else false end - ) + else + false + end + # Subexpressions - @test_desugar(a < b+c < d, - if (ssa1 = b+c; a < ssa1) - ssa1 < d - else - false - end - ) + a < b+c < d + if (ssa1 = b+c; a < ssa1) + ssa1 < d + else + false + end # Interaction with broadcast syntax - @test_desugar(a < b .< c, - Top.materialize(Top.broadcasted(&, a < b, Top.broadcasted(<, b, c))) - ) - @test_desugar(a .< b+c < d, - Top.materialize(Top.broadcasted(&, - begin - ssa1 = b+c - # Is this a bug? - Top.materialize(Top.broadcasted(<, a, ssa1)) - end, - ssa1 < d)) - ) - @test_desugar(a < b+c .< d, - Top.materialize(Top.broadcasted(&, - begin - ssa1 = b+c - a < ssa1 - end, - Top.broadcasted(<, ssa1, d))) - ) + a < b .< c + Top.materialize(Top.broadcasted(&, a < b, Top.broadcasted(<, b, c))) + + a .< b+c < d + Top.materialize(Top.broadcasted(&, + begin + ssa1 = b+c + # Is this a bug? + Top.materialize(Top.broadcasted(<, a, ssa1)) + end, + ssa1 < d)) + + a < b+c .< d + Top.materialize(Top.broadcasted(&, + begin + ssa1 = b+c + a < ssa1 + end, + Top.broadcasted(<, ssa1, d))) end -@testset "Short circuit; ternary" begin +@testset_desugar "Short circuit; ternary" begin # flisp: (expand-or) (expand-and) - @test_desugar a || b if a; a else b end - @test_desugar a && b if a; b else false end - @test_desugar a ? x : y if a; x else y end + a || b + if a + a + else + b + end + + a && b + if a + b + else + false + end + + a ? x : y + if a + x + else + y + end end -@testset "Misc operators" begin - @test_desugar a' Top.adjoint(a) +@testset_desugar "Misc operators" begin + a' + Top.adjoint(a) + # <: and >: are special Expr heads which need to be turned into Expr(:call) # when used as operators - @test_desugar a <: b $(Expr(:call, :(<:), :a, :b)) - @test_desugar a >: b $(Expr(:call, :(>:), :a, :b)) + a <: b + $(Expr(:call, :(<:), :a, :b)) + + a >: b + $(Expr(:call, :(>:), :a, :b)) - @test_desugar_error $(Expr(:$, :x)) "\"\$\" expression outside quote" + $(Expr(:$, :x)) + @Expr(:error, "\"\$\" expression outside quote") end -@testset "Broadcast" begin +@testset_desugar "Broadcast" begin # flisp: (expand-fuse-broadcast) # Basic - @test_desugar x .+ y Top.materialize(Top.broadcasted(+, x, y)) - @test_desugar f.(x) Top.materialize(Top.broadcasted(f, x)) + x .+ y + Top.materialize(Top.broadcasted(+, x, y)) + + f.(x) + Top.materialize(Top.broadcasted(f, x)) + # Fusing - @test_desugar f.(x) .+ g.(y) Top.materialize(Top.broadcasted(+, Top.broadcasted(f, x), - Top.broadcasted(g, y))) + f.(x) .+ g.(y) + Top.materialize(Top.broadcasted(+, Top.broadcasted(f, x), Top.broadcasted(g, y))) + # Keywords don't participate - @test_desugar(f.(x, a=1), - Top.materialize( - begin - ssa1 = Top.broadcasted_kwsyntax - ssa2 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) - Core.kwfunc(ssa1)(ssa2, ssa1, f, x) - end - ) + f.(x, a=1) + Top.materialize( + begin + ssa1 = Top.broadcasted_kwsyntax + ssa2 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) + Core.kwfunc(ssa1)(ssa2, ssa1, f, x) + end ) + # Nesting - @test_desugar f.(g(x)) Top.materialize(Top.broadcasted(f, g(x))) - @test_desugar f.(g(h.(x))) Top.materialize(Top.broadcasted(f, - g(Top.materialize(Top.broadcasted(h, x))))) + f.(g(x)) + Top.materialize(Top.broadcasted(f, g(x))) + + f.(g(h.(x))) + Top.materialize(Top.broadcasted(f, g(Top.materialize(Top.broadcasted(h, x))))) # In place - @test_desugar x .= a Top.materialize!(x, Top.broadcasted(Top.identity, a)) - @test_desugar x .= f.(a) Top.materialize!(x, Top.broadcasted(f, a)) - @test_desugar x .+= a Top.materialize!(x, Top.broadcasted(+, x, a)) + x .= a + Top.materialize!(x, Top.broadcasted(Top.identity, a)) + + x .= f.(a) + Top.materialize!(x, Top.broadcasted(f, a)) + + x .+= a + Top.materialize!(x, Top.broadcasted(+, x, a)) end -@testset "Call with keyword arguments" begin - @test_desugar( - f(x,a=1), - begin - ssa1 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) - Core.kwfunc(f)(ssa1, f, x) - end - ) +@testset_desugar "Call with keyword arguments" begin + f(x,a=1) + begin + ssa1 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) + Core.kwfunc(f)(ssa1, f, x) + end end -@testset "In place update operators" begin +@testset_desugar "In place update operators" begin # flisp: (lower-update-op) - @test_desugar x += a x = x+a - @test_desugar x::Int += a x = x::Int + a - @test_desugar(x[end] += a, - begin - ssa1 = Top.lastindex(x) - begin - ssa2 = Top.getindex(x, ssa1) + a - Top.setindex!(x, ssa2, ssa1) - maybe_unused(ssa2) - end - end - ) - @test_desugar(x[f(y)] += a, - begin - ssa1 = f(y) - begin - ssa2 = Top.getindex(x, ssa1) + a - Top.setindex!(x, ssa2, ssa1) - maybe_unused(ssa2) - end - end - ) - @test_desugar((x,y) .+= a, + x += a + x = x+a + + x::Int += a + x = x::Int + a + + x[end] += a + begin + ssa1 = Top.lastindex(x) begin - ssa1 = Core.tuple(x, y) - Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) + ssa2 = Top.getindex(x, ssa1) + a + Top.setindex!(x, ssa2, ssa1) + maybe_unused(ssa2) end - ) - @test_desugar([x y] .+= a, + end + + x[f(y)] += a + begin + ssa1 = f(y) begin - ssa1 = Top.hcat(x, y) - Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) + ssa2 = Top.getindex(x, ssa1) + a + Top.setindex!(x, ssa2, ssa1) + maybe_unused(ssa2) end - ) - @test_desugar_error (x+y) += 1 "invalid assignment location \"(x + y)\"" + end + + (x,y) .+= a + begin + ssa1 = Core.tuple(x, y) + Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) + end + + [x y] .+= a + begin + ssa1 = Top.hcat(x, y) + Top.materialize!(ssa1, Top.broadcasted(+, ssa1, a)) + end + + (x+y) += 1 + @Expr(:error, "invalid assignment location \"(x + y)\"") end -@testset "Assignment" begin +@testset_desugar "Assignment" begin # flisp: (lambda in expand-table) # Assignment chain; nontrivial rhs - @test_desugar(x = y = f(a), + x = y = f(a) + begin + ssa1 = f(a) + y = ssa1 + x = ssa1 + maybe_unused(ssa1) + end + + # Multiple Assignment + + # Simple multiple assignment exact match + (x,y) = (a,b) + begin + x = a + y = b + maybe_unused(Core.tuple(a,b)) + end + + # Destructuring + (x,y) = a + begin begin - ssa1 = f(a) - y = ssa1 - x = ssa1 - maybe_unused(ssa1) + ssa1 = Top.indexed_iterate(a, 1) + x = Core.getfield(ssa1, 1) + gsym1 = Core.getfield(ssa1, 2) + ssa1 end - ) + begin + ssa2 = Top.indexed_iterate(a, 2, gsym1) + y = Core.getfield(ssa2, 1) + ssa2 + end + maybe_unused(a) + end - @testset "Multiple Assignemnt" begin - # Simple multiple assignment exact match - @test_desugar((x,y) = (a,b), - begin - x = a - y = b - maybe_unused(Core.tuple(a,b)) - end - ) - # Destructuring - @test_desugar((x,y) = a, - begin - begin - ssa1 = Top.indexed_iterate(a, 1) - x = Core.getfield(ssa1, 1) - gsym1 = Core.getfield(ssa1, 2) - ssa1 - end - begin - ssa2 = Top.indexed_iterate(a, 2, gsym1) - y = Core.getfield(ssa2, 1) - ssa2 - end - maybe_unused(a) - end - ) - # Nested destructuring - @test_desugar((x,(y,z)) = a, + # Nested destructuring + (x,(y,z)) = a + begin + begin + ssa1 = Top.indexed_iterate(a, 1) + x = Core.getfield(ssa1, 1) + gsym1 = Core.getfield(ssa1, 2) + ssa1 + end + begin + ssa2 = Top.indexed_iterate(a, 2, gsym1) begin + ssa3 = Core.getfield(ssa2, 1) begin - ssa1 = Top.indexed_iterate(a, 1) - x = Core.getfield(ssa1, 1) - gsym1 = Core.getfield(ssa1, 2) - ssa1 + ssa4 = Top.indexed_iterate(ssa3, 1) + y = Core.getfield(ssa4, 1) + gsym2 = Core.getfield(ssa4, 2) + ssa4 end begin - ssa2 = Top.indexed_iterate(a, 2, gsym1) - begin - ssa3 = Core.getfield(ssa2, 1) - begin - ssa4 = Top.indexed_iterate(ssa3, 1) - y = Core.getfield(ssa4, 1) - gsym2 = Core.getfield(ssa4, 2) - ssa4 - end - begin - ssa5 = Top.indexed_iterate(ssa3, 2, gsym2) - z = Core.getfield(ssa5, 1) - ssa5 - end - maybe_unused(ssa3) - end - ssa2 + ssa5 = Top.indexed_iterate(ssa3, 2, gsym2) + z = Core.getfield(ssa5, 1) + ssa5 end - maybe_unused(a) + maybe_unused(ssa3) end - ) + ssa2 + end + maybe_unused(a) end # Invalid assignments - @test_desugar_error 1=a "invalid assignment location \"1\"" - @test_desugar_error true=a "invalid assignment location \"true\"" - @test_desugar_error "str"=a "invalid assignment location \"\"str\"\"" - @test_desugar_error [x y]=c "invalid assignment location \"[x y]\"" - @test_desugar_error a[x y]=c "invalid spacing in left side of indexed assignment" - @test_desugar_error a[x;y]=c "unexpected \";\" in left side of indexed assignment" - @test_desugar_error [x;y]=c "use \"(a, b) = ...\" to assign multiple values" + 1 = a + @Expr(:error, "invalid assignment location \"1\"") + + true = a + @Expr(:error, "invalid assignment location \"true\"") + + "str" = a + @Expr(:error, "invalid assignment location \"\"str\"\"") + + [x y] = c + @Expr(:error, "invalid assignment location \"[x y]\"") + + a[x y] = c + @Expr(:error, "invalid spacing in left side of indexed assignment") + + a[x;y] = c + @Expr(:error, "unexpected \";\" in left side of indexed assignment") + + [x;y] = c + @Expr(:error, "use \"(a, b) = ...\" to assign multiple values") # Old deprecation (6575e12ba46) - @test_desugar_error x.(y)=c "invalid syntax \"x.(y) = ...\"" + x.(y)=c + @Expr(:error, "invalid syntax \"x.(y) = ...\"") end -@testset "Declarations" begin +@testset_desugar "Declarations" begin # flisp: (expand-decls) (expand-local-or-global-decl) (expand-const-decl) # const - @test_desugar((const x=a), + const x=a + begin + @Expr :const x # `const x` is invalid surface syntax + x = a + end + + const x,y = a,b + begin + @Expr :const x + @Expr :const y begin - @Expr :const x # `const x` is invalid surface syntax x = a + y = b + maybe_unused(Core.tuple(a,b)) end - ) - @test_desugar((const x,y = a,b), - begin - @Expr :const x - @Expr :const y - begin - x = a - y = b - maybe_unused(Core.tuple(a,b)) - end - end - ) + end # local - @test_desugar((local x, y), - begin - local y - local x - end - ) + local x, y + begin + local y + local x + end + # Locals with initialization. Note parentheses are needed for this to parse # as individual assignments rather than multiple assignment. - @test_desugar((local (x=a), (y=b), z), + local (x=a), (y=b), z + begin + local z + local y + local x + x = a + y = b + end + + # Multiple assignment form + begin + local x,y = a,b + end + begin + local x + local y begin - local z - local y - local x x = a y = b + maybe_unused(Core.tuple(a,b)) end - ) - # Multiple assignment form - @test_desugar(begin - local x,y = a,b - end, - begin - local x - local y - begin - x = a - y = b - maybe_unused(Core.tuple(a,b)) - end - end - ) + end # global - @test_desugar((global x, (y=a)), - begin - global y - global x - y = a - end - ) + global x, (y=a) + begin + global y + global x + y = a + end # type decl - @test_desugar(x::T = a, - begin - @Expr :decl x T - x = a - end - ) + x::T = a + begin + @Expr :decl x T + x = a + end # type aliases - @test_desugar(A{T} = B{T}, - begin - @Expr :const_if_global A - A = @Expr(:scope_block, - begin - @Expr :local_def T - T = Core.TypeVar(:T) - Core.UnionAll(T, Core.apply_type(B, T)) - end) - end - ) - + A{T} = B{T} + begin + @Expr :const_if_global A + A = @Expr(:scope_block, + begin + @Expr :local_def T + T = Core.TypeVar(:T) + Core.UnionAll(T, Core.apply_type(B, T)) + end) + end end -@testset "let blocks" begin +@testset_desugar "let blocks" begin # flisp: (expand-let) - @test_desugar(let x::Int - body - end, - @Expr(:scope_block, begin - begin - local x - @Expr(:decl, x, Int) - end - body - end) - ) - @test_desugar(let x,y - body - end, - @Expr(:scope_block, + let x::Int + body + end + @Expr(:scope_block, begin begin local x - @Expr(:scope_block, - begin - local y - body - end) - end) - ) + @Expr(:decl, x, Int) + end + body + end) + + # Let without assignment + let x,y + body + end + @Expr(:scope_block, + begin + local x + @Expr(:scope_block, + begin + local y + body + end) + end) + # Let with assignment - @test_desugar(let x=a,y=b - body - end, - @Expr(:scope_block, + let x=a,y=b + body + end + @Expr(:scope_block, + begin + @Expr :local_def x + x = a + @Expr(:scope_block, + begin + @Expr :local_def y + y = b + body + end) + end) + + # Let with function declaration + let f(x) = 1 + body + end + @Expr(:scope_block, + begin + @Expr(:local_def, f) begin - @Expr :local_def x - x = a - @Expr(:scope_block, - begin - @Expr :local_def y - y = b - body - end) - end) - ) - - @test_desugar(let f(x) = 1 - body - end, - @Expr(:scope_block, - begin - @Expr(:local_def, f) - begin - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), @Expr(:scope_block, 1))) - end - body - end) - ) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), @Expr(:scope_block, 1))) + end + body + end) # Other things in the variable list should produce an error - @test_desugar_error let f(x); body end "invalid let syntax" + let f(x) + body + end + @Expr(:error, "invalid let syntax") end -@testset "Loops" begin +@testset_desugar "Loops" begin # flisp: (expand-for) (lambda in expand-forms) - @test_desugar( - while cond - body1 - continue - body2 - break - body3 - end, - @Expr(:break_block, loop_exit, - @Expr(:_while, cond, - @Expr(:break_block, loop_cont, - @Expr(:scope_block, begin - body1 - @Expr :break loop_cont - body2 - @Expr :break loop_exit - body3 - end)))) - ) - - @test_desugar( - for i = a - body1 - continue - body2 - break - end, - @Expr(:break_block, loop_exit, - begin - ssa1 = a - gsym1 = Top.iterate(ssa1) - if Top.not_int(Core.:(===)(gsym1, $nothing)) - @Expr(:_do_while, - begin - @Expr(:break_block, loop_cont, - @Expr(:scope_block, + while cond + body1 + continue + body2 + break + body3 + end + @Expr(:break_block, loop_exit, + @Expr(:_while, cond, + @Expr(:break_block, loop_cont, + @Expr(:scope_block, begin + body1 + @Expr :break loop_cont + body2 + @Expr :break loop_exit + body3 + end)))) + + for i = a + body1 + continue + body2 + break + end + @Expr(:break_block, loop_exit, + begin + ssa1 = a + gsym1 = Top.iterate(ssa1) + if Top.not_int(Core.:(===)(gsym1, $nothing)) + @Expr(:_do_while, + begin + @Expr(:break_block, loop_cont, + @Expr(:scope_block, + begin + local i begin - local i - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - begin - body1 - @Expr :break loop_cont - body2 - @Expr :break loop_exit - end - end)) - gsym1 = Top.iterate(ssa1, ssa3) - end, - Top.not_int(Core.:(===)(gsym1, $nothing))) - end - end) - ) + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + begin + body1 + @Expr :break loop_cont + body2 + @Expr :break loop_exit + end + end)) + gsym1 = Top.iterate(ssa1, ssa3) + end, + Top.not_int(Core.:(===)(gsym1, $nothing))) + end + end) # For loops with `outer` - @test_desugar(for outer i = a - body - end, - @Expr(:break_block, loop_exit, - begin - ssa1 = a - gsym1 = Top.iterate(ssa1) - @Expr(:require_existing_local, i) - if Top.not_int(Core.:(===)(gsym1, $nothing)) - @Expr(:_do_while, - begin - @Expr(:break_block, loop_cont, - @Expr(:scope_block, + for outer i = a + body + end + @Expr(:break_block, loop_exit, + begin + ssa1 = a + gsym1 = Top.iterate(ssa1) + @Expr(:require_existing_local, i) + if Top.not_int(Core.:(===)(gsym1, $nothing)) + @Expr(:_do_while, + begin + @Expr(:break_block, loop_cont, + @Expr(:scope_block, + begin begin - begin - ssa2 = gsym1 - i = Core.getfield(ssa2, 1) - ssa3 = Core.getfield(ssa2, 2) - ssa2 - end - body - end)) - gsym1 = Top.iterate(ssa1, ssa3) - end, - Top.not_int(Core.:(===)(gsym1, $nothing))) - end - end) - ) + ssa2 = gsym1 + i = Core.getfield(ssa2, 1) + ssa3 = Core.getfield(ssa2, 2) + ssa2 + end + body + end)) + gsym1 = Top.iterate(ssa1, ssa3) + end, + Top.not_int(Core.:(===)(gsym1, $nothing))) + end + end) end -@testset "Functions" begin +@testset_desugar "Functions" begin # Short form - @test_desugar(f(x) = body(x), + f(x) = body(x) + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, + body(x)))) + maybe_unused(f) + end + + # Long form with argument annotations + function f(x::T, y) + body(x) + end + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:scope_block, + body(x)))) + maybe_unused(f) + end + + # Default arguments + function f(x=a, y=b) + body(x,y) + end + begin begin @Expr(:method, f) @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), - @Expr(:scope_block, - body(x)))) + Core.svec(Core.svec(Core.Typeof(f)), Core.svec()), + @Expr(:lambda, $([:_self_]), $([]), + @Expr(:scope_block, _self_(a, b)))) maybe_unused(f) end - ) - - # Long form with argument annotations - @test_desugar(function f(x::T, y) - body(x) - end, begin @Expr(:method, f) @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :y]), $([]), - @Expr(:scope_block, - body(x)))) + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, _self_(x, b)))) maybe_unused(f) end - ) - - # Default arguments - @test_desugar(function f(x=a, y=b) - body(x,y) - end, - begin - begin - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f)), Core.svec()), - @Expr(:lambda, $([:_self_]), $([]), - @Expr(:scope_block, _self_(a, b)))) - maybe_unused(f) - end - begin - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), - @Expr(:scope_block, _self_(x, b)))) - maybe_unused(f) - end - begin - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :y]), $([]), - @Expr(:scope_block, body(x, y)))) - maybe_unused(f) - end - end - ) - - # Varargs - @test_desugar(function f(x, args...) - body(x, args) - end, begin @Expr(:method, f) @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any, - Core.apply_type(Vararg, Core.Any)), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :args]), $([]), - @Expr(:scope_block, body(x, args)))) + Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:scope_block, body(x, y)))) maybe_unused(f) end - ) + end + + # Varargs + function f(x, args...) + body(x, args) + end + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any, + Core.apply_type(Vararg, Core.Any)), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :args]), $([]), + @Expr(:scope_block, body(x, args)))) + maybe_unused(f) + end # Keyword arguments - @test_desugar(function f(x; k1=v1, k2=v2) - body - end, - Core.ifelse(false, false, + function f(x; k1=v1, k2=v2) + body + end + Core.ifelse(false, false, begin @Expr(:method, f) begin @@ -940,112 +1064,115 @@ end end f end) - ) # Return type declaration - @test_desugar(function f(x)::T - body(x) - end, - begin - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), - @Expr(:scope_block, - begin - ssa1 = T - @Expr(:meta, ret_type, ssa1) - body(x) - end))) - maybe_unused(f) - end - ) + function f(x)::T + body(x) + end + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, + begin + ssa1 = T + @Expr(:meta, ret_type, ssa1) + body(x) + end))) + maybe_unused(f) + end # Anon functions - @test_desugar((x,y)->body(x,y), + (x,y)->body(x,y) + begin + local gsym1 begin - local gsym1 - begin - @Expr(:method, gsym1) - @Expr(:method, gsym1, - Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :y]), $([]), - @Expr(:scope_block, body(x, y)))) - maybe_unused(gsym1) - end + @Expr(:method, gsym1) + @Expr(:method, gsym1, + Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:scope_block, body(x, y)))) + maybe_unused(gsym1) end - ) + end # Invalid names - @test_desugar_error ccall(x)=body "invalid function name \"ccall\"" - @test_desugar_error cglobal(x)=body "invalid function name \"cglobal\"" - @test_desugar_error true(x)=body "invalid function name \"true\"" - @test_desugar_error false(x)=body "invalid function name \"false\"" + ccall(x)=body + @Expr(:error, "invalid function name \"ccall\"") + + cglobal(x)=body + @Expr(:error, "invalid function name \"cglobal\"") + + true(x)=body + @Expr(:error, "invalid function name \"true\"") + + false(x)=body + @Expr(:error, "invalid function name \"false\"") end -@testset "Generated functions" begin - ln = LineNumberNode(@__LINE__()+2, Symbol(@__FILE__)) - @test_desugar(function f(x) - body1(x) - if $(Expr(:generated)) - gen_body1(x) - else - normal_body1(x) - end - body2 - if $(Expr(:generated)) - gen_body2 - else - normal_body2 - end - end, +ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) +@testset_desugar "Generated function; optionally generated" begin + function f(x) + body1(x) + if $(Expr(:generated)) + gen_body1(x) + else + normal_body1(x) + end + body2 + if $(Expr(:generated)) + gen_body2 + else + normal_body2 + end + end + begin begin + global gsym1 begin - global gsym1 - begin - @Expr(:method, gsym1) - @Expr(:method, gsym1, - Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :gsym2, :x]), $([]), - @Expr(:scope_block, - begin - @Expr(:meta, nospecialize, gsym2, x) - Core._expr(:block, - $(QuoteNode(LineNumberNode(ln.line, ln.file))), - @Expr(:copyast, $(QuoteNode(:(body1(x))))), - # FIXME: These line numbers seem buggy? - $(QuoteNode(LineNumberNode(ln.line+1, ln.file))), - gen_body1(x), - $(QuoteNode(LineNumberNode(ln.line+6, ln.file))), - :body2, - $(QuoteNode(LineNumberNode(ln.line+7, ln.file))), - gen_body2) - end))) - maybe_unused(gsym1) - end + @Expr(:method, gsym1) + @Expr(:method, gsym1, + Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :gsym2, :x]), $([]), + @Expr(:scope_block, + begin + @Expr(:meta, nospecialize, gsym2, x) + Core._expr(:block, + $(QuoteNode(LineNumberNode(ln.line, ln.file))), + @Expr(:copyast, $(QuoteNode(:(body1(x))))), + # FIXME: These line numbers seem buggy? + $(QuoteNode(LineNumberNode(ln.line+1, ln.file))), + gen_body1(x), + $(QuoteNode(LineNumberNode(ln.line+6, ln.file))), + :body2, + $(QuoteNode(LineNumberNode(ln.line+7, ln.file))), + gen_body2) + end))) + maybe_unused(gsym1) end - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), - @Expr(:scope_block, - begin - @Expr(:meta, generated, - @Expr(:new, - Core.GeneratedFunctionStub, - gsym1, $([:_self_, :x]), - nothing, - $(ln.line), - $(QuoteNode(ln.file)), - false)) - body1(x) - normal_body1(x) - body2 - normal_body2 - end))) - maybe_unused(f) end - ) + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:scope_block, + begin + @Expr(:meta, generated, + @Expr(:new, + Core.GeneratedFunctionStub, + gsym1, $([:_self_, :x]), + nothing, + $(ln.line), + $(QuoteNode(ln.file)), + false)) + body1(x) + normal_body1(x) + body2 + normal_body2 + end))) + maybe_unused(f) + end end @testset "Forms without desugaring" begin From 29c91276d2a632890014fb6fe16c7ad437702614 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 08:07:13 +1000 Subject: [PATCH 30/56] Finish testing lowering of `let` --- test/compiler/lowering.jl | 91 +++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 13 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index b127a66043095..eb16cdcd660c4 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -258,17 +258,6 @@ macro testset_desugar(name, block) end end -macro test_desugar_error(input, msg) - ex = quote - input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) - @test input == Expr(:error, $msg) - end - # Attribute the test to the correct line number - @assert ex.args[4].args[1] == Symbol("@test") - ex.args[4].args[2] = __source__ - ex -end - #------------------------------------------------------------------------------- # Tests @@ -809,7 +798,7 @@ end end) # Let with assignment - let x=a,y=b + let x=a, y=b body end @Expr(:scope_block, @@ -840,11 +829,87 @@ end body end) - # Other things in the variable list should produce an error + # Local recursive function + let f(x) = f(x) + body + end + @Expr(:scope_block, begin + local f + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :x]), $([]), @Expr(:scope_block, f(x)))) + end + body + end) + + # Let with existing var on rhs + let x = x + a + body + end + @Expr(:scope_block, begin + ssa1 = x + a + @Expr(:scope_block, begin + @Expr(:local_def, x) + x = ssa1 + body + end) + end) + + # Destructuring + let (a, b) = (c, d) + body + end + @Expr(:scope_block, + begin + @Expr(:local_def, a) + @Expr(:local_def, b) + begin + a = c + b = d + maybe_unused(Core.tuple(c, d)) + end + body + end) + + # Destructuring with existing vars on rhs + let (a, b) = (a, d) + body + end + begin + ssa1 = Core.tuple(a, d) + @Expr(:scope_block, + begin + @Expr(:local_def, a) + @Expr(:local_def, b) + begin + begin + ssa2 = Top.indexed_iterate(ssa1, 1) + a = Core.getfield(ssa2, 1) + gsym1 = Core.getfield(ssa2, 2) + ssa2 + end + begin + ssa3 = Top.indexed_iterate(ssa1, 2, gsym1) + b = Core.getfield(ssa3, 1) + ssa3 + end + maybe_unused(ssa1) + end + body + end) + end + + # Other expressions in the variable list should produce an error let f(x) body end @Expr(:error, "invalid let syntax") + + let x[i] = a + end + @Expr(:error, "invalid let syntax") end @testset_desugar "Loops" begin From e3af5ff8ac41da6ef5c3c39b398e2b060b574110 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 08:21:01 +1000 Subject: [PATCH 31/56] Tests for macro lowering --- test/compiler/lowering.jl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index eb16cdcd660c4..843bd804c2082 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1240,6 +1240,37 @@ ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) end end +@testset_desugar "Macros" begin + macro foo + end + @Expr(:method, $(Symbol("@foo"))) + + macro foo(ex) + body(ex) + end + begin + @Expr(:method, $(Symbol("@foo"))) + @Expr(:method, $(Symbol("@foo")), + Core.svec(Core.svec(Core.Typeof($(Symbol("@foo"))), Core.LineNumberNode, Core.Module, Core.Any), Core.svec()), + @Expr(:lambda, $([:_self_, :__source__, :__module__, :ex]), $([]), + @Expr(:scope_block, + begin + @Expr(:meta, nospecialize, ex) + body(ex) + end))) + maybe_unused($(Symbol("@foo"))) + end + + macro foo(ex; x=a) + body(ex) + end + @Expr(:error, "macros cannot accept keyword arguments") + + macro () + end + @Expr(:error, "invalid macro definition") +end + @testset "Forms without desugaring" begin # (expand-forms) # The following Expr heads are currently not touched by desugaring From 1cd0b0043c3537660a781d2f7653a6d42d4aec99 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 10:37:15 +1000 Subject: [PATCH 32/56] Translate true and false in flisp ASTs to Julia Also add a more helpful error message for the cases where translating an flisp AST into Juila fails. --- src/ast.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ast.c b/src/ast.c index 1462e55f2f075..c8a91ce0289d5 100644 --- a/src/ast.c +++ b/src/ast.c @@ -613,7 +613,11 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m if (iscvalue(e) && cv_class((cvalue_t*)ptr(e)) == jl_ast_ctx(fl_ctx)->jvtype) { return *(jl_value_t**)cv_data((cvalue_t*)ptr(e)); } - jl_error("malformed tree"); + if (e == fl_ctx->T || e == fl_ctx->F) { + return e == fl_ctx->T ? jl_true : jl_false; + } + value_t e_str = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "string")), e); + jl_errorf("malformed tree %.*s", cvalue_len(e_str), (char*)cvalue_data(e_str)); } static value_t julia_to_scm_(fl_context_t *fl_ctx, jl_value_t *v); From 8559e5a3ebb7d96339fef95323233bc122f91581 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 16:46:34 +1000 Subject: [PATCH 33/56] fixup! Rename internal Expr heads to use _ rather than - --- src/julia-syntax.scm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 27ababb57deb6..e4cac1041ef41 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -174,7 +174,7 @@ s (error (string "invalid type parameter name \"" (deparse s) "\"")))) (cond ((atom? e) (list (check-sym e) #f #f)) - ((eq? (car e) 'var-bounds) (cdr e)) + ((eq? (car e) 'var_bounds) (cdr e)) ((and (eq? (car e) 'comparison) (length= e 6)) (cons (check-sym (cadddr e)) (cond ((and (eq? (caddr e) '|<:|) (eq? (caddr (cddr e)) '|<:|)) @@ -662,7 +662,7 @@ `(curly ,name ,@params) name) ,@field-names) - (map (lambda (b) (cons 'var-bounds b)) bounds)) + (map (lambda (b) (cons 'var_bounds b)) bounds)) (block ,@locs (call new ,@field-names))))) @@ -688,7 +688,7 @@ (let ((field-names (safe-field-names field-names field-types))) `(function ,(with-wheres `(call ,name ,@(map make-decl field-names field-types)) - (map (lambda (b) (cons 'var-bounds b)) bounds)) + (map (lambda (b) (cons 'var_bounds b)) bounds)) (block ,@locs (call (curly ,name ,@params) ,@field-names))))) From 08406eea6333709637ba92597bb9c543584a394a Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 18:02:54 +1000 Subject: [PATCH 34/56] Clean up partially expanded forms in flisp rather than C Convert partially expanded lambda forms into something we can test against more easily on the julia side. --- src/ast.c | 25 +++---------------------- src/jlfrontend.scm | 18 +++++++++++++----- test/compiler/lowering.jl | 2 +- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/src/ast.c b/src/ast.c index c8a91ce0289d5..f1c35e1b1ce72 100644 --- a/src/ast.c +++ b/src/ast.c @@ -572,25 +572,8 @@ static jl_value_t *scm_to_julia_(fl_context_t *fl_ctx, value_t e, jl_module_t *m jl_array_ptr_set(((jl_expr_t*)ex)->args, i, scm_to_julia_(fl_ctx, car_(e), mod, formonly)); e = cdr_(e); } - if (sym == lambda_sym) { - if (formonly) { - if (jl_array_len(((jl_expr_t*)ex)->args) >= 1 && - jl_is_expr(jl_array_ptr_ref(((jl_expr_t*)ex)->args, 0))) { - // Hack: fixup translation of lambda arg names (only used in testing) - jl_expr_t *argexpr = (jl_expr_t*)jl_array_ptr_ref(((jl_expr_t*)ex)->args, 0); - size_t nargs = jl_array_len(argexpr->args) + 1; - jl_array_t *args = jl_alloc_vec_any(nargs); - JL_GC_PUSH1(&args); - jl_array_ptr_set(args, 0, argexpr->head); - for (size_t i = 1; i < nargs; ++i) - jl_array_ptr_set(args, i, jl_array_ptr_ref(argexpr->args, i-1)); - jl_array_ptr_set(((jl_expr_t*)ex)->args, 0, args); - JL_GC_POP(); - } - } - else { - ex = (jl_value_t*)jl_new_code_info_from_ast((jl_expr_t*)ex); - } + if (sym == lambda_sym && !formonly) { + ex = (jl_value_t*)jl_new_code_info_from_ast((jl_expr_t*)ex); } JL_GC_POP(); if (sym == list_sym) @@ -950,9 +933,7 @@ JL_DLLEXPORT jl_value_t *jl_call_scm_on_ast_formonly(const char *funcname, jl_va fl_context_t *fl_ctx = &ctx->fl; JL_AST_PRESERVE_PUSH(ctx, old_roots, inmodule); value_t arg = julia_to_scm(fl_ctx, expr); - value_t e = fl_applyn(fl_ctx, 2, - symbol_value(symbol(fl_ctx, "jl-apply-with-error-wrap")), - symbol_value(symbol(fl_ctx, funcname)), arg); + value_t e = fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, funcname)), arg); jl_value_t *result = NULL; JL_GC_PUSH1(&result); int err = 0; diff --git a/src/jlfrontend.scm b/src/jlfrontend.scm index e0baa445dd207..4c07bc6554470 100644 --- a/src/jlfrontend.scm +++ b/src/jlfrontend.scm @@ -142,11 +142,19 @@ (error-wrap (lambda () (julia-expand-macroscope expr)))) -;; Debug tool: call named frontend function, translating flisp errors to AST -;; to avoid crashing -(define (jl-apply-with-error-wrap f . args) - (error-wrap (lambda () - (apply f args)))) +;; "Clean up" lambda expressions generated by the first lowering pass so that +;; they can be translated by the julia code. +(define (cleanup-lambdas e) + (if (not (pair? e)) + e + (let ((e2 (map cleanup-lambdas e))) + (if (eq? (car e) 'lambda) + `(lambda (vect ,@(cadr e2)) (vect ,@(caddr e2)) ,(cadddr e)) + e2)))) + +;; Lowering testing: call first pass only +(define (jl-expand-forms e) + (error-wrap (lambda () (cleanup-lambdas (expand-forms e))))) ;; construct default definitions of `eval` for non-bare modules ;; called by jl_eval_module_expr diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 843bd804c2082..0b7d39bc7a891 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -3,7 +3,7 @@ using Base: remove_linenums! # Call into lowering stage 1; syntax desugaring function fl_expand_forms(ex) - ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "expand-forms", ex, Main) + ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "jl-expand-forms", ex, Main) end # Make it easy to replace fl_expand_forms with a julia version in the future. From 218dc97e8457abc8dd89d28d3f0165441ec8bd6b Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 18:12:34 +1000 Subject: [PATCH 35/56] Fix lowering tests for neater lambda form variant --- test/compiler/lowering.jl | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 0b7d39bc7a891..5d91f62809f3d 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -824,7 +824,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), @Expr(:scope_block, 1))) + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, 1))) end body end) @@ -839,7 +839,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), @Expr(:scope_block, f(x)))) + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, f(x)))) end body end) @@ -1005,7 +1005,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, body(x)))) maybe_unused(f) @@ -1019,7 +1019,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), T, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:lambda, [_self_, x, y], [], @Expr(:scope_block, body(x)))) maybe_unused(f) @@ -1034,7 +1034,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f)), Core.svec()), - @Expr(:lambda, $([:_self_]), $([]), + @Expr(:lambda, [_self_], [], @Expr(:scope_block, _self_(a, b)))) maybe_unused(f) end @@ -1042,7 +1042,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, _self_(x, b)))) maybe_unused(f) end @@ -1050,7 +1050,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:lambda, [_self_, x, y], [], @Expr(:scope_block, body(x, y)))) maybe_unused(f) end @@ -1065,7 +1065,7 @@ end @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any, Core.apply_type(Vararg, Core.Any)), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :args]), $([]), + @Expr(:lambda, [_self_, x, args], [], @Expr(:scope_block, body(x, args)))) maybe_unused(f) end @@ -1081,7 +1081,7 @@ end @Expr(:method, gsym1) @Expr(:method, gsym1, Core.svec(Core.svec(Core.typeof(gsym1), Core.Any, Core.Any, Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:gsym1, :k1, :k2, :_self_, :x]), $([]), + @Expr(:lambda, [gsym1, k1, k2, _self_, x], [], @Expr(:scope_block, body))) maybe_unused(gsym1) end @@ -1089,7 +1089,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, return gsym1(v1, v2, _self_, x)))) maybe_unused(f) end @@ -1097,7 +1097,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.kwftype(Core.Typeof(f)), Core.Any, Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:gsym2, :gsym3, :_self_, :x]), $([]), + @Expr(:lambda, [gsym2, gsym3, _self_, x], [], @Expr(:scope_block, @Expr(:scope_block, begin @@ -1138,7 +1138,7 @@ end @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, begin ssa1 = T @@ -1156,7 +1156,7 @@ end @Expr(:method, gsym1) @Expr(:method, gsym1, Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x, :y]), $([]), + @Expr(:lambda, [_self_, x, y], [], @Expr(:scope_block, body(x, y)))) maybe_unused(gsym1) end @@ -1199,7 +1199,7 @@ ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) @Expr(:method, gsym1) @Expr(:method, gsym1, Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :gsym2, :x]), $([]), + @Expr(:lambda, [_self_, gsym2, x], [], @Expr(:scope_block, begin @Expr(:meta, nospecialize, gsym2, x) @@ -1220,7 +1220,7 @@ ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) @Expr(:method, f) @Expr(:method, f, Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :x]), $([]), + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, begin @Expr(:meta, generated, @@ -1252,7 +1252,7 @@ end @Expr(:method, $(Symbol("@foo"))) @Expr(:method, $(Symbol("@foo")), Core.svec(Core.svec(Core.Typeof($(Symbol("@foo"))), Core.LineNumberNode, Core.Module, Core.Any), Core.svec()), - @Expr(:lambda, $([:_self_, :__source__, :__module__, :ex]), $([]), + @Expr(:lambda, [_self_, __source__, __module__, ex], [], @Expr(:scope_block, begin @Expr(:meta, nospecialize, ex) From f8e7311857221054fba729b3df6ac4f3e6cb14c6 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 27 Jun 2019 18:26:28 +1000 Subject: [PATCH 36/56] A few more tests for function lowering --- test/compiler/lowering.jl | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 5d91f62809f3d..d836103d92db5 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1162,7 +1162,44 @@ end end end - # Invalid names + # Where syntax + function f(x::T, y::S) where {T <: S, S <: U} + body(x, y) + end + begin + @Expr(:method, f) + @Expr(:method, f, + begin + ssa1 = Core.TypeVar(:T, S) + ssa2 = Core.TypeVar(:S, U) + Core.svec(Core.svec(Core.Typeof(f), ssa1, ssa2), Core.svec(ssa1, ssa2)) + end, + @Expr(:lambda, [_self_, x, y], [], @Expr(:scope_block, body(x, y)))) + maybe_unused(f) + end + + # Type constraints + #= + function f(x::T{<:S}) + body(x, y) + end + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), + @Expr(:scope_block, + begin + @Expr(:local_def, gsym1) + gsym1 = Core.TypeVar(Symbol("#s167"), S) + Core.UnionAll(gsym1, Core.apply_type(T, gsym1)) + end)), Core.svec()), + @Expr(:lambda, [_self_, x], [], @Expr(:scope_block, body(x, y)))) + maybe_unused(f) + end + FIXME + =# + + # Invalid function names ccall(x)=body @Expr(:error, "invalid function name \"ccall\"") From bfaa5291ab16165677979ff39d1fd9973e319c3b Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 29 Jun 2019 01:02:21 +1000 Subject: [PATCH 37/56] Rearrange some tests by expr head --- test/compiler/lowering.jl | 89 +++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index d836103d92db5..5e84e70f84ac0 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -7,7 +7,7 @@ function fl_expand_forms(ex) end # Make it easy to replace fl_expand_forms with a julia version in the future. -expand_forms = ex->fl_expand_forms(ex) +_expand_forms = fl_expand_forms function lift_lowered_expr!(ex, nextids, valmap, lift_full) if ex isa SSAValue @@ -150,7 +150,7 @@ end # For interactive convenience in constructing test cases with flisp based lowering function desugar(ex; lift=:full) - expanded = expand_forms(ex) + expanded = _expand_forms(ex) if lift == :full || lift == :partial lift_lowered_expr(expanded; lift_full=(lift == :full)) else @@ -236,7 +236,7 @@ macro testset_desugar(name, block) input = exs[1] ref = exs[2] ex = quote - input = lift_lowered_expr(expand_forms($(Expr(:quote, input)))) + input = lift_lowered_expr(_expand_forms($(Expr(:quote, input)))) ref = lower_ref_expr($(Expr(:quote, ref))) @test input == ref if input != ref @@ -261,28 +261,6 @@ end #------------------------------------------------------------------------------- # Tests -@testset_desugar "Property notation" begin - # flisp: (expand-fuse-broadcast) - a.b - Top.getproperty(a, :b) - - a.b.c - Top.getproperty(Top.getproperty(a, :b), :c) - - a.b=c - begin - Top.setproperty!(a, :b, c) - maybe_unused(c) - end - - a.b.c=d - begin - ssa1 = Top.getproperty(a, :b) - Top.setproperty!(ssa1, :c, d) - maybe_unused(d) - end -end - @testset_desugar "Index notation; getindex" begin # Indexing a[i] @@ -328,22 +306,6 @@ end end end -@testset_desugar "Index notation; setindex!" begin - # flisp: (lambda in expand-table) - a[i] = b - begin - Top.setindex!(a, b, i) - maybe_unused(b) - end - - a[i,end] = b+c - begin - ssa1 = b+c - Top.setindex!(a, ssa1, i, Top.lastindex(a,2)) - maybe_unused(ssa1) - end -end - @testset_desugar "Array Literals" begin # flisp: (in expand-table) [a,b] @@ -499,8 +461,17 @@ end @Expr(:error, "\"\$\" expression outside quote") end -@testset_desugar "Broadcast" begin +@testset_desugar "Dot syntax; broadcast/getproperty" begin # flisp: (expand-fuse-broadcast) + + # Property access + a.b + Top.getproperty(a, :b) + + a.b.c + Top.getproperty(Top.getproperty(a, :b), :c) + + # Broadcast # Basic x .+ y Top.materialize(Top.broadcasted(+, x, y)) @@ -595,6 +566,34 @@ end @testset_desugar "Assignment" begin # flisp: (lambda in expand-table) + # property notation + a.b = c + begin + Top.setproperty!(a, :b, c) + maybe_unused(c) + end + + a.b.c = d + begin + ssa1 = Top.getproperty(a, :b) + Top.setproperty!(ssa1, :c, d) + maybe_unused(d) + end + + # setindex + a[i] = b + begin + Top.setindex!(a, b, i) + maybe_unused(b) + end + + a[i,end] = b+c + begin + ssa1 = b+c + Top.setindex!(a, ssa1, i, Top.lastindex(a,2)) + maybe_unused(ssa1) + end + # Assignment chain; nontrivial rhs x = y = f(a) begin @@ -1313,11 +1312,11 @@ end # The following Expr heads are currently not touched by desugaring for head in [:quote, :top, :core, :globalref, :outerref, :module, :toplevel, :null, :meta, :using, :import, :export] ex = Expr(head, Expr(:foobar, :junk, nothing, 42)) - @test expand_forms(ex) == ex + @test _expand_forms(ex) == ex end # flisp: inert,line have special representations on the julia side - @test expand_forms(QuoteNode(Expr(:$, :x))) == QuoteNode(Expr(:$, :x)) # flisp: `(inert ,expr) - @test expand_forms(LineNumberNode(1, :foo)) == LineNumberNode(1, :foo) # flisp: `(line ,line ,file) + @test _expand_forms(QuoteNode(Expr(:$, :x))) == QuoteNode(Expr(:$, :x)) # flisp: `(inert ,expr) + @test _expand_forms(LineNumberNode(1, :foo)) == LineNumberNode(1, :foo) # flisp: `(line ,line ,file) end From ac5f10c91a3ecddb908184ea730d2ad31aaba378 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 29 Jun 2019 02:23:34 +1000 Subject: [PATCH 38/56] Begin implementing ref desugaring --- base/compiler/lowering/desugar.jl | 78 +++++++++++++++++++++++++++++++ test/compiler/lowering.jl | 4 ++ 2 files changed, 82 insertions(+) create mode 100644 base/compiler/lowering/desugar.jl diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl new file mode 100644 index 0000000000000..1d999546f2057 --- /dev/null +++ b/base/compiler/lowering/desugar.jl @@ -0,0 +1,78 @@ +# Lowering pass 1: Syntax desugaring +# +# In this pass, we simplify the AST by transforming much of the rich surface +# syntax into a smaller core syntax containing fewer expression heads. +# +# Some of this core syntax is also part of the surface syntax, but some is +# unique to the lowered code. For example, `Expr(:scope_block, ...)` all +# scoping in the core syntax is +# handled by the scope_block + +using Core: SSAValue + +# :(f(arg; par)).arg s => [:f, Expr(:parameters, :par), :arg] +has_parameters(args) = length(args) >= 2 && args[2] isa Expr && args[2].head === :parameters + + +isquoted(ex) = ex isa QuoteNode || (ex isa Expr && + ex.head in (:quote, :top, :core, :globalref, :outerref, :break, :inert, :meta)) + +# TODO: Shouldn't be global +const _ssa_index = Ref(0) +make_ssa() = SSAValue(_ssa_index += 1) + +top(ex) = Expr(:top, ex) +core(ex) = Expr(:core, ex) +mapargs(f, ex) = ex isa Expr ? Expr(ex.head, map(f, ex.args)...) : ex + +# replace `end` for the closest ref expression; don't go inside nested refs +function replace_end(ex, a, n, tuples, last) + if ex === :end + # the appropriate computation for an `end` symbol for indexing + # the array `a` in the `n`th index. + # `tuples` are a list of the splatted arguments that precede index `n` + # `last` = is this last index? + # returns a call to lastindex(a) or lastindex(a,n) + if isempty(tuples) + last && n == 1 ? Expr(:call, top(:lastindex), a) : + Expr(:call, top(:lastindex), a, n) + # @top(lastindex($a)) : @top(lastindex($a, $n)) ?? + else + dimno = Expr(:call, top(:+), n - length(tuples), + map(t->:(Expr(:call, top(:length), t)), tuples)...) + Expr(:call, top(:lastindex), a, dimno) + end + elseif !(ex isa Expr) || isquoted(ex) + ex + elseif ex.head == :ref + # Only recurse into first argument of ref, not into index list. + Expr(:ref, replace_end(ex.args[1], a, n, tuples, last), ex.args[2:end]...) + else + mapargs(x->replace_end(x, a, n, tuples, last), ex) + end +end + +# go through indices and replace the `end` symbol +# a = array being indexed, i = list of indices +# returns (values, index_list, stmts) where stmts are statements that need to +# execute first. +function process_indices(a, inds) + +end + +function partially_expand_ref(ex) + a = ex.args[1] + idxs = ex.args[2] +end + +function expand_forms(ex) + if ex isa Symbol + ex + elseif ex isa Expr + head = ex.head + if head == :ref + !has_parameters(ex.args) || error("unexpected semicolon in array expression") + expand_forms(partially_expand_ref(ex)) + end + end +end diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 5e84e70f84ac0..8a956e9e4c806 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -289,6 +289,10 @@ end a[end][b[i]] Top.getindex(Top.getindex(a, Top.lastindex(a)), Top.getindex(b,i)) + # `end` replacment for first agument of Expr(:ref) + a[f(end)[i]] + Top.getindex(a, Top.getindex(f(Top.lastindex(a)), i)) + # Interaction of `end` with splatting a[I..., end, J..., end] Core._apply(Top.getindex, Core.tuple(a), From 2e9a5efb864351631a702b774c832e4c6813202e Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 30 Jun 2019 23:17:52 +1000 Subject: [PATCH 39/56] Bugfix: a.b is no longer effect free --- src/ast.scm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ast.scm b/src/ast.scm index b9d4ac24c19ca..31990ef98ade5 100644 --- a/src/ast.scm +++ b/src/ast.scm @@ -357,7 +357,7 @@ ;; identify some expressions that are safe to repeat (define (effect-free? e) - (or (not (pair? e)) (ssavalue? e) (sym-dot? e) (quoted? e) (equal? e '(null)))) + (or (not (pair? e)) (ssavalue? e) (quoted? e) (equal? e '(null)))) ;; get the variable name part of a declaration, x::int => x (define (decl-var v) From 64139bb40aa47629e0a70e642945448f1f77f1be Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 30 Jun 2019 23:18:57 +1000 Subject: [PATCH 40/56] Avoid traversing index list of refs twice Instead of trying (unnecessarily) to figure out if an `end` is present, just use an extra ssavalue. --- src/julia-syntax.scm | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index e4cac1041ef41..9e33a361957d3 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1467,11 +1467,9 @@ (define (partially-expand-ref e) (let ((a (cadr e)) (idxs (cddr e))) - (let* ((reuse (and (pair? a) - (contains (lambda (x) (eq? x 'end)) - idxs))) - (arr (if reuse (make-ssavalue) a)) - (stmts (if reuse `((= ,arr ,a)) '()))) + (let* ((rename (not (effect-free? a))) + (arr (if rename (make-ssavalue) a)) + (stmts (if rename `((= ,arr ,a)) '()))) (receive (new-idxs stuff) (process-indices arr idxs) `(block From 7962228b14d98055b14b4436fe8358154b560aec Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Mon, 1 Jul 2019 00:12:08 +1000 Subject: [PATCH 41/56] Implement lowering of Expr(:ref) * Skeleton of julia version of expand_forms * Lowering of ref * Allow to use either juila or flisp expander in tests * Adjust tests for ref to allow some extra ssa values * Add a few extra test cases --- base/compiler/lowering/desugar.jl | 241 +++++++++++++++++++++++++----- test/compiler/lowering.jl | 61 ++++++-- 2 files changed, 256 insertions(+), 46 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index 1d999546f2057..feae46a90819d 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -10,69 +10,242 @@ using Core: SSAValue +#------------------------------------------------------------------------------- +# AST tools + # :(f(arg; par)).arg s => [:f, Expr(:parameters, :par), :arg] has_parameters(args) = length(args) >= 2 && args[2] isa Expr && args[2].head === :parameters - +has_assignment(args) = any(isassignment, args) isquoted(ex) = ex isa QuoteNode || (ex isa Expr && ex.head in (:quote, :top, :core, :globalref, :outerref, :break, :inert, :meta)) +issymbollike(ex) = ex isa Symbol || ex isa SSAValue + +isassignment(ex) = ex isa Expr && ex.head == :(=) + +# True if `ex` is trivially free of side effects (and hence safe to repeat) +iseffectfree(ex) = !(ex isa Expr) || isquoted(ex) -# TODO: Shouldn't be global -const _ssa_index = Ref(0) -make_ssa() = SSAValue(_ssa_index += 1) +# FIXME: Counter Should be thread local or in expansion ctx +let ssa_index = Ref(0) + global make_ssavalue() = SSAValue(ssa_index[] += 1) +end top(ex) = Expr(:top, ex) core(ex) = Expr(:core, ex) mapargs(f, ex) = ex isa Expr ? Expr(ex.head, map(f, ex.args)...) : ex -# replace `end` for the closest ref expression; don't go inside nested refs -function replace_end(ex, a, n, tuples, last) +# Symbol `s` occurs in ex +occursin_ex(s::Symbol, ex::Symbol) = s === ex +occursin_ex(s::Symbol, ex::Expr) = occursin_ex(s, ex.args) +occursin_ex(s::Symbol, exs) = any(e->occursin_ex(s, e), exs) + +""" + make_ssa_if(need_ssa, ex, stmts) + +Return a name for the value of `ex` that can be used multiple times. +An extra assignment is recorded into `stmts` if necessary. +""" +function make_ssa_if(need_ssa::Bool, ex, stmts) + if need_ssa + v = make_ssavalue() + push!(stmts, Expr(:(=), v, ex)) + v + else + ex + end +end + +make_ssa_if(need_ssa::Function, ex, stmts) = make_ssa_if(need_ssa(ex), ex, stmts) + +function check_no_assigments(ex) + for e in ex.args + !isassignment(e) || error("misplaced assigment statement in `$ex`") + end +end +error_unexpected_semicolon(ex) = error("unexpected semicolon in `$ex`") + + +#------------------------------------------------------------------------------- + +""" +Replace `end` for the closest ref expression; don't go inside nested refs +`preceding_splats` are a list of the splatted arguments that precede index `n`. +`end`s are replaced with a call to `lastindex(a)` if `n == nothing`, or +`lastindex(a,n)`. +""" +function replace_end(ex, a, n, preceding_splats) if ex === :end # the appropriate computation for an `end` symbol for indexing # the array `a` in the `n`th index. - # `tuples` are a list of the splatted arguments that precede index `n` - # `last` = is this last index? - # returns a call to lastindex(a) or lastindex(a,n) - if isempty(tuples) - last && n == 1 ? Expr(:call, top(:lastindex), a) : - Expr(:call, top(:lastindex), a, n) - # @top(lastindex($a)) : @top(lastindex($a, $n)) ?? + if isempty(preceding_splats) + n === nothing ? Expr(:call, top(:lastindex), a) : + Expr(:call, top(:lastindex), a, n) else - dimno = Expr(:call, top(:+), n - length(tuples), - map(t->:(Expr(:call, top(:length), t)), tuples)...) + dimno = Expr(:call, top(:+), n - length(preceding_splats), + map(t->:(Expr(:call, top(:length), t)), preceding_splats)...) Expr(:call, top(:lastindex), a, dimno) end elseif !(ex isa Expr) || isquoted(ex) ex elseif ex.head == :ref # Only recurse into first argument of ref, not into index list. - Expr(:ref, replace_end(ex.args[1], a, n, tuples, last), ex.args[2:end]...) + Expr(:ref, replace_end(ex.args[1], a, n, preceding_splats), ex.args[2:end]...) else - mapargs(x->replace_end(x, a, n, tuples, last), ex) + mapargs(x->replace_end(x, a, n, preceding_splats), ex) end end -# go through indices and replace the `end` symbol -# a = array being indexed, i = list of indices -# returns (values, index_list, stmts) where stmts are statements that need to -# execute first. -function process_indices(a, inds) - -end - +# Expand Expr(:ref, indexable, indices...) by replacing `end` within `indices` +# as necessary function partially_expand_ref(ex) a = ex.args[1] - idxs = ex.args[2] + stmts = [] + arr = make_ssa_if(!iseffectfree, a, stmts) + preceding_splats = [] + new_idxs = [] + N = length(ex.args) - 1 + # go through indices and replace any embedded `end` symbols + for i = 1:N + idx = ex.args[i+1] + n = N == 1 ? nothing : i + if idx isa Expr && idx.head == :... + idx = replace_end(idx.args[1], arr, n, preceding_splats) + tosplat = make_ssa_if(issymbollike, idx, stmts) + push!(preceding_splats, tosplat) + push!(new_idxs, Expr(:..., tosplat)) + else + push!(new_idxs, replace_end(idx, arr, n, preceding_splats)) + end + end + Expr(:block, + stmts..., + Expr(:call, top(:getindex), arr, new_idxs...)) +end + + +#------------------------------------------------------------------------------- +# Expansion entry point + +function expand_todo(ex) + Expr(ex.head, map(e->expand_forms(e), ex.args)...) end function expand_forms(ex) - if ex isa Symbol - ex - elseif ex isa Expr - head = ex.head - if head == :ref - !has_parameters(ex.args) || error("unexpected semicolon in array expression") - expand_forms(partially_expand_ref(ex)) - end + if !(ex isa Expr) + return ex + end + head = ex.head + args = ex.args + # TODO: Use a hash table here like expand-table? + if head == :function + expand_todo(ex) # expand-function-def + elseif head == :-> + expand_todo(ex) # expand-arrow + elseif head == :let + expand_todo(ex) # expand-let + elseif head == :macro + expand_todo(ex) # expand-macro-def + elseif head == :struct + expand_todo(ex) # expand-struct-def + elseif head == :try + expand_todo(ex) # expand-try + elseif head == :lambda + expand_todo(ex) # expand-table + elseif head == :block + expand_todo(ex) # expand-table + elseif head == :. + expand_todo(ex) # expand-fuse-broadcast + elseif head == :.= + expand_todo(ex) # expand-fuse-broadcast + elseif head == :<: + expand_todo(ex) # expand-table + elseif head == :>: + expand_todo(ex) # expand-table + elseif head == :where + expand_todo(ex) # expand-wheres + elseif head == :const + expand_todo(ex) # expand-const-decl + elseif head == :local + expand_todo(ex) # expand-local-or-global-decl + elseif head == :global + expand_todo(ex) # expand-local-or-global-decl + elseif head == :local_def + expand_todo(ex) # expand-local-or-global-decl + elseif head == :(=) + expand_todo(ex) # expand-table + elseif head == :abstract + expand_todo(ex) # expand-table + elseif head == :primitive + expand_todo(ex) # expand-table + elseif head == :comparison + expand_todo(ex) # expand-compare-chain + elseif head == :ref + !has_parameters(args) || error_unexpected_semicolon(ex) + expand_forms(partially_expand_ref(ex)) + elseif head == :curly + expand_todo(ex) # expand-table + elseif head == :call + expand_todo(ex) # expand-table + elseif head == :do + expand_todo(ex) # expand-table + elseif head == :tuple + # TODO: NamedTuple lower-named-tuple + #if has_parameters(args) + #end + expand_forms(Expr(:call, core(:tuple), args...)) + elseif head == :braces + expand_todo(ex) # expand-table + elseif head == :bracescat + expand_todo(ex) # expand-table + elseif head == :string + expand_todo(ex) # expand-table + elseif head == :(::) + expand_todo(ex) # expand-table + elseif head == :while + expand_todo(ex) # expand-table + elseif head == :break + expand_todo(ex) # expand-table + elseif head == :continue + expand_todo(ex) # expand-table + elseif head == :for + expand_todo(ex) # expand-for + elseif head == :&& + expand_todo(ex) # expand-and + elseif head == :|| + expand_todo(ex) # expand-or + elseif head in (:(+=), :(-=), :(*=), :(.*=), :(/=), :(./=), :(//=), :(.//=), + :(\=), :(.\=), :(.+=), :(.-=), :(^=), :(.^=), :(÷=), :(.÷=), + :(%=), :(.%=), :(|=), :(.|=), :(&=), :(.&=), :($=), :(⊻=), + :(.⊻=), :(<<=), :(.<<=), :(>>=), :(.>>=), :(>>>=), :(.>>>=)) + expand_todo(ex) # lower-update-op + elseif head == :... + expand_todo(ex) # expand-table + elseif head == :$ + expand_todo(ex) # expand-table + elseif head == :vect + !has_parameters(args) || error_unexpected_semicolon(ex) + check_no_assigments(ex) + expand_forms(Expr(:call, top(:vect), args...)) + elseif head == :hcat + expand_todo(ex) # expand-table + elseif head == :vcat + expand_todo(ex) # expand-table + elseif head == :typed_hcat + expand_todo(ex) # expand-table + elseif head == :typed_vcat + expand_todo(ex) # expand-table + elseif head == Symbol("'") + expand_todo(ex) # expand-table + elseif head == :generator + expand_todo(ex) # expand-generator + elseif head == :flatten + expand_todo(ex) # expand-generator + elseif head == :comprehension + expand_todo(ex) # expand-table + elseif head == :typed_comprehension + expand_todo(ex) # lower-comprehension + else + Expr(head, map(e->expand_forms(e), args)...) end end diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 8a956e9e4c806..8b88366af5bc3 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -6,8 +6,13 @@ function fl_expand_forms(ex) ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "jl-expand-forms", ex, Main) end +include("../../base/compiler/lowering/desugar.jl") + # Make it easy to replace fl_expand_forms with a julia version in the future. -_expand_forms = fl_expand_forms +if !isdefined(@__MODULE__, :use_flisp) + use_flisp = true +end +_expand_forms(ex) = use_flisp ? fl_expand_forms(ex) : expand_forms(ex) function lift_lowered_expr!(ex, nextids, valmap, lift_full) if ex isa SSAValue @@ -118,7 +123,7 @@ lower_ref_expr(ex) = lower_ref_expr!(remove_linenums!(deepcopy(ex))) function diffdump(io::IOContext, ex1, ex2, n, prefix, indent) if ex1 == ex2 isempty(prefix) || print(io, prefix) - dump(io, ex1, n, indent) + dump(io, ex1, 2, indent) else if ex1 isa Expr && ex2 isa Expr && ex1.head == ex2.head && length(ex1.args) == length(ex2.args) isempty(prefix) || print(io, prefix) @@ -127,13 +132,13 @@ function diffdump(io::IOContext, ex1, ex2, n, prefix, indent) println(io, indent, " args: Array{Any}(", size(ex1.args), ")") for i in 1:length(ex1.args) prefix = string(indent, " ", i, ": ") - diffdump(io, ex1.args[i], ex2.args[i], n - 1, prefix, string(" ", indent)) + diffdump(io, ex1.args[i], ex2.args[i], 4, prefix, string(" ", indent)) i < length(ex1.args) && println(io) end else - printstyled(io, string(prefix, sprint(dump, ex1, n, indent; context=io)), color=:red) + printstyled(io, string(prefix, sprint(dump, ex1, 4, indent; context=io)), color=:green) println() - printstyled(io, string(prefix, sprint(dump, ex2, n, indent; context=io)), color=:green) + printstyled(io, string(prefix, sprint(dump, ex2, 4, indent; context=io)), color=:red) end end end @@ -236,14 +241,14 @@ macro testset_desugar(name, block) input = exs[1] ref = exs[2] ex = quote - input = lift_lowered_expr(_expand_forms($(Expr(:quote, input)))) - ref = lower_ref_expr($(Expr(:quote, ref))) - @test input == ref - if input != ref + expanded = lift_lowered_expr(_expand_forms($(Expr(:quote, input)))) + reference = lower_ref_expr($(Expr(:quote, ref))) + @test expanded == reference + if expanded != reference # Kinda crude. Would be much neater if Test supported custom/more # capable diffing for failed tests. println("Diff dump:") - diffdump(input, ref) + diffdump(expanded, reference) end end # Attribute the test to the correct line number @@ -286,12 +291,25 @@ end a[f(end) + 1] Top.getindex(a, f(Top.lastindex(a)) + 1) + # array expr is only emitted once if it can have side effects + (f(x))[end] + begin + ssa1 = f(x) + Top.getindex(ssa1, Top.lastindex(ssa1)) + end + a[end][b[i]] - Top.getindex(Top.getindex(a, Top.lastindex(a)), Top.getindex(b,i)) + begin + ssa1 = Top.getindex(a, Top.lastindex(a)) + Top.getindex(ssa1, Top.getindex(b, i)) + end # `end` replacment for first agument of Expr(:ref) a[f(end)[i]] - Top.getindex(a, Top.getindex(f(Top.lastindex(a)), i)) + Top.getindex(a, begin + ssa1 = f(Top.lastindex(a)) + Top.getindex(ssa1, i) + end) # Interaction of `end` with splatting a[I..., end, J..., end] @@ -364,6 +382,14 @@ end (x=a,y=b) Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) + + # Why do we allow this form? + (;x=a,y=b) + Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) + + # Mixed tuple + named tuple + (1; x=a, y=b) + @Expr(:error, "unexpected semicolon in tuple") end @testset_desugar "Splatting" begin @@ -551,6 +577,17 @@ end end end + # getproperty(x,y) only eval'd once. + x.y.z += a + begin + ssa1 = Top.getproperty(x, :y) + begin + ssa2 = Top.getproperty(ssa1, :z) + a + Top.setproperty!(ssa1, :z, ssa2) + maybe_unused(ssa2) + end + end + (x,y) .+= a begin ssa1 = Core.tuple(x, y) From ba00a73eb77556de8dadba3957b1c9983175be0a Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 2 Jul 2019 18:40:40 +1000 Subject: [PATCH 42/56] Remove ternary test - this is lowered by the parser. --- test/compiler/lowering.jl | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 8b88366af5bc3..cbed2fb8d6623 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -451,7 +451,7 @@ end Top.broadcasted(<, ssa1, d))) end -@testset_desugar "Short circuit; ternary" begin +@testset_desugar "Short circuit boolean operators" begin # flisp: (expand-or) (expand-and) a || b if a @@ -466,13 +466,6 @@ end else false end - - a ? x : y - if a - x - else - y - end end @testset_desugar "Misc operators" begin From 3f8f0786f37e2c2cb8713b49a1f2aac429c141d9 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 2 Jul 2019 19:07:31 +1000 Subject: [PATCH 43/56] Implement a few more easy lowerings --- base/compiler/lowering/desugar.jl | 24 ++++++++++++++++-------- test/compiler/lowering.jl | 28 +++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index feae46a90819d..191b9d285d2a6 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -33,6 +33,7 @@ end top(ex) = Expr(:top, ex) core(ex) = Expr(:core, ex) +blockify(ex) = ex isa Expr && ex.head !== :block ? ex : Expr(:block, ex) # TODO: null Expr? mapargs(f, ex) = ex isa Expr ? Expr(ex.head, map(f, ex.args)...) : ex # Symbol `s` occurs in ex @@ -67,6 +68,9 @@ error_unexpected_semicolon(ex) = error("unexpected semicolon in `$ex`") #------------------------------------------------------------------------------- +struct LoweringError <: Exception + msg::AbstractString +end """ Replace `end` for the closest ref expression; don't go inside nested refs @@ -159,9 +163,9 @@ function expand_forms(ex) elseif head == :.= expand_todo(ex) # expand-fuse-broadcast elseif head == :<: - expand_todo(ex) # expand-table + expand_forms(Expr(:call, :<:, args...)) elseif head == :>: - expand_todo(ex) # expand-table + expand_forms(Expr(:call, :>:, args...)) elseif head == :where expand_todo(ex) # expand-wheres elseif head == :const @@ -199,15 +203,19 @@ function expand_forms(ex) elseif head == :bracescat expand_todo(ex) # expand-table elseif head == :string - expand_todo(ex) # expand-table + expand_forms(Expr(:call, top(:string), args...)) elseif head == :(::) expand_todo(ex) # expand-table elseif head == :while - expand_todo(ex) # expand-table + Expr(:break_block, :loop_exit, + Expr(:_while, expand_forms(args[1]), + Expr(:break_block, :loop_cont, + Expr(:scope_block, + blockify(map(expand_forms, args[2:end])...))))) elseif head == :break - expand_todo(ex) # expand-table + isempty(args) ? Expr(:break, :loop_exit) : ex elseif head == :continue - expand_todo(ex) # expand-table + isempty(args) ? Expr(:break, :loop_cont) : ex elseif head == :for expand_todo(ex) # expand-for elseif head == :&& @@ -222,7 +230,7 @@ function expand_forms(ex) elseif head == :... expand_todo(ex) # expand-table elseif head == :$ - expand_todo(ex) # expand-table + throw(LoweringError("`\$` expression outside quote")) elseif head == :vect !has_parameters(args) || error_unexpected_semicolon(ex) check_no_assigments(ex) @@ -236,7 +244,7 @@ function expand_forms(ex) elseif head == :typed_vcat expand_todo(ex) # expand-table elseif head == Symbol("'") - expand_todo(ex) # expand-table + expand_forms(Expr(:call, top(:adjoint), args...)) elseif head == :generator expand_todo(ex) # expand-generator elseif head == :flatten diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index cbed2fb8d6623..4fb3aa1108414 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -12,7 +12,18 @@ include("../../base/compiler/lowering/desugar.jl") if !isdefined(@__MODULE__, :use_flisp) use_flisp = true end -_expand_forms(ex) = use_flisp ? fl_expand_forms(ex) : expand_forms(ex) +function _expand_forms(ex) + if use_flisp + fl_expand_forms(ex) + else + try + expand_forms(ex) + catch exc + exc isa LoweringError || rethrow() + return Expr(:error, exc.msg) + end + end +end function lift_lowered_expr!(ex, nextids, valmap, lift_full) if ex isa SSAValue @@ -266,6 +277,8 @@ end #------------------------------------------------------------------------------- # Tests +@testset "Lowering" begin + @testset_desugar "Index notation; getindex" begin # Indexing a[i] @@ -482,6 +495,9 @@ end $(Expr(:$, :x)) @Expr(:error, "\"\$\" expression outside quote") + + x... + @Expr(:error, "\"...\" expression outside call") end @testset_desugar "Dot syntax; broadcast/getproperty" begin @@ -947,24 +963,25 @@ end @testset_desugar "Loops" begin # flisp: (expand-for) (lambda in expand-forms) - while cond - body1 + while cond' + body1' continue body2 break body3 end @Expr(:break_block, loop_exit, - @Expr(:_while, cond, + @Expr(:_while, Top.adjoint(cond), @Expr(:break_block, loop_cont, @Expr(:scope_block, begin - body1 + Top.adjoint(body1) @Expr :break loop_cont body2 @Expr :break loop_exit body3 end)))) + for i = a body1 continue @@ -1353,6 +1370,7 @@ end @test _expand_forms(LineNumberNode(1, :foo)) == LineNumberNode(1, :foo) # flisp: `(line ,line ,file) end +end #------------------------------------------------------------------------------- # Julia AST Notes From a83cae20baf97932f2753c0f55c388f24844e63b Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 2 Jul 2019 22:43:08 +1000 Subject: [PATCH 44/56] A few more function call tests --- test/compiler/lowering.jl | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 4fb3aa1108414..6481f25a0e315 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -405,14 +405,6 @@ end @Expr(:error, "unexpected semicolon in tuple") end -@testset_desugar "Splatting" begin - f(i,j,v...,k) - Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) - - x... - @Expr(:error, "\"...\" expression outside call") -end - @testset_desugar "Comparison chains" begin # flisp: (expand-compare-chain) a < b < c @@ -550,12 +542,32 @@ end Top.materialize!(x, Top.broadcasted(+, x, a)) end -@testset_desugar "Call with keyword arguments" begin - f(x,a=1) +@testset_desugar "Function call syntax" begin + # zero arg call + g[i]() + Top.getindex(g, i)() + + # ccall + + # splatting + f(i, j, v..., k) + Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) + + x... + @Expr(:error, "\"...\" expression outside call") + + # keyword arguments + f(x, a=1) begin ssa1 = Core.apply_type(Core.NamedTuple, Core.tuple(:a))(Core.tuple(1)) Core.kwfunc(f)(ssa1, f, x) end + + f(x; a=1) + begin + ssa1 = (Core.apply_type(Core.NamedTuple, Core.tuple(:a)))(Core.tuple(1)) + (Core.kwfunc(f))(ssa1, f, x) + end end @testset_desugar "In place update operators" begin From dbde6b3cf32504f28d40a3b9d3753e3ae6970c45 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Tue, 2 Jul 2019 22:43:43 +1000 Subject: [PATCH 45/56] Desugaring of array concatenation --- base/compiler/lowering/desugar.jl | 64 +++++++++++++++++++++++-------- src/julia-syntax.scm | 2 +- 2 files changed, 50 insertions(+), 16 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index 191b9d285d2a6..57b5481768ef3 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -32,6 +32,7 @@ let ssa_index = Ref(0) end top(ex) = Expr(:top, ex) +topcall(head, args...) = Expr(:call, Expr(:top, head), args...) core(ex) = Expr(:core, ex) blockify(ex) = ex isa Expr && ex.head !== :block ? ex : Expr(:block, ex) # TODO: null Expr? mapargs(f, ex) = ex isa Expr ? Expr(ex.head, map(f, ex.args)...) : ex @@ -61,10 +62,10 @@ make_ssa_if(need_ssa::Function, ex, stmts) = make_ssa_if(need_ssa(ex), ex, stmts function check_no_assigments(ex) for e in ex.args - !isassignment(e) || error("misplaced assigment statement in `$ex`") + !isassignment(e) || throw(LoweringError("misplaced assigment statement in `$ex`")) end end -error_unexpected_semicolon(ex) = error("unexpected semicolon in `$ex`") +error_unexpected_semicolon(ex) = throw(LoweringError("unexpected semicolon in `$ex`")) #------------------------------------------------------------------------------- @@ -83,12 +84,12 @@ function replace_end(ex, a, n, preceding_splats) # the appropriate computation for an `end` symbol for indexing # the array `a` in the `n`th index. if isempty(preceding_splats) - n === nothing ? Expr(:call, top(:lastindex), a) : - Expr(:call, top(:lastindex), a, n) + n === nothing ? topcall(:lastindex, a) : + topcall(:lastindex, a, n) else - dimno = Expr(:call, top(:+), n - length(preceding_splats), - map(t->:(Expr(:call, top(:length), t)), preceding_splats)...) - Expr(:call, top(:lastindex), a, dimno) + dimno = topcall(:+, n - length(preceding_splats), + map(t->:(topcall(:length, t)), preceding_splats)...) + topcall(:lastindex, a, dimno) end elseif !(ex isa Expr) || isquoted(ex) ex @@ -124,9 +125,30 @@ function partially_expand_ref(ex) end Expr(:block, stmts..., - Expr(:call, top(:getindex), arr, new_idxs...)) + topcall(:getindex, arr, new_idxs...)) end +function expand_hvcat(ex) + # rows inside vcat -> hvcat + lengths = Int[] + vals = [] + istyped = ex.head == :typed_vcat + for i in (istyped ? 2 : 1):length(ex.args) + e = ex.args[i] + if e isa Expr && e.head == :row + push!(lengths, length(e.args)) + append!(vals, e.args) + else + push!(lengths, 1) + push!(vals, e) + end + end + if istyped + expand_forms(topcall(:typed_hvcat, ex.args[1], Expr(:tuple, lengths...), vals...)) + else + expand_forms(topcall(:hvcat, Expr(:tuple, lengths...), vals...)) + end +end #------------------------------------------------------------------------------- # Expansion entry point @@ -203,7 +225,7 @@ function expand_forms(ex) elseif head == :bracescat expand_todo(ex) # expand-table elseif head == :string - expand_forms(Expr(:call, top(:string), args...)) + expand_forms(topcall(:string, args...)) elseif head == :(::) expand_todo(ex) # expand-table elseif head == :while @@ -234,17 +256,29 @@ function expand_forms(ex) elseif head == :vect !has_parameters(args) || error_unexpected_semicolon(ex) check_no_assigments(ex) - expand_forms(Expr(:call, top(:vect), args...)) + expand_forms(topcall(:vect, args...)) elseif head == :hcat - expand_todo(ex) # expand-table + check_no_assigments(ex) + expand_forms(topcall(:hcat, args...)) elseif head == :vcat - expand_todo(ex) # expand-table + check_no_assigments(ex) + if any(e->e isa Expr && e.head == :row, args) + expand_hvcat(ex) + else + expand_forms(topcall(:vcat, args...)) + end elseif head == :typed_hcat - expand_todo(ex) # expand-table + check_no_assigments(ex) + expand_forms(topcall(:typed_hcat, args...)) elseif head == :typed_vcat - expand_todo(ex) # expand-table + check_no_assigments(ex) + if any(e->e isa Expr && e.head == :row, args) + expand_hvcat(ex) + else + expand_forms(topcall(:typed_vcat, args...)) + end elseif head == Symbol("'") - expand_forms(Expr(:call, top(:adjoint), args...)) + expand_forms(topcall(:adjoint, args...)) elseif head == :generator expand_todo(ex) # expand-generator elseif head == :flatten diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 9e33a361957d3..4ef9bf1b11e1c 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1385,7 +1385,7 @@ (car e) (map (lambda (x) (cond ((effect-free? x) x) - ((or (eq? (car x) '...) (eq? (car x) '&)) + ((eq? (car x) '...) (if (effect-free? (cadr x)) x (let ((g (make-ssavalue))) From ddc917be769cb201d09756f6ddfbcc0b6f1a6640 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 3 Jul 2019 07:20:08 +1000 Subject: [PATCH 46/56] Quote all code in lowering error messages with backticks --- src/julia-syntax.scm | 142 +++++++++++++++++++------------------- test/compiler/lowering.jl | 48 +++++++------ 2 files changed, 96 insertions(+), 94 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 4ef9bf1b11e1c..db52f709d5fe6 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -10,7 +10,7 @@ a)) (define (fix-arglist l (unused #t)) (if (any vararg? (butlast l)) - (error "invalid ... on non-final argument")) + (error "invalid `...` on non-final argument")) (map (lambda (a) (cond ((and (pair? a) (eq? (car a) 'kw)) `(kw ,(fill-missing-argname (cadr a) unused) ,(caddr a))) @@ -172,19 +172,19 @@ (define (check-sym s) (if (symbol? s) s - (error (string "invalid type parameter name \"" (deparse s) "\"")))) + (error (string "invalid type parameter name `" (deparse s) "`")))) (cond ((atom? e) (list (check-sym e) #f #f)) ((eq? (car e) 'var_bounds) (cdr e)) ((and (eq? (car e) 'comparison) (length= e 6)) (cons (check-sym (cadddr e)) (cond ((and (eq? (caddr e) '|<:|) (eq? (caddr (cddr e)) '|<:|)) (list (cadr e) (last e))) - (else (error "invalid bounds in \"where\""))))) + (else (error "invalid bounds in `where`"))))) ((eq? (car e) '|<:|) (list (check-sym (cadr e)) #f (caddr e))) ((eq? (car e) '|>:|) (list (check-sym (cadr e)) (caddr e) #f)) - (else (error "invalid variable expression in \"where\"")))) + (else (error "invalid variable expression in `where`")))) (define (sparam-name-bounds params) (let ((bounds (map analyze-typevar params))) @@ -316,7 +316,7 @@ (if (any (lambda (x) (and (not (eq? x UNUSED)) (memq x names))) anames) (error "function argument and static parameter names must be distinct")) (if (or (and name (not (sym-ref? name))) (not (valid-name? name))) - (error (string "invalid function name \"" (deparse name) "\""))) + (error (string "invalid function name `" (deparse name) "`"))) (let* ((generator (if (expr-contains-p if-generated? body (lambda (x) (not (function-def? x)))) (let* ((gen (generated-version body)) (nongen (non-generated-version body)) @@ -603,8 +603,8 @@ (if (pair? invalid) (if (and (pair? (car invalid)) (eq? 'parameters (caar invalid))) (error "more than one semicolon in argument list") - (error (string "invalid keyword argument syntax \"" - (deparse (car invalid)) "\"")))))) + (error (string "invalid keyword argument syntax `" + (deparse (car invalid)) "`")))))) ; replace unassigned kw args with assignment to throw() call (forcing the caller to assign the keyword) (define (throw-unassigned-kw-args argl) @@ -695,9 +695,9 @@ (define (new-call Tname type-params params args field-names field-types) (if (any kwarg? args) - (error "\"new\" does not accept keyword arguments")) + (error "`new` does not accept keyword arguments")) (if (length> params (length type-params)) - (error "too few type parameters specified in \"new{...}\"")) + (error "too few type parameters specified in `new{...}`")) (let ((Texpr (if (null? type-params) `(outerref ,Tname) `(curly (outerref ,Tname) @@ -834,10 +834,10 @@ defs)) (min-initialized (min (ctors-min-initialized defs) (length fields)))) (let ((dups (has-dups field-names))) - (if dups (error (string "duplicate field name: \"" (car dups) "\" is not unique")))) + (if dups (error (string "duplicate field name: `" (car dups) "` is not unique")))) (for-each (lambda (v) (if (not (symbol? v)) - (error (string "field name \"" (deparse v) "\" is not a symbol")))) + (error (string "field name `" (deparse v) "` is not a symbol")))) field-names) `(block (global ,name) (const ,name) @@ -964,7 +964,7 @@ (define (check-lhs a) (if (expr-contains-p (lambda (e) (or (decl? e) (assignment? e) (kwarg? e))) a) - (error (string "invalid argument destructuring syntax \"" (deparse a) "\"")) + (error (string "invalid argument destructuring syntax `" (deparse a) "`")) a)) (define (transform-arg a) (cond ((and (pair? a) (eq? (car a) 'tuple)) @@ -995,7 +995,7 @@ (let ((w (flatten-where-expr name))) (begin0 (cddr w) (if (not (and (pair? (cadr w)) (memq (caadr w) '(call |::|)))) - (error (string "invalid assignment location \"" (deparse name) "\""))) + (error (string "invalid assignment location `" (deparse name) "`"))) (set! name (cadr w)))) #f)) (dcl (and (pair? name) (eq? (car name) '|::|))) @@ -1003,7 +1003,7 @@ (name (if dcl (cadr name) name))) (cond ((and (length= e 2) (or (symbol? name) (globalref? name))) (if (not (valid-name? name)) - (error (string "invalid function name \"" name "\""))) + (error (string "invalid function name `" name "`"))) `(method ,name)) ((not (pair? name)) e) ((eq? (car name) 'call) @@ -1038,7 +1038,7 @@ (expand-forms (method-def-expr name sparams argl body rett)))) (else - (error (string "invalid assignment location \"" (deparse name) "\"")))))) + (error (string "invalid assignment location `" (deparse name) "`")))))) ;; handle ( )->( ) function expressions. blocks `(a;b=1)` on the left need to be ;; converted to argument lists with kwargs. @@ -1181,7 +1181,7 @@ (cond ((or (symbol? x) (decl? x) (linenum? x)) (loop (cdr f))) ((and (assignment? x) (or (symbol? (cadr x)) (decl? (cadr x)))) - (error (string "\"" (deparse x) "\" inside type definition is reserved"))) + (error (string "`" (deparse x) "` inside type definition is reserved"))) (else '()))))) (expand-forms (receive (name params super) (analyze-type-sig sig) @@ -1235,7 +1235,7 @@ `(trycatch (scope_block ,tryb) (scope_block ,catchb))))) (else - (error "invalid \"try\" form"))))) + (error "invalid `try` form"))))) (define (expand-unionall-def name type-ex) (if (and (pair? name) @@ -1243,7 +1243,7 @@ (let ((name (cadr name)) (params (cddr name))) (if (null? params) - (error (string "empty type parameter list in \"" (deparse `(= (curly ,name) ,type-ex)) "\""))) + (error (string "empty type parameter list in `" (deparse `(= (curly ,name) ,type-ex)) "`"))) `(block (const_if_global ,name) ,(expand-forms @@ -1259,7 +1259,7 @@ (case (car arg) ((global local local_def) (for-each (lambda (b) (if (not (assignment? b)) - (error "expected assignment after \"const\""))) + (error "expected assignment after `const`"))) (cdr arg)) (expand-forms (expand-decls (car arg) (cdr arg) #t))) ((= |::|) @@ -1282,7 +1282,7 @@ ;; local x, (y=2), z => local x;local y;local z;y = 2 (define (expand-decls what binds const?) (if (not (list? binds)) - (error (string "invalid \"" what "\" declaration"))) + (error (string "invalid `" what "` declaration"))) (let loop ((b binds) (vars '()) (assigns '())) @@ -1306,7 +1306,7 @@ ((symbol? x) (loop (cdr b) (cons x vars) assigns)) (else - (error (string "invalid syntax in \"" what "\" declaration")))))))) + (error (string "invalid syntax in `" what "` declaration")))))))) ;; convert (lhss...) = (tuple ...) to assignments, eliminating the tuple (define (tuple-to-assignments lhss0 x) @@ -1432,8 +1432,8 @@ `(block ,@(if (eq? f fexpr) '() `((= ,f, fexpr))) (= ,kw-container ,(lower-named-tuple kw - (lambda (name) (string "keyword argument \"" name - "\" repeated in call to \"" (deparse fexpr) "\"")) + (lambda (name) (string "keyword argument `" name + "` repeated in call to `" (deparse fexpr) "`")) "keyword argument" "keyword argument syntax")) ,(if (every vararg? kw) @@ -1457,7 +1457,7 @@ ;; if remove-argument-side-effects needed to replace an expression with ;; an ssavalue, then it can't be updated by assignment. issue #30062 (begin (if (and (ssavalue? (car a)) (not (ssavalue? (car b)))) - (error (string "invalid multiple assignment location \"" (deparse (car b)) "\""))) + (error (string "invalid multiple assignment location `" (deparse (car b)) "`"))) (loop (cdr a) (cdr b)))))) `(block ,@(cdr e) ,(if (null? declT) @@ -1494,7 +1494,7 @@ (else (if (and (pair? lhs) (eq? op= '=) (not (memq (car lhs) '(|.| tuple vcat typed_hcat typed_vcat)))) - (error (string "invalid assignment location \"" (deparse lhs) "\""))) + (error (string "invalid assignment location `" (deparse lhs) "`"))) (expand-update-operator- op op= lhs rhs declT)))) (define (lower-update-op e) @@ -1670,7 +1670,7 @@ (list f (cadr x) (expand-forms `(call (call (core apply_type) (top Val) ,(caddr x)))))) (make-fuse f (cdr x)))) (else - (error (string "invalid syntax \"" (deparse e) "\""))))) + (error (string "invalid syntax `" (deparse e) "`"))))) (if (and (pair? e) (eq? (car e) 'call) (dotop-named? (cadr e))) (let ((f (undotop (cadr e))) (x (cddr e))) (if (and (eq? (identifier-name f) '^) (length= x 2) (integer? (cadr x))) @@ -1724,7 +1724,7 @@ (call (core tuple) ,@values))) (define (lower-named-tuple lst - (dup-error-fn (lambda (name) (string "field name \"" name "\" repeated in named tuple"))) + (dup-error-fn (lambda (name) (string "field name `" name "` repeated in named tuple"))) (name-str "named tuple field") (syntax-str "named tuple element")) (let* ((names (apply append @@ -1758,7 +1758,7 @@ (let ((el (car L))) (cond ((or (assignment? el) (kwarg? el)) (if (not (symbol? (cadr el))) - (error (string "invalid " name-str " name \"" (deparse (cadr el)) "\""))) + (error (string "invalid " name-str " name `" (deparse (cadr el)) "`"))) (loop (cdr L) (cons (cadr el) current-names) (cons (caddr el) current-vals) @@ -1790,7 +1790,7 @@ (merge current (cadr el)) `(call (top merge) (call (top NamedTuple)) ,(cadr el)))))) (else - (error (string "invalid " syntax-str " \"" (deparse el) "\"")))))))) + (error (string "invalid " syntax-str " `" (deparse el) "`")))))))) (define (expand-forms e) (if (or (atom? e) (memq (car e) '(quote inert top core globalref outerref line module toplevel ssavalue null meta using import export))) @@ -1883,7 +1883,7 @@ ((and (symbol-like? lhs) (valid-name? lhs)) `(= ,lhs ,(expand-forms (caddr e)))) ((atom? lhs) - (error (string "invalid assignment location \"" (deparse lhs) "\""))) + (error (string "invalid assignment location `" (deparse lhs) "`"))) (else (case (car lhs) ((globalref) @@ -1900,8 +1900,8 @@ (b (caddr lhs)) (rhs (caddr e))) (if (and (length= b 2) (eq? (car b) 'tuple)) - (error (string "invalid syntax \"" - (string (deparse a) ".(" (deparse (cadr b)) ") = ...") "\""))) + (error (string "invalid syntax `" + (string (deparse a) ".(" (deparse (cadr b)) ") = ...") "`"))) (let ((aa (if (symbol-like? a) a (make-ssavalue))) (bb (if (or (atom? b) (symbol-like? b) (and (pair? b) (quoted? b))) b (make-ssavalue))) @@ -1950,7 +1950,7 @@ ((typed_hcat) (error "invalid spacing in left side of indexed assignment")) ((typed_vcat) - (error "unexpected \";\" in left side of indexed assignment")) + (error "unexpected `;` in left side of indexed assignment")) ((ref) ;; (= (ref a . idxs) rhs) (let ((a (cadr lhs)) @@ -1985,9 +1985,9 @@ (= ,(car e) ,rhs)))))) ((vcat) ;; (= (vcat . args) rhs) - (error "use \"(a, b) = ...\" to assign multiple values")) + (error "use `(a, b) = ...` to assign multiple values")) (else - (error (string "invalid assignment location \"" (deparse lhs) "\""))))))) + (error (string "invalid assignment location `" (deparse lhs) "`"))))))) 'abstract (lambda (e) @@ -2017,9 +2017,9 @@ 'curly (lambda (e) (if (has-parameters? (cddr e)) - (error (string "unexpected semicolon in \"" (deparse e) "\""))) + (error (string "unexpected semicolon in `" (deparse e) "`"))) (if (any assignment? (cddr e)) - (error (string "misplaced assignment statement in \"" (deparse e) "\"" ))) + (error (string "misplaced assignment statement in `" (deparse e) "`" ))) (let* ((p (extract-implicit-whereparams e)) (curlyparams (car p)) (whereparams (cdr p))) @@ -2047,8 +2047,8 @@ (eq? (car argtypes) 'tuple))) (if (and (pair? RT) (eq? (car RT) 'tuple)) - (error "ccall argument types must be a tuple; try \"(T,)\" and check if you specified a correct return type") - (error "ccall argument types must be a tuple; try \"(T,)\""))) + (error "ccall argument types must be a tuple; try `(T,)` and check if you specified a correct return type") + (error "ccall argument types must be a tuple; try `(T,)`"))) (expand-forms (lower-ccall name RT (cdr argtypes) args (if have-cconv cconv 'ccall)))))) @@ -2119,7 +2119,7 @@ '|::| (lambda (e) (if (not (length= e 3)) - (error "invalid \"::\" syntax")) + (error "invalid `::` syntax")) (if (not (symbol-like? (cadr e))) `(call (core typeassert) ,(expand-forms (cadr e)) ,(expand-forms (caddr e))) @@ -2183,30 +2183,30 @@ '.>>>= lower-update-op '|...| - (lambda (e) (error "\"...\" expression outside call")) + (lambda (e) (error "`...` expression outside call")) '$ - (lambda (e) (error "\"$\" expression outside quote")) + (lambda (e) (error "`$` expression outside quote")) 'vect (lambda (e) (if (has-parameters? (cdr e)) (error "unexpected semicolon in array expression")) (if (any assignment? (cdr e)) - (error (string "misplaced assignment statement in \"" (deparse e) "\""))) + (error (string "misplaced assignment statement in `" (deparse e) "`"))) (expand-forms `(call (top vect) ,@(cdr e)))) 'hcat (lambda (e) (if (any assignment? (cdr e)) - (error (string "misplaced assignment statement in \"" (deparse e) "\""))) + (error (string "misplaced assignment statement in `" (deparse e) "`"))) (expand-forms `(call (top hcat) ,@(cdr e)))) 'vcat (lambda (e) (let ((a (cdr e))) (if (any assignment? a) - (error (string "misplaced assignment statement in \"" (deparse e) "\""))) + (error (string "misplaced assignment statement in `" (deparse e) "`"))) (if (has-parameters? a) (error "unexpected semicolon in array expression") ;; Obsolete? (expand-forms @@ -2227,7 +2227,7 @@ 'typed_hcat (lambda (e) (if (any assignment? (cddr e)) - (error (string "misplaced assignment statement in \"" (deparse e) "\""))) + (error (string "misplaced assignment statement in `" (deparse e) "`"))) (expand-forms `(call (top typed_hcat) ,@(cdr e)))) 'typed_vcat @@ -2235,7 +2235,7 @@ (let ((t (cadr e)) (a (cddr e))) (if (any assignment? (cddr e)) - (error (string "misplaced assignment statement in \"" (deparse e) "\""))) + (error (string "misplaced assignment statement in `" (deparse e) "`"))) (expand-forms (if (any (lambda (x) (and (pair? x) (eq? (car x) 'row))) @@ -2290,7 +2290,7 @@ (define (check-no-return e) (if (has-return? e) - (error "\"return\" not allowed inside comprehension or generator"))) + (error "`return` not allowed inside comprehension or generator"))) (define (has-break-or-continue? e) (expr-contains-p (lambda (x) (and (pair? x) (memq (car x) '(break continue)))) @@ -2403,7 +2403,7 @@ (define (check-valid-name e) (or (valid-name? e) - (error (string "invalid identifier name \"" e "\"")))) + (error (string "invalid identifier name `" e "`")))) (define (make-scope (lam #f) (args '()) (locals '()) (globals '()) (sp '()) (renames '()) (prev #f)) (vector lam args locals globals sp renames prev)) @@ -2462,7 +2462,7 @@ '(null)) ((eq? (car e) 'require_existing_local) (if (not (in-scope? (cadr e) scope)) - (error "no outer local variable declaration exists for \"for outer\"")) + (error "no outer local variable declaration exists for `for outer`")) '(null)) ((eq? (car e) 'locals) (let* ((names (filter (lambda (v) @@ -2515,17 +2515,17 @@ (newnames-def (append (diff locals-def need-rename-def) renamed-def))) (for-each (lambda (v) (if (or (memq v locals-def) (memq v local-decls)) - (error (string "variable \"" v "\" declared both local and global")))) + (error (string "variable `" v "` declared both local and global")))) globals) (if (and (pair? argnames) (eq? e (lam:body lam))) (for-each (lambda (v) (if (memq v argnames) - (error (string "local variable name \"" v "\" conflicts with an argument")))) + (error (string "local variable name `" v "` conflicts with an argument")))) local-decls)) (if (eq? e (lam:body lam)) (for-each (lambda (v) (if (or (memq v locals-def) (memq v local-decls) (memq v implicit-locals)) - (error (string "local variable name \"" v "\" conflicts with a static parameter")))) + (error (string "local variable name `" v "` conflicts with a static parameter")))) (scope:sp scope))) (if lam (set-car! (cddr lam) @@ -2546,7 +2546,7 @@ (map (lambda (v) `(local_def ,v)) newnames-def))) )) ((eq? (car e) 'module) - (error "\"module\" expression not at top level")) + (error "`module` expression not at top level")) ((eq? (car e) 'break_block) `(break_block ,(cadr e) ;; ignore type symbol of break_block expression ,(resolve-scopes- (caddr e) scope))) ;; body of break_block expression @@ -2675,11 +2675,11 @@ (let ((vi (var-info-for (cadr e) env))) (if vi (begin (if (not (equal? (vinfo:type vi) '(core Any))) - (error (string "multiple type declarations for \"" - (cadr e) "\""))) + (error (string "multiple type declarations for `" + (cadr e) "`"))) (if (assq (cadr e) captvars) - (error (string "type of \"" (cadr e) - "\" declared in inner scope"))) + (error (string "type of `" (cadr e) + "` declared in inner scope"))) (vinfo:set-type! vi (caddr e)))))) ((lambda) (analyze-vars-lambda e env captvars sp '())) @@ -2864,7 +2864,7 @@ f(x) = yt(x) ((ssavalue? var) `(= ,var ,rhs0)) (else - (error (string "invalid assignment location \"" (deparse var) "\""))))) + (error (string "invalid assignment location `" (deparse var) "`"))))) (define (rename-sig-types ex namemap) (pattern-replace @@ -3299,8 +3299,8 @@ f(x) = yt(x) (caddr e)))) (if (has? namemap s) #f - (error (string "local variable " s - " cannot be used in closure declaration"))) + (error (string "local variable `" s + "` cannot be used in closure declaration"))) #t) #f))) (caddr e) @@ -3554,11 +3554,11 @@ f(x) = yt(x) (define (check-top-level e) (define (head-to-text h) (case h - ((abstract_type) "\"abstract type\"") - ((primitive_type) "\"primitive type\"") - ((struct_type) "\"struct\"") + ((abstract_type) "`abstract type`") + ((primitive_type) "`primitive type`") + ((struct_type) "`struct`") ((method) "method definition") - (else (string "\"" h "\"")))) + (else (string "`" h "`")))) (if (not (null? (cadr lam))) (error (string (head-to-text (car e)) " expression not at top level")))) ;; evaluate the arguments of a call, creating temporary locations as needed @@ -3790,7 +3790,7 @@ f(x) = yt(x) ((label symboliclabel) (if (eq? (car e) 'symboliclabel) (if (has? label-nesting (cadr e)) - (error (string "label \"" (cadr e) "\" defined multiple times")) + (error (string "label `" (cadr e) "` defined multiple times")) (put! label-nesting (cadr e) (list handler-level catch-token-stack)))) (let ((m (get label-map (cadr e) #f))) (if m @@ -3881,7 +3881,7 @@ f(x) = yt(x) (emit e) #f)) ((global) ; keep global declarations as statements - (if value (error "misplaced \"global\" declaration")) + (if value (error "misplaced `global` declaration")) (let ((vname (cadr e))) (if (var-info-for vname vi) ;; issue #7264 (error (string "`global " vname "`: " vname " is a local variable in its enclosing scope")) @@ -3905,7 +3905,7 @@ f(x) = yt(x) ((method) (if (not (null? (cadr lam))) (error (string "Global method definition" (linenode-string current-loc) - " needs to be placed at the top level, or use \"eval\"."))) + " needs to be placed at the top level, or use `eval`."))) (if (length> e 2) (let* ((sig (let ((sig (compile (caddr e) break-labels #t #f))) (if (valid-ir-argument? sig) @@ -3995,7 +3995,7 @@ f(x) = yt(x) ((error) (error (cadr e))) (else - (error (string "invalid syntax " (deparse e))))))) + (error (string "invalid syntax `" (deparse e) "`")))))) ;; introduce new slots for assigned arguments (for-each (lambda (v) (if (vinfo:asgn v) @@ -4012,10 +4012,10 @@ f(x) = yt(x) (lab (cadddr x))) (let ((target-nesting (get label-nesting lab #f))) (if (not target-nesting) - (error (string "label \"" lab "\" referenced but not defined"))) + (error (string "label `" lab "` referenced but not defined"))) (let ((target-level (car target-nesting))) (cond ((> target-level hl) - (error (string "cannot goto label \"" lab "\" inside try/catch block"))) + (error (string "cannot goto label `" lab "` inside try/catch block"))) ((= target-level hl) (set-cdr! point (cddr point))) ;; remove empty slot (else diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 6481f25a0e315..b13730e42395f 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -353,7 +353,7 @@ end @Expr(:error, "unexpected semicolon in array expression") [a=b,c] - @Expr(:error, "misplaced assignment statement in \"[a = b, c]\"") + @Expr(:error, "misplaced assignment statement in `[a = b, c]`") end @testset_desugar "Array Concatenation" begin @@ -377,16 +377,16 @@ end Top.typed_hvcat(T, Core.tuple(2,1), a, b, c) [a b=c] - @Expr(:error, "misplaced assignment statement in \"[a b = c]\"") + @Expr(:error, "misplaced assignment statement in `[a b = c]`") [a; b=c] - @Expr(:error, "misplaced assignment statement in \"[a; b = c]\"") + @Expr(:error, "misplaced assignment statement in `[a; b = c]`") T[a b=c] - @Expr(:error, "misplaced assignment statement in \"T[a b = c]\"") + @Expr(:error, "misplaced assignment statement in `T[a b = c]`") T[a; b=c] - @Expr(:error, "misplaced assignment statement in \"T[a; b = c]\"") + @Expr(:error, "misplaced assignment statement in `T[a; b = c]`") end @testset_desugar "Tuples" begin @@ -396,7 +396,7 @@ end (x=a,y=b) Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) - # Why do we allow this form? + # Expr(:parameters) version also works (;x=a,y=b) Core.apply_type(Core.NamedTuple, Core.tuple(:x, :y))(Core.tuple(a, b)) @@ -486,10 +486,10 @@ end $(Expr(:call, :(>:), :a, :b)) $(Expr(:$, :x)) - @Expr(:error, "\"\$\" expression outside quote") + @Expr(:error, "`\$` expression outside quote") x... - @Expr(:error, "\"...\" expression outside call") + @Expr(:error, "`...` expression outside call") end @testset_desugar "Dot syntax; broadcast/getproperty" begin @@ -553,9 +553,6 @@ end f(i, j, v..., k) Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) - x... - @Expr(:error, "\"...\" expression outside call") - # keyword arguments f(x, a=1) begin @@ -622,7 +619,7 @@ end end (x+y) += 1 - @Expr(:error, "invalid assignment location \"(x + y)\"") + @Expr(:error, "invalid assignment location `(x + y)`") end @testset_desugar "Assignment" begin @@ -725,29 +722,29 @@ end # Invalid assignments 1 = a - @Expr(:error, "invalid assignment location \"1\"") + @Expr(:error, "invalid assignment location `1`") true = a - @Expr(:error, "invalid assignment location \"true\"") + @Expr(:error, "invalid assignment location `true`") "str" = a - @Expr(:error, "invalid assignment location \"\"str\"\"") + @Expr(:error, "invalid assignment location `\"str\"`") [x y] = c - @Expr(:error, "invalid assignment location \"[x y]\"") + @Expr(:error, "invalid assignment location `[x y]`") a[x y] = c @Expr(:error, "invalid spacing in left side of indexed assignment") a[x;y] = c - @Expr(:error, "unexpected \";\" in left side of indexed assignment") + @Expr(:error, "unexpected `;` in left side of indexed assignment") [x;y] = c - @Expr(:error, "use \"(a, b) = ...\" to assign multiple values") + @Expr(:error, "use `(a, b) = ...` to assign multiple values") # Old deprecation (6575e12ba46) x.(y)=c - @Expr(:error, "invalid syntax \"x.(y) = ...\"") + @Expr(:error, "invalid syntax `x.(y) = ...`") end @testset_desugar "Declarations" begin @@ -831,6 +828,11 @@ end end end +@testset_desugar "where to UnionAll expansion" begin + T where a <: T(x) <: b + @Expr(:error, "invalid type parameter name `T(x)`") +end + @testset_desugar "let blocks" begin # flisp: (expand-let) let x::Int @@ -1263,16 +1265,16 @@ end # Invalid function names ccall(x)=body - @Expr(:error, "invalid function name \"ccall\"") + @Expr(:error, "invalid function name `ccall`") cglobal(x)=body - @Expr(:error, "invalid function name \"cglobal\"") + @Expr(:error, "invalid function name `cglobal`") true(x)=body - @Expr(:error, "invalid function name \"true\"") + @Expr(:error, "invalid function name `true`") false(x)=body - @Expr(:error, "invalid function name \"false\"") + @Expr(:error, "invalid function name `false`") end ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) From 6dbc694301bd025dcebf87003241c8473cfc6c2c Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 3 Jul 2019 08:16:36 +1000 Subject: [PATCH 47/56] Tests for `where` expansion --- base/compiler/lowering/desugar.jl | 2 +- test/compiler/lowering.jl | 37 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index 57b5481768ef3..e6d956206f128 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -250,7 +250,7 @@ function expand_forms(ex) :(.⊻=), :(<<=), :(.<<=), :(>>=), :(.>>=), :(>>>=), :(.>>>=)) expand_todo(ex) # lower-update-op elseif head == :... - expand_todo(ex) # expand-table + throw(LoweringError("`...` expression outside call")) elseif head == :$ throw(LoweringError("`\$` expression outside quote")) elseif head == :vect diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index b13730e42395f..59a40762389c0 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -829,6 +829,43 @@ end end @testset_desugar "where to UnionAll expansion" begin + A{T} where T + @Expr(:scope_block, begin + @Expr(:local_def, T) + T = Core.TypeVar(:T) + Core.UnionAll(T, Core.apply_type(A, T)) + end) + + A{T} where T <: S + @Expr(:scope_block, begin + @Expr(:local_def, T) + T = Core.TypeVar(:T, S) + Core.UnionAll(T, Core.apply_type(A, T)) + end) + + A{T} where T >: S + @Expr(:scope_block, begin + @Expr(:local_def, T) + T = Core.TypeVar(:T, S, Core.Any) + Core.UnionAll(T, Core.apply_type(A, T)) + end) + + A{T} where S' <: T <: V' + @Expr(:scope_block, begin + @Expr(:local_def, T) + T = Core.TypeVar(:T, Top.adjoint(S), Top.adjoint(V)) + Core.UnionAll(T, Core.apply_type(A, T)) + end) + + A{T} where S <: T <: V <: W + @Expr(:error, "invalid variable expression in `where`") + + A{T} where S <: T < V + @Expr(:error, "invalid bounds in `where`") + + A{T} where S < T <: V + @Expr(:error, "invalid bounds in `where`") + T where a <: T(x) <: b @Expr(:error, "invalid type parameter name `T(x)`") end From 18378614212691584db7e67d51393ed965988486 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 7 Jul 2019 10:14:44 +1000 Subject: [PATCH 48/56] Tests for do syntax --- test/compiler/lowering.jl | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 59a40762389c0..9937f8874737d 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -567,6 +567,38 @@ end end end +@testset_desugar "Do syntax" begin + f(x) do y + body(y) + end + f(begin + local gsym1 + begin + @Expr(:method, gsym1) + @Expr(:method, gsym1, Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any), Core.svec()), @Expr(:lambda, [_self_, y], [], @Expr(:scope_block, body(y)))) + maybe_unused(gsym1) + end + end, x) + + f(x; a=1) do y + body(y) + end + begin + ssa1 = begin + local gsym1 + begin + @Expr(:method, gsym1) + @Expr(:method, gsym1, Core.svec(Core.svec(Core.Typeof(gsym1), Core.Any), Core.svec()), @Expr(:lambda, [_self_, y], [], @Expr(:scope_block, body(y)))) + maybe_unused(gsym1) + end + end + begin + ssa2 = (Core.apply_type(Core.NamedTuple, Core.tuple(:a)))(Core.tuple(1)) + (Core.kwfunc(f))(ssa2, f, ssa1, x) + end + end +end + @testset_desugar "In place update operators" begin # flisp: (lower-update-op) x += a @@ -828,7 +860,7 @@ end end end -@testset_desugar "where to UnionAll expansion" begin +@testset_desugar "where -> UnionAll expansion" begin A{T} where T @Expr(:scope_block, begin @Expr(:local_def, T) From dfe0c8dcc27213574096d6f3d3cdaf81b2aefc7c Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 7 Jul 2019 23:16:58 +1000 Subject: [PATCH 49/56] More expansion; do, let + tidying --- base/compiler/lowering/desugar.jl | 216 ++++++++++++++++++++++++------ test/compiler/lowering.jl | 2 +- 2 files changed, 173 insertions(+), 45 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index e6d956206f128..e03c85c9c1f58 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -10,37 +10,106 @@ using Core: SSAValue -#------------------------------------------------------------------------------- -# AST tools - -# :(f(arg; par)).arg s => [:f, Expr(:parameters, :par), :arg] -has_parameters(args) = length(args) >= 2 && args[2] isa Expr && args[2].head === :parameters -has_assignment(args) = any(isassignment, args) +# AST predicates +#--------------- isquoted(ex) = ex isa QuoteNode || (ex isa Expr && - ex.head in (:quote, :top, :core, :globalref, :outerref, :break, :inert, :meta)) + ex.head in (:quote, :top, :core, :globalref, + :outerref, :break, :inert, :meta)) + issymbollike(ex) = ex isa Symbol || ex isa SSAValue isassignment(ex) = ex isa Expr && ex.head == :(=) +isdecl(ex) = ex isa Expr && ex.head == :(::) + # True if `ex` is trivially free of side effects (and hence safe to repeat) iseffectfree(ex) = !(ex isa Expr) || isquoted(ex) -# FIXME: Counter Should be thread local or in expansion ctx -let ssa_index = Ref(0) - global make_ssavalue() = SSAValue(ssa_index[] += 1) +# True if `ex` is lhs of short-form function definition +# f(args...) +is_eventually_call(ex) = ex isa Expr && + (ex.head == :call || (ex.head in (:where, :(::)) && + is_eventually_call(ex.args[1]))) + +# Symbol `s` occurs in ex, excluding expression heads and quoted Exprs +function occursin_ex(s::Symbol, ex) + s === ex || (ex isa Expr && !isquoted(ex) && any(e->occursin_ex(s, e), ex.args)) +end + +# As above, but test each expression with predicate `pred`. Optionally, filter +# expressions with `filt`. +function occursin_ex(pred::Function, ex; filt=e->true) + filt(ex) && (pred(ex) || (ex isa Expr && !isquoted(ex) && + any(e->occursin_ex(pred, e, filt=filt), ex.args))) +end + +# Check for `f(args...; pars...)` syntax +has_parameters(ex::Expr) = length(ex.args) >= 2 && ex.args[2] isa Expr && + ex.args[2].head === :parameters + +# has_assignment(args) = any(isassignment, args) + +# AST matching +#------------- + +decl_var(ex) = isdecl(ex) ? ex.args[1] : ex + +# Given a complex assignment LHS, return the symbol that will ultimately be assigned to +function assigned_name(ex) + if ex isa Expr && ex.head in (:call, :curly, :where) || (ex.head == :(::) && + is_eventually_call(ex)) + assigned_name(ex.args[1]) + else + ex + end +end + +# Get list of variable names on lhs of expression +lhs_vars(ex) = lhs_vars!(Symbol[], ex) +function lhs_vars!(vars, ex) + if ex isa Symbol + push!(vars, ex) + elseif isdecl(ex) + push!(vars, decl_var(ex)) + elseif ex isa Expr && ex.head == :tuple + foreach(e->lhs_vars!(vars, e), ex.args) + end + vars +end + +# Error checking utilities +#------------------------- +struct LoweringError <: Exception + msg::String + ex end +LoweringError(msg::AbstractString) = LoweringError(msg, nothing) + +function Base.show(io::IO, err::LoweringError) + print(io, err.msg, " in `", err.ex, "`") +end + +function check_no_assignments(ex) + for e in ex.args + !isassignment(e) || throw(LoweringError("misplaced assignment statement", ex)) + end +end +error_unexpected_semicolon(ex) = throw(LoweringError("unexpected semicolon", ex)) + + +# Utilities for constructing lowered ASTs +#---------------------------------------- -top(ex) = Expr(:top, ex) topcall(head, args...) = Expr(:call, Expr(:top, head), args...) -core(ex) = Expr(:core, ex) +corecall(head, args...) = Expr(:call, Expr(:core, head), args...) blockify(ex) = ex isa Expr && ex.head !== :block ? ex : Expr(:block, ex) # TODO: null Expr? mapargs(f, ex) = ex isa Expr ? Expr(ex.head, map(f, ex.args)...) : ex -# Symbol `s` occurs in ex -occursin_ex(s::Symbol, ex::Symbol) = s === ex -occursin_ex(s::Symbol, ex::Expr) = occursin_ex(s, ex.args) -occursin_ex(s::Symbol, exs) = any(e->occursin_ex(s, e), exs) +# FIXME: Counter Should be thread local or in expansion ctx +let ssa_index = Ref(0) + global make_ssavalue() = SSAValue(ssa_index[] += 1) +end """ make_ssa_if(need_ssa, ex, stmts) @@ -57,20 +126,78 @@ function make_ssa_if(need_ssa::Bool, ex, stmts) ex end end - make_ssa_if(need_ssa::Function, ex, stmts) = make_ssa_if(need_ssa(ex), ex, stmts) -function check_no_assigments(ex) - for e in ex.args - !isassignment(e) || throw(LoweringError("misplaced assigment statement in `$ex`")) - end -end -error_unexpected_semicolon(ex) = throw(LoweringError("unexpected semicolon in `$ex`")) - #------------------------------------------------------------------------------- -struct LoweringError <: Exception - msg::AbstractString + +function expand_let(ex) + bindings = !(ex.args[1] isa Expr) ? throw(LoweringError("Invalid let syntax", ex)) : + ex.args[1].head == :block ? ex.args[1].args : [ex.args[1]] + body = isempty(bindings) ? Expr(:scope_block, blockify(ex.args[2])) : ex.args[2] + for binding in reverse(bindings) + body = + if binding isa Symbol || isdecl(binding) + # Just symbol -> add local + Expr(:scope_block, + Expr(:block, + Expr(:local, binding), + body)) + elseif binding isa Expr && binding.head == :(=) && length(binding.args) == 2 + # Some kind of assignment + lhs = binding.args[1] + rhs = binding.args[2] + if is_eventually_call(lhs) + # f() = c + error("TODO") # Needs expand_function to be implemented + elseif lhs isa Symbol || isdecl(lhs) + # `x = c` or `x::T = c` + varname = decl_var(lhs) + if occursin_ex(varname, rhs) + tmp = make_ssavalue() + Expr(:scope_block, + Expr(:block, + Expr(:(=), tmp, rhs), + Expr(:scope_block, + Expr(:block, + Expr(:local_def, lhs), + Expr(:(=), varname, tmp), + body)))) + else + Expr(:scope_block, + Expr(:block, + Expr(:local_def, lhs), + Expr(:(=), varname, rhs), + body)) + end + elseif lhs isa Expr && lhs.head == :tuple + # (a, b, c, ...) = rhs + vars = lhs_vars(lhs) + if occursin_ex(e->e isa Symbol && e in vars, rhs) + tmp = make_ssavalue() + Expr(:scope_block, + Expr(:block, + Expr(:(=), tmp, rhs), + Expr(:scope_block, + Expr(:block, + [Expr(:local_def, v) for v in vars]..., + Expr(:(=), lhs, tmp), + body)))) + else + Expr(:scope_block, + Expr(:block, + [Expr(:local_def, v) for v in vars]..., + binding, + body)) + end + else + throw(LoweringError("invalid binding in let syntax", binding)) + end + else + throw(LoweringError("invalid binding in let syntax", binding)) + end + end + expand_forms(body) end """ @@ -169,7 +296,7 @@ function expand_forms(ex) elseif head == :-> expand_todo(ex) # expand-arrow elseif head == :let - expand_todo(ex) # expand-let + expand_let(ex) elseif head == :macro expand_todo(ex) # expand-macro-def elseif head == :struct @@ -191,7 +318,7 @@ function expand_forms(ex) elseif head == :where expand_todo(ex) # expand-wheres elseif head == :const - expand_todo(ex) # expand-const-decl + expand_todo(ex) elseif head == :local expand_todo(ex) # expand-local-or-global-decl elseif head == :global @@ -207,23 +334,24 @@ function expand_forms(ex) elseif head == :comparison expand_todo(ex) # expand-compare-chain elseif head == :ref - !has_parameters(args) || error_unexpected_semicolon(ex) + !has_parameters(ex) || error_unexpected_semicolon(ex) expand_forms(partially_expand_ref(ex)) elseif head == :curly expand_todo(ex) # expand-table elseif head == :call expand_todo(ex) # expand-table elseif head == :do - expand_todo(ex) # expand-table + callex = args[1] + anonfunc = args[2] + expand_forms(has_parameters(callex) ? + Expr(:call, callex.args[1], callex.args[2], anonfunc, callex.args[3:end]...) : + Expr(:call, callex.args[1], anonfunc, callex.args[2:end]...) + ) elseif head == :tuple # TODO: NamedTuple lower-named-tuple - #if has_parameters(args) + #if has_parameters(ex) #end - expand_forms(Expr(:call, core(:tuple), args...)) - elseif head == :braces - expand_todo(ex) # expand-table - elseif head == :bracescat - expand_todo(ex) # expand-table + expand_forms(corecall(:tuple, args...)) elseif head == :string expand_forms(topcall(:string, args...)) elseif head == :(::) @@ -250,28 +378,28 @@ function expand_forms(ex) :(.⊻=), :(<<=), :(.<<=), :(>>=), :(.>>=), :(>>>=), :(.>>>=)) expand_todo(ex) # lower-update-op elseif head == :... - throw(LoweringError("`...` expression outside call")) + throw(LoweringError("`...` expression outside call", ex)) elseif head == :$ - throw(LoweringError("`\$` expression outside quote")) + throw(LoweringError("`\$` expression outside quote", ex)) elseif head == :vect - !has_parameters(args) || error_unexpected_semicolon(ex) - check_no_assigments(ex) + !has_parameters(ex) || error_unexpected_semicolon(ex) + check_no_assignments(ex) expand_forms(topcall(:vect, args...)) elseif head == :hcat - check_no_assigments(ex) + check_no_assignments(ex) expand_forms(topcall(:hcat, args...)) elseif head == :vcat - check_no_assigments(ex) + check_no_assignments(ex) if any(e->e isa Expr && e.head == :row, args) expand_hvcat(ex) else expand_forms(topcall(:vcat, args...)) end elseif head == :typed_hcat - check_no_assigments(ex) + check_no_assignments(ex) expand_forms(topcall(:typed_hcat, args...)) elseif head == :typed_vcat - check_no_assigments(ex) + check_no_assignments(ex) if any(e->e isa Expr && e.head == :row, args) expand_hvcat(ex) else diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 9937f8874737d..f1c6ff92667da 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -20,7 +20,7 @@ function _expand_forms(ex) expand_forms(ex) catch exc exc isa LoweringError || rethrow() - return Expr(:error, exc.msg) + return Expr(:error, sprint(show, exc)) end end end From c2a1b1ab6c382c5eb99c6043e3ed8a84daa0d71f Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Thu, 18 Jul 2019 21:33:26 +1000 Subject: [PATCH 50/56] Simplify flisp lowering of || --- src/julia-syntax.scm | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index db52f709d5fe6..62a89a8b442ac 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1524,13 +1524,9 @@ 'false (if (null? (cdr tail)) (car tail) - (if (symbol-like? (car tail)) - `(if ,(car tail) ,(car tail) - ,(loop (cdr tail))) - (let ((g (make-ssavalue))) - `(block (= ,g ,(car tail)) - (if ,g ,g - ,(loop (cdr tail))))))))))) + `(if ,(car tail) + true + ,(loop (cdr tail)))))))) (define (expand-for lhss itrs body) (define (outer? x) (and (pair? x) (eq? (car x) 'outer))) From 652e1c1ab2141bec72642455dbae20587c8c1b97 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sun, 14 Jul 2019 22:54:11 +1000 Subject: [PATCH 51/56] Expansion of && and || --- base/compiler/lowering/desugar.jl | 36 +++++++++++++++++++++++++++++-- test/compiler/lowering.jl | 27 +++++++++++++++++++++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index e03c85c9c1f58..a996c597dda14 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -277,6 +277,38 @@ function expand_hvcat(ex) end end +# Flatten nested Expr(head, args) with depth first traversal of args. +function flatten_ex_args!(args, head, ex) + if ex isa Expr && ex.head == head + for a in ex.args + flatten_ex_args!(args, head, a) + end + else + push!(args, ex) + end + args +end + +function expand_and(ex) + args = flatten_ex_args!([], :&&, ex) + @assert length(args) > 1 + e = args[end] + for i = length(args)-1:-1:1 + e = Expr(:if, args[i], e, false) + end + e +end + +function expand_or(ex) + args = flatten_ex_args!([], :||, ex) + @assert length(args) > 1 + e = args[end] + for i = length(args)-1:-1:1 + e = Expr(:if, args[i], true, e) + end + e +end + #------------------------------------------------------------------------------- # Expansion entry point @@ -369,9 +401,9 @@ function expand_forms(ex) elseif head == :for expand_todo(ex) # expand-for elseif head == :&& - expand_todo(ex) # expand-and + expand_forms(expand_and(ex)) elseif head == :|| - expand_todo(ex) # expand-or + expand_forms(expand_or(ex)) elseif head in (:(+=), :(-=), :(*=), :(.*=), :(/=), :(./=), :(//=), :(.//=), :(\=), :(.\=), :(.+=), :(.-=), :(^=), :(.^=), :(÷=), :(.÷=), :(%=), :(.%=), :(|=), :(.|=), :(&=), :(.&=), :($=), :(⊻=), diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index f1c6ff92667da..f099ac29924c3 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -457,20 +457,43 @@ end end @testset_desugar "Short circuit boolean operators" begin - # flisp: (expand-or) (expand-and) + # flisp: (expand-or) a || b if a - a + true else b end + f(a) || f(b) || f(c) + if f(a) + true + else + if f(b) + true + else + f(c) + end + end + + # flisp: (expand-and) a && b if a b else false end + + f(a) && f(b) && f(c) + if f(a) + if f(b) + f(c) + else + false + end + else + false + end end @testset_desugar "Misc operators" begin From 19c88c77fb6474119370c6f6ba31bb22bfb05c31 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Fri, 19 Jul 2019 21:39:16 +1000 Subject: [PATCH 52/56] Errors for braces notation --- base/compiler/lowering/desugar.jl | 9 ++++++++- test/compiler/lowering.jl | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index a996c597dda14..84ab417ce84fd 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -87,7 +87,10 @@ end LoweringError(msg::AbstractString) = LoweringError(msg, nothing) function Base.show(io::IO, err::LoweringError) - print(io, err.msg, " in `", err.ex, "`") + print(io, err.msg) + if err.ex !== nothing + print(io, " in `", err.ex, "`") + end end function check_no_assignments(ex) @@ -384,6 +387,10 @@ function expand_forms(ex) #if has_parameters(ex) #end expand_forms(corecall(:tuple, args...)) + elseif head == :braces + throw(LoweringError("{ } vector syntax is discontinued", ex)) + elseif head == :bracescat + throw(LoweringError("{ } matrix syntax is discontinued", ex)) elseif head == :string expand_forms(topcall(:string, args...)) elseif head == :(::) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index f099ac29924c3..fb43a04607806 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -513,6 +513,12 @@ end x... @Expr(:error, "`...` expression outside call") + + {a, b} + @Expr(:error, "{ } vector syntax is discontinued") + + {a; b} + @Expr(:error, "{ } matrix syntax is discontinued") end @testset_desugar "Dot syntax; broadcast/getproperty" begin @@ -1087,7 +1093,6 @@ end body3 end)))) - for i = a body1 continue From 4d8e45660b576d973972cdd93f24164c05903f21 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Fri, 19 Jul 2019 22:21:08 +1000 Subject: [PATCH 53/56] Tests for desugaring of try --- test/compiler/lowering.jl | 111 +++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 1 deletion(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index fb43a04607806..769d23c16468a 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -20,7 +20,9 @@ function _expand_forms(ex) expand_forms(ex) catch exc exc isa LoweringError || rethrow() - return Expr(:error, sprint(show, exc)) + # Hack: show only msg for more compatibility with flisp error forms + return Expr(:error, exc.msg) + #return Expr(:error, sprint(show, exc)) end end end @@ -1073,6 +1075,14 @@ end @Expr(:error, "invalid let syntax") end +@testset_desugar "Blocks" begin + $(Expr(:block)) + $nothing + + $(Expr(:block, :a)) + a +end + @testset_desugar "Loops" begin # flisp: (expand-for) (lambda in expand-forms) while cond' @@ -1159,6 +1169,105 @@ end end) end +@testset_desugar "Try blocks" begin + # flisp: expand-try + try + a + catch + b + end + @Expr(:trycatch, + @Expr(:scope_block, begin a end), + @Expr(:scope_block, begin b end)) + + try + a + catch exc + b + end + @Expr(:trycatch, + @Expr(:scope_block, begin a end), + @Expr(:scope_block, + begin + exc = @Expr(:the_exception) + b + end)) + + try + catch exc + end + @Expr(:trycatch, + @Expr(:scope_block, $nothing), + @Expr(:scope_block, begin + exc = @Expr(:the_exception) + begin + end + end)) + + try + a + finally + b + end + @Expr(:tryfinally, + @Expr(:scope_block, begin a end), + @Expr(:scope_block, begin b end)) + + try + a + catch + b + finally + c + end + @Expr(:tryfinally, + @Expr(:trycatch, + @Expr(:scope_block, begin a end), + @Expr(:scope_block, begin b end)), + @Expr(:scope_block, begin c end)) + + # goto with label anywhere within try block is ok + try + begin + let + $(Expr(:symbolicgoto, :x)) # @goto x + end + end + begin + $(Expr(:symboliclabel, :x)) # @label x + end + finally + end + @Expr(:tryfinally, + @Expr(:scope_block, + begin + begin + @Expr(:scope_block, + @Expr(:symbolicgoto, x)) + end + begin + @Expr(:symboliclabel, x) + end + end), + @Expr(:scope_block, + begin + end)) + + # goto not allowed without associated label in try/finally + try + begin + let + $(Expr(:symbolicgoto, :x)) # @goto x + end + end + finally + end + @Expr(:error, "goto from a try/finally block is not permitted") + + $(Expr(:try, :a, :b, :c, :d, :e)) + @Expr(:error, "invalid `try` form") +end + @testset_desugar "Functions" begin # Short form f(x) = body(x) From 3c0818563552b18a8486cb61ffe1ac53b09a7bfd Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 20 Jul 2019 09:40:12 +1000 Subject: [PATCH 54/56] Lowering of try --- base/compiler/lowering/desugar.jl | 61 +++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/base/compiler/lowering/desugar.jl b/base/compiler/lowering/desugar.jl index 84ab417ce84fd..c126bd36f821c 100644 --- a/base/compiler/lowering/desugar.jl +++ b/base/compiler/lowering/desugar.jl @@ -134,6 +134,55 @@ make_ssa_if(need_ssa::Function, ex, stmts) = make_ssa_if(need_ssa(ex), ex, stmts #------------------------------------------------------------------------------- +function find_symbolic_labels!(labels, gotos, ex) + if ex isa Expr + if ex.head == :symboliclabel + push!(labels, ex.args[1]) + elseif ex.head == :symbolicgoto + push!(gotos, ex.args[1]) + elseif !isquoted(ex) + for arg in ex.args + find_symbolic_labels!(labels, gotos, arg) + end + end + end +end + +function has_unmatched_symbolic_goto(ex) + labels = Set{Symbol}() + gotos = Set{Symbol}() + find_symbolic_labels!(labels, gotos, ex) + !all(target in labels for target in gotos) +end + +function expand_try(ex) + if length(ex.args) < 3 || length(ex.args) > 4 + throw(LoweringError("invalid `try` form", ex)) + end + try_block = ex.args[1] + exc_var = ex.args[2] + catch_block = ex.args[3] + finally_block = length(ex.args) < 4 ? false : ex.args[4] + if has_unmatched_symbolic_goto(try_block) + throw(LoweringError("goto from a try/finally block is not permitted", ex)) + end + if exc_var !== false + catch_block = Expr(:block, + Expr(:(=), exc_var, Expr(:the_exception)), + catch_block) + end + trycatch = catch_block !== false ? + Expr(:trycatch, + Expr(:scope_block, try_block), + Expr(:scope_block, catch_block)) : + Expr(:scope_block, try_block) + lowered = finally_block !== false ? + Expr(:tryfinally, + trycatch, + Expr(:scope_block, finally_block)) : trycatch + expand_forms(lowered) +end + function expand_let(ex) bindings = !(ex.args[1] isa Expr) ? throw(LoweringError("Invalid let syntax", ex)) : ex.args[1].head == :block ? ex.args[1].args : [ex.args[1]] @@ -152,7 +201,7 @@ function expand_let(ex) rhs = binding.args[2] if is_eventually_call(lhs) # f() = c - error("TODO") # Needs expand_function to be implemented + Expr(:scope_block, body) # FIXME Needs expand_function to be implemented elseif lhs isa Symbol || isdecl(lhs) # `x = c` or `x::T = c` varname = decl_var(lhs) @@ -337,11 +386,17 @@ function expand_forms(ex) elseif head == :struct expand_todo(ex) # expand-struct-def elseif head == :try - expand_todo(ex) # expand-try + expand_try(ex) elseif head == :lambda expand_todo(ex) # expand-table elseif head == :block - expand_todo(ex) # expand-table + if length(args) == 0 + nothing + elseif length(args) == 1 && !(args[1] isa LineNumberNode) + expand_forms(args[1]) + else + Expr(:block, map(expand_forms, args)...) + end elseif head == :. expand_todo(ex) # expand-fuse-broadcast elseif head == :.= From fafee6f55e4283a44893821fe30e91865f01e69f Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 20 Jul 2019 11:48:50 +1000 Subject: [PATCH 55/56] Arrange tests by input Expr head --- test/compiler/lowering.jl | 118 +++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 59 deletions(-) diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 769d23c16468a..3159e8b2ebb37 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -277,11 +277,11 @@ macro testset_desugar(name, block) end #------------------------------------------------------------------------------- -# Tests +# Tests, organized by the Expr head which needs to be lowered. @testset "Lowering" begin -@testset_desugar "Index notation; getindex" begin +@testset_desugar "ref end" begin # Indexing a[i] Top.getindex(a, i) @@ -343,14 +343,11 @@ end end end -@testset_desugar "Array Literals" begin +@testset_desugar "vect" begin # flisp: (in expand-table) [a,b] Top.vect(a,b) - T[a,b] - Top.getindex(T, a,b) # Only so much syntax to go round :-/ - [a,b;c] @Expr(:error, "unexpected semicolon in array expression") @@ -358,7 +355,7 @@ end @Expr(:error, "misplaced assignment statement in `[a = b, c]`") end -@testset_desugar "Array Concatenation" begin +@testset_desugar "hcat vcat hvcat" begin # flisp: (lambda in expand-table) [a b] Top.hcat(a,b) @@ -391,7 +388,7 @@ end @Expr(:error, "misplaced assignment statement in `T[a; b = c]`") end -@testset_desugar "Tuples" begin +@testset_desugar "tuple" begin (x,y) Core.tuple(x,y) @@ -407,7 +404,7 @@ end @Expr(:error, "unexpected semicolon in tuple") end -@testset_desugar "Comparison chains" begin +@testset_desugar "comparison" begin # flisp: (expand-compare-chain) a < b < c if a < b @@ -458,7 +455,7 @@ end Top.broadcasted(<, ssa1, d))) end -@testset_desugar "Short circuit boolean operators" begin +@testset_desugar "|| &&" begin # flisp: (expand-or) a || b if a @@ -498,7 +495,7 @@ end end end -@testset_desugar "Misc operators" begin +@testset_desugar "' <: :>" begin a' Top.adjoint(a) @@ -510,6 +507,9 @@ end a >: b $(Expr(:call, :(>:), :a, :b)) +end + +@testset_desugar "\$ ... {}" begin $(Expr(:$, :x)) @Expr(:error, "`\$` expression outside quote") @@ -523,7 +523,7 @@ end @Expr(:error, "{ } matrix syntax is discontinued") end -@testset_desugar "Dot syntax; broadcast/getproperty" begin +@testset_desugar ". .=" begin # flisp: (expand-fuse-broadcast) # Property access @@ -573,13 +573,11 @@ end Top.materialize!(x, Top.broadcasted(+, x, a)) end -@testset_desugar "Function call syntax" begin +@testset_desugar "call" begin # zero arg call g[i]() Top.getindex(g, i)() - # ccall - # splatting f(i, j, v..., k) Core._apply(f, Core.tuple(i,j), v, Core.tuple(k)) @@ -598,7 +596,10 @@ end end end -@testset_desugar "Do syntax" begin + # ccall + + +@testset_desugar "do" begin f(x) do y body(y) end @@ -630,7 +631,7 @@ end end end -@testset_desugar "In place update operators" begin +@testset_desugar "+= .+= etc" begin # flisp: (lower-update-op) x += a x = x+a @@ -685,7 +686,7 @@ end @Expr(:error, "invalid assignment location `(x + y)`") end -@testset_desugar "Assignment" begin +@testset_desugar "=" begin # flisp: (lambda in expand-table) # property notation @@ -783,6 +784,37 @@ end maybe_unused(a) end + # type decl + x::T = a + begin + @Expr :decl x T + x = a + end + + # type aliases + A{T} = B{T} + begin + @Expr :const_if_global A + A = @Expr(:scope_block, + begin + @Expr :local_def T + T = Core.TypeVar(:T) + Core.UnionAll(T, Core.apply_type(B, T)) + end) + end + + # Short form function definitions + f(x) = body(x) + begin + @Expr(:method, f) + @Expr(:method, f, + Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), + @Expr(:lambda, [_self_, x], [], + @Expr(:scope_block, + body(x)))) + maybe_unused(f) + end + # Invalid assignments 1 = a @Expr(:error, "invalid assignment location `1`") @@ -810,9 +842,8 @@ end @Expr(:error, "invalid syntax `x.(y) = ...`") end -@testset_desugar "Declarations" begin +@testset_desugar "const local global" begin # flisp: (expand-decls) (expand-local-or-global-decl) (expand-const-decl) - # const const x=a begin @@ -870,28 +901,9 @@ end global x y = a end - - # type decl - x::T = a - begin - @Expr :decl x T - x = a - end - - # type aliases - A{T} = B{T} - begin - @Expr :const_if_global A - A = @Expr(:scope_block, - begin - @Expr :local_def T - T = Core.TypeVar(:T) - Core.UnionAll(T, Core.apply_type(B, T)) - end) - end end -@testset_desugar "where -> UnionAll expansion" begin +@testset_desugar "where" begin A{T} where T @Expr(:scope_block, begin @Expr(:local_def, T) @@ -933,7 +945,7 @@ end @Expr(:error, "invalid type parameter name `T(x)`") end -@testset_desugar "let blocks" begin +@testset_desugar "let" begin # flisp: (expand-let) let x::Int body @@ -1075,7 +1087,7 @@ end @Expr(:error, "invalid let syntax") end -@testset_desugar "Blocks" begin +@testset_desugar "block" begin $(Expr(:block)) $nothing @@ -1083,7 +1095,7 @@ end a end -@testset_desugar "Loops" begin +@testset_desugar "while for" begin # flisp: (expand-for) (lambda in expand-forms) while cond' body1' @@ -1169,7 +1181,7 @@ end end) end -@testset_desugar "Try blocks" begin +@testset_desugar "try catch finally" begin # flisp: expand-try try a @@ -1268,19 +1280,7 @@ end @Expr(:error, "invalid `try` form") end -@testset_desugar "Functions" begin - # Short form - f(x) = body(x) - begin - @Expr(:method, f) - @Expr(:method, f, - Core.svec(Core.svec(Core.Typeof(f), Core.Any), Core.svec()), - @Expr(:lambda, [_self_, x], [], - @Expr(:scope_block, - body(x)))) - maybe_unused(f) - end - +@testset_desugar "function" begin # Long form with argument annotations function f(x::T, y) body(x) @@ -1484,7 +1484,7 @@ end end ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) -@testset_desugar "Generated function; optionally generated" begin +@testset_desugar "@generated function" begin function f(x) body1(x) if $(Expr(:generated)) @@ -1547,7 +1547,7 @@ ln = LineNumberNode(@__LINE__()+3, Symbol(@__FILE__)) end end -@testset_desugar "Macros" begin +@testset_desugar "macro" begin macro foo end @Expr(:method, $(Symbol("@foo"))) From df61138fcf97d03dcbbba10e962571af9700db56 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 20 Jul 2019 14:40:43 +1000 Subject: [PATCH 56/56] Move lowering test tools into their own file --- test/compiler/lowering.jl | 286 +------------------------------- test/compiler/lowering_tools.jl | 277 +++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 282 deletions(-) create mode 100644 test/compiler/lowering_tools.jl diff --git a/test/compiler/lowering.jl b/test/compiler/lowering.jl index 3159e8b2ebb37..4a66a314d1113 100644 --- a/test/compiler/lowering.jl +++ b/test/compiler/lowering.jl @@ -1,284 +1,6 @@ -using Core: SSAValue -using Base: remove_linenums! - -# Call into lowering stage 1; syntax desugaring -function fl_expand_forms(ex) - ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "jl-expand-forms", ex, Main) -end - -include("../../base/compiler/lowering/desugar.jl") - -# Make it easy to replace fl_expand_forms with a julia version in the future. -if !isdefined(@__MODULE__, :use_flisp) - use_flisp = true -end -function _expand_forms(ex) - if use_flisp - fl_expand_forms(ex) - else - try - expand_forms(ex) - catch exc - exc isa LoweringError || rethrow() - # Hack: show only msg for more compatibility with flisp error forms - return Expr(:error, exc.msg) - #return Expr(:error, sprint(show, exc)) - end - end -end - -function lift_lowered_expr!(ex, nextids, valmap, lift_full) - if ex isa SSAValue - # Rename SSAValues into renumbered symbols - return get!(valmap, ex) do - newid = nextids[1] - nextids[1] = newid+1 - Symbol("ssa$newid") - end - end - if ex isa Symbol - if ex == Symbol("#self#") - return :_self_ - end - # Rename gensyms - name = string(ex) - if startswith(name, "#") - return get!(valmap, ex) do - newid = nextids[2] - nextids[2] = newid+1 - Symbol("gsym$newid") - end - end - end - if ex isa Expr - if ex.head == :block && length(ex.args) == 1 - # Remove trivial blocks - return lift_lowered_expr!(ex.args[1], nextids, valmap, lift_full) - end - map!(ex.args, ex.args) do e - lift_lowered_expr!(e, nextids, valmap, lift_full) - end - if lift_full - # Lift exotic Expr heads into standard julia syntax for ease in - # writing test case expressions. - if ex.head == :top || ex.head == :core - # Special global refs renamed to look like modules - newhead = ex.head == :top ? :Top : :Core - return Expr(:(.), newhead, QuoteNode(ex.args[1])) - elseif ex.head == :unnecessary - # `unnecessary` marks expressions generated by lowering that - # do not need to be evaluated if their value is unused. - return Expr(:call, :maybe_unused, ex.args...) - #= - elseif ex.head in [:_while, :_do_while, :scope_block, :break_block, - :break, :local_def, :require_existing_local] - return Expr(:macrocall, Symbol("@Expr"), nothing, - QuoteNode(ex.head), ex.args...) - =# - end - end - elseif ex isa Vector # Occasional case of lambdas - map!(ex, ex) do e - lift_lowered_expr!(e, nextids, valmap, lift_full) - end - end - return ex -end - -""" -Clean up an `Expr` into an equivalent form which can be easily entered by -hand - -* Replacing `SSAValue(id)` with consecutively numbered symbols :ssa\$i -* Remove trivial blocks -""" -function lift_lowered_expr(ex; lift_full=false) - valmap = Dict{Union{Symbol,SSAValue},Symbol}() - lift_lowered_expr!(remove_linenums!(deepcopy(ex)), ones(Int,2), valmap, lift_full) -end - -""" -Very slight lowering of reference expressions to allow comparison with -desugared forms. - -* Remove trivial blocks -* Translate psuedo-module expressions Top.x and Core.x to Expr(:top) and - Expr(:core) -""" -function lower_ref_expr!(ex) - if ex isa Expr - map!(lower_ref_expr!, ex.args, ex.args) - if ex.head == :block && length(ex.args) == 1 - # Remove trivial blocks - return lower_ref_expr!(ex.args[1]) - end - # Translate a selection of special expressions into the exotic Expr - # heads used in lowered code. - if ex.head == :(.) && length(ex.args) >= 1 && (ex.args[1] == :Top || - ex.args[1] == :Core) - if !(length(ex.args) == 2 && ex.args[2] isa QuoteNode) - throw("Unexpected top/core expression $(sprint(dump, ex))") - end - return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) - elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused - return Expr(:unnecessary, ex.args[2:end]...) - elseif ex.head == :macrocall && ex.args[1] == Symbol("@Expr") - head = ex.args[3] - head isa QuoteNode || throw(ArgumentError("`head` argument to @Expr should be quoted")) - return Expr(head.value, ex.args[4:end]...) - end - end - return ex -end -lower_ref_expr(ex) = lower_ref_expr!(remove_linenums!(deepcopy(ex))) - - -function diffdump(io::IOContext, ex1, ex2, n, prefix, indent) - if ex1 == ex2 - isempty(prefix) || print(io, prefix) - dump(io, ex1, 2, indent) - else - if ex1 isa Expr && ex2 isa Expr && ex1.head == ex2.head && length(ex1.args) == length(ex2.args) - isempty(prefix) || print(io, prefix) - println(io, "Expr") - println(io, indent, " head: ", ex1.head) - println(io, indent, " args: Array{Any}(", size(ex1.args), ")") - for i in 1:length(ex1.args) - prefix = string(indent, " ", i, ": ") - diffdump(io, ex1.args[i], ex2.args[i], 4, prefix, string(" ", indent)) - i < length(ex1.args) && println(io) - end - else - printstyled(io, string(prefix, sprint(dump, ex1, 4, indent; context=io)), color=:green) - println() - printstyled(io, string(prefix, sprint(dump, ex2, 4, indent; context=io)), color=:red) - end - end -end - -""" -Display colored differences between two expressions `ex1` and `ex2` using the -`dump` format. -""" -function diffdump(ex1, ex2; maxdepth=20) - mod = get(stdout, :module, Main) - diffdump(IOContext(stdout, :limit => true, :module => mod), ex1, ex2, maxdepth, "", "") - println(stdout) -end - -# For interactive convenience in constructing test cases with flisp based lowering -function desugar(ex; lift=:full) - expanded = _expand_forms(ex) - if lift == :full || lift == :partial - lift_lowered_expr(expanded; lift_full=(lift == :full)) - else - expanded - end -end - -# Macro for producing exotic `Expr`s found in lowered ASTs -# -# Note that this is provided for convenience/reference but in practice we -# expand it "manually" inside lower_ref_expr! -macro Expr(head, args...) - head isa QuoteNode || throw(ArgumentError("`head` argument to @Expr should be quoted")) - esc(Expr(head.value, args...)) -end - -""" - @desugar ex [kws...] - -Convenience macro, equivalent to `desugar(:(ex), kws...)`. -""" -macro desugar(ex, kws...) - quote - desugar($(Expr(:quote, ex)); $(map(esc, kws)...)) - end -end - -# Convenience macro to print code for a test case -macro maketest(ex) - Base.remove_linenums!(ex) - quote - print($(QuoteNode(ex)), "\n") - print(desugar($(Expr(:quote, ex))), "\n") - end -end - -""" - @testset_desugar(name, exprs) - -Test that a set of expressions lower correctly to desugared AST form. This -creates a new `@testset` with the given `name`. The statements in the block -`exprs` are interpreted as a flat list of any number of `input_expr`, -`ref_expr` pairs, where `input_expr` should be transformed by lowering into -`ref_expr` by the desugaring pass. For example, - -``` -@testset_desugar "Property notation" begin - # flisp: (expand-fuse-broadcast) - a.b - Top.getproperty(a, :b) - - a.b.c - Top.getproperty(Top.getproperty(a, :b), :c) -end -``` -""" -macro testset_desugar(name, block) - if !(block isa Expr && block.head == :block) - throw(ArgumentError("@testset_desugar requires a block as the second argument")) - end - loc = nothing - tests = [] - i = 1 - while i <= length(block.args) - if block.args[i] isa LineNumberNode - loc = block.args[i] - i += 1 - continue - end - exs = [] - while i <= length(block.args) && length(exs) < 2 - if !(block.args[i] isa LineNumberNode) - push!(exs, block.args[i]) - end - i += 1 - end - if length(exs) == 0 - break - end - if length(exs) == 1 - throw(ArgumentError("List of expressions to @testset_desugar must consist of input,ref pairs")) - end - input = exs[1] - ref = exs[2] - ex = quote - expanded = lift_lowered_expr(_expand_forms($(Expr(:quote, input)))) - reference = lower_ref_expr($(Expr(:quote, ref))) - @test expanded == reference - if expanded != reference - # Kinda crude. Would be much neater if Test supported custom/more - # capable diffing for failed tests. - println("Diff dump:") - diffdump(expanded, reference) - end - end - # Attribute the test to the correct line number - @assert ex.args[6].args[1] == Symbol("@test") - ex.args[6].args[2] = loc - push!(tests, ex) - end - quote - @testset $name begin - $(tests...) - end - end -end - -#------------------------------------------------------------------------------- -# Tests, organized by the Expr head which needs to be lowered. +include("lowering_tools.jl") +# Tests are organized by the Expr head which needs to be lowered. @testset "Lowering" begin @testset_desugar "ref end" begin @@ -596,8 +318,8 @@ end end end - # ccall - +@testset_desugar "ccall" begin +end @testset_desugar "do" begin f(x) do y diff --git a/test/compiler/lowering_tools.jl b/test/compiler/lowering_tools.jl new file mode 100644 index 0000000000000..59301a22208cb --- /dev/null +++ b/test/compiler/lowering_tools.jl @@ -0,0 +1,277 @@ +using Core: SSAValue +using Base: remove_linenums! + +# Call into lowering stage 1; syntax desugaring +function fl_expand_forms(ex) + ccall(:jl_call_scm_on_ast_formonly, Any, (Cstring, Any, Any), "jl-expand-forms", ex, Main) +end + +include("../../base/compiler/lowering/desugar.jl") + +# Make it easy to swap fl_expand_forms with the julia version. +if !isdefined(@__MODULE__, :use_flisp) + use_flisp = true +end +function _expand_forms(ex) + if use_flisp + fl_expand_forms(ex) + else + try + expand_forms(ex) + catch exc + exc isa LoweringError || rethrow() + # Hack: show only msg for more compatibility with flisp error forms + return Expr(:error, exc.msg) + #return Expr(:error, sprint(show, exc)) + end + end +end + +function lift_lowered_expr!(ex, nextids, valmap, lift_full) + if ex isa SSAValue + # Rename SSAValues into renumbered symbols + return get!(valmap, ex) do + newid = nextids[1] + nextids[1] = newid+1 + Symbol("ssa$newid") + end + end + if ex isa Symbol + if ex == Symbol("#self#") + return :_self_ + end + # Rename gensyms + name = string(ex) + if startswith(name, "#") + return get!(valmap, ex) do + newid = nextids[2] + nextids[2] = newid+1 + Symbol("gsym$newid") + end + end + end + if ex isa Expr + if ex.head == :block && length(ex.args) == 1 + # Remove trivial blocks + return lift_lowered_expr!(ex.args[1], nextids, valmap, lift_full) + end + map!(ex.args, ex.args) do e + lift_lowered_expr!(e, nextids, valmap, lift_full) + end + if lift_full + # Lift exotic Expr heads into standard julia syntax for ease in + # writing test case expressions. + if ex.head == :top || ex.head == :core + # Special global refs renamed to look like modules + newhead = ex.head == :top ? :Top : :Core + return Expr(:(.), newhead, QuoteNode(ex.args[1])) + elseif ex.head == :unnecessary + # `unnecessary` marks expressions generated by lowering that + # do not need to be evaluated if their value is unused. + return Expr(:call, :maybe_unused, ex.args...) + #= + elseif ex.head in [:_while, :_do_while, :scope_block, :break_block, + :break, :local_def, :require_existing_local] + return Expr(:macrocall, Symbol("@Expr"), nothing, + QuoteNode(ex.head), ex.args...) + =# + end + end + elseif ex isa Vector # Occasional case of lambdas + map!(ex, ex) do e + lift_lowered_expr!(e, nextids, valmap, lift_full) + end + end + return ex +end + +""" +Clean up an `Expr` into an equivalent form which can be easily entered by +hand + +* Replacing `SSAValue(id)` with consecutively numbered symbols :ssa\$i +* Remove trivial blocks +""" +function lift_lowered_expr(ex; lift_full=false) + valmap = Dict{Union{Symbol,SSAValue},Symbol}() + lift_lowered_expr!(remove_linenums!(deepcopy(ex)), ones(Int,2), valmap, lift_full) +end + +""" +Very slight lowering of reference expressions to allow comparison with +desugared forms. + +* Remove trivial blocks +* Translate psuedo-module expressions Top.x and Core.x to Expr(:top) and + Expr(:core) +""" +function lower_ref_expr!(ex) + if ex isa Expr + map!(lower_ref_expr!, ex.args, ex.args) + if ex.head == :block && length(ex.args) == 1 + # Remove trivial blocks + return lower_ref_expr!(ex.args[1]) + end + # Translate a selection of special expressions into the exotic Expr + # heads used in lowered code. + if ex.head == :(.) && length(ex.args) >= 1 && (ex.args[1] == :Top || + ex.args[1] == :Core) + if !(length(ex.args) == 2 && ex.args[2] isa QuoteNode) + throw("Unexpected top/core expression $(sprint(dump, ex))") + end + return Expr(ex.args[1] == :Top ? :top : :core, ex.args[2].value) + elseif ex.head == :call && length(ex.args) >= 1 && ex.args[1] == :maybe_unused + return Expr(:unnecessary, ex.args[2:end]...) + elseif ex.head == :macrocall && ex.args[1] == Symbol("@Expr") + head = ex.args[3] + head isa QuoteNode || throw(ArgumentError("`head` argument to @Expr should be quoted")) + return Expr(head.value, ex.args[4:end]...) + end + end + return ex +end +lower_ref_expr(ex) = lower_ref_expr!(remove_linenums!(deepcopy(ex))) + + +function diffdump(io::IOContext, ex1, ex2, n, prefix, indent) + if ex1 == ex2 + isempty(prefix) || print(io, prefix) + dump(io, ex1, 2, indent) + else + if ex1 isa Expr && ex2 isa Expr && ex1.head == ex2.head && length(ex1.args) == length(ex2.args) + isempty(prefix) || print(io, prefix) + println(io, "Expr") + println(io, indent, " head: ", ex1.head) + println(io, indent, " args: Array{Any}(", size(ex1.args), ")") + for i in 1:length(ex1.args) + prefix = string(indent, " ", i, ": ") + diffdump(io, ex1.args[i], ex2.args[i], 4, prefix, string(" ", indent)) + i < length(ex1.args) && println(io) + end + else + printstyled(io, string(prefix, sprint(dump, ex1, 4, indent; context=io)), color=:green) + println() + printstyled(io, string(prefix, sprint(dump, ex2, 4, indent; context=io)), color=:red) + end + end +end + +""" +Display colored differences between two expressions `ex1` and `ex2` using the +`dump` format. +""" +function diffdump(ex1, ex2; maxdepth=20) + mod = get(stdout, :module, Main) + diffdump(IOContext(stdout, :limit => true, :module => mod), ex1, ex2, maxdepth, "", "") + println(stdout) +end + +# For interactive convenience in constructing test cases with flisp based lowering +function desugar(ex; lift=:full) + expanded = _expand_forms(ex) + if lift == :full || lift == :partial + lift_lowered_expr(expanded; lift_full=(lift == :full)) + else + expanded + end +end + +# Macro for producing exotic `Expr`s found in lowered ASTs +# +# Note that this is provided for convenience/reference but in practice we +# expand it "manually" inside lower_ref_expr! +macro Expr(head, args...) + head isa QuoteNode || throw(ArgumentError("`head` argument to @Expr should be quoted")) + esc(Expr(head.value, args...)) +end + +""" + @desugar ex [kws...] + +Convenience macro, equivalent to `desugar(:(ex), kws...)`. +""" +macro desugar(ex, kws...) + quote + desugar($(Expr(:quote, ex)); $(map(esc, kws)...)) + end +end + +# Convenience macro to print code for a test case +macro maketest(ex) + Base.remove_linenums!(ex) + quote + print($(QuoteNode(ex)), "\n") + print(desugar($(Expr(:quote, ex))), "\n") + end +end + +""" + @testset_desugar(name, exprs) + +Test that a set of expressions lower correctly to desugared AST form. This +creates a new `@testset` with the given `name`. The statements in the block +`exprs` are interpreted as a flat list of any number of `input_expr`, +`ref_expr` pairs, where `input_expr` should be transformed by lowering into +`ref_expr` by the desugaring pass. For example, + +``` +@testset_desugar "Property notation" begin + # flisp: (expand-fuse-broadcast) + a.b + Top.getproperty(a, :b) + + a.b.c + Top.getproperty(Top.getproperty(a, :b), :c) +end +``` +""" +macro testset_desugar(name, block) + if !(block isa Expr && block.head == :block) + throw(ArgumentError("@testset_desugar requires a block as the second argument")) + end + loc = nothing + tests = [] + i = 1 + while i <= length(block.args) + if block.args[i] isa LineNumberNode + loc = block.args[i] + i += 1 + continue + end + exs = [] + while i <= length(block.args) && length(exs) < 2 + if !(block.args[i] isa LineNumberNode) + push!(exs, block.args[i]) + end + i += 1 + end + if length(exs) == 0 + break + end + if length(exs) == 1 + throw(ArgumentError("List of expressions to @testset_desugar must consist of input,ref pairs")) + end + input = exs[1] + ref = exs[2] + ex = quote + expanded = lift_lowered_expr(_expand_forms($(Expr(:quote, input)))) + reference = lower_ref_expr($(Expr(:quote, ref))) + @test expanded == reference + if expanded != reference + # Kinda crude. Would be much neater if Test supported custom/more + # capable diffing for failed tests. + println("Diff dump:") + diffdump(expanded, reference) + end + end + # Attribute the test to the correct line number + @assert ex.args[6].args[1] == Symbol("@test") + ex.args[6].args[2] = loc + push!(tests, ex) + end + quote + @testset $name begin + $(tests...) + end + end +end