adjustments for Julia v1.11 (#615)

A bunch of bugfixes and updates.
Also fixed some of test cases.

---------

Co-authored-by: Shuhei Kadowaki <aviatesk@gmail.com>

.github/workflows/CI.yml

@@ -13,17 +13,31 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version:
-          - '1.6'     # LTS
-          - '1'       # current stable
-          - '1.11-nightly' # next stable
-          - 'nightly'
-        os:
-          - ubuntu-latest
-          - macOS-latest
-          - windows-latest
-        arch:
-          - x64
+        include:
+          - version: '1.6' # old LTS
+            os: ubuntu-latest
+            arch: x64
+          - version: '1.10' # current stable
+            os: ubuntu-latest
+            arch: x64
+          - version: '~1.11.0-0' # next release
+            os: ubuntu-latest
+            arch: x64
+          - version: 'nightly' # dev
+            os: ubuntu-latest
+            arch: x64
+          #- version: '1' # x86 ubuntu -- disabled since PyCall/conda is broken on this platform
+          #  os: ubuntu-latest
+          #  arch: x86
+          - version: '1' # x86 windows
+            os: windows-latest
+            arch: x86
+          - version: '1' # x64 windows
+            os: windows-latest
+            arch: x64
+          - version: '1' # x64 macOS
+            os: macos-latest
+            arch: x64
     env:
       PYTHON: ""
     steps:

bin/generate_builtins.jl

@@ -2,16 +2,14 @@
 # Should be run on the latest Julia nightly
 using InteractiveUtils
 
-# All bultins added since 1.6. Needs to be updated whenever a new builtin is added
-const RECENTLY_ADDED = Core.Builtin[
-    Core._call_in_world_total, Core.donotdelete,
-    Core.get_binding_type, Core.set_binding_type!,
-    Core.getglobal, Core.setglobal!,
-    Core.modifyfield!, Core.replacefield!, Core.swapfield!,
-    Core.finalizer, Core._compute_sparams, Core._svec_ref,
-    Core.compilerbarrier,
-    Core.memoryref, Core.memoryref_isassigned, Core.memoryrefget, Core.memoryrefoffset, Core.memoryrefset!,
-    #=Core.current_scope=#
+# All builtins present in 1.6
+const ALWAYS_PRESENT = Core.Builtin[
+    (<:), (===), Core._abstracttype, Core._apply_iterate, Core._apply_pure,
+    Core._call_in_world, Core._call_latest, Core._equiv_typedef, Core._expr,
+    Core._primitivetype, Core._setsuper!, Core._structtype, Core._typebody!,
+    Core._typevar, Core.apply_type, Core.ifelse, Core.sizeof, Core.svec,
+    applicable, fieldtype, getfield, invoke, isa, isdefined, nfields,
+    setfield!, throw, tuple, typeassert, typeof
 ]
 # Builtins present in 1.6, not builtins (potentially still normal functions) anymore
 const RECENTLY_REMOVED = GlobalRef.(Ref(Core), [
@@ -232,7 +230,7 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
 
         id = findfirst(isequal(f), Core.Compiler.T_FFUNC_KEY)
         fcall = generate_fcall(f, Core.Compiler.T_FFUNC_VAL, id)
-        if f in RECENTLY_ADDED
+        if !(f in ALWAYS_PRESENT)
             print(io,
 """
     $head @static isdefined($(ft.name.module), $(repr(nameof(f)))) && f === $fname

src/breakpoints.jl

@@ -247,7 +247,7 @@ function breakpoint!(framecode::FrameCode, pc, condition::Condition=nothing, ena
         framecode.breakpoints[stmtidx] = BreakpointState(enabled, Core.eval(mod, fex))
     end
 end
-breakpoint!(framecode::FrameCode, pcs::AbstractArray, condition::Condition=nothing, enabled=true) = 
+breakpoint!(framecode::FrameCode, pcs::AbstractArray, condition::Condition=nothing, enabled=true) =
     foreach(pc -> breakpoint!(framecode, pc, condition, enabled), pcs)
 breakpoint!(frame::Frame, pc=frame.pc, condition::Condition=nothing) =
     breakpoint!(frame.framecode, pc, condition)
@@ -426,13 +426,14 @@ macro breakpoint(call_expr, args...)
     end
 end
 
-const __BREAKPOINT_MARKER__ = nothing
+struct BreakPointMarker end
+const __BREAK_POINT_MARKER__ = BreakPointMarker()
 
 """
     @bp
 
 Insert a breakpoint at a location in the source code.
 """
 macro bp()
-    return esc(:($(JuliaInterpreter).__BREAKPOINT_MARKER__))
+    return :(__BREAK_POINT_MARKER__)
 end

src/builtins.jl

@@ -184,18 +184,42 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         else
             return Some{Any}(Core.memoryrefget(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :memoryrefmodify!) && f === Core.memoryrefmodify!
+        if nargs == 5
+            return Some{Any}(Core.memoryrefmodify!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6])))
+        else
+            return Some{Any}(Core.memoryrefmodify!(getargs(args, frame)...))
+        end
     elseif @static isdefined(Core, :memoryrefoffset) && f === Core.memoryrefoffset
         if nargs == 1
             return Some{Any}(Core.memoryrefoffset(@lookup(frame, args[2])))
         else
             return Some{Any}(Core.memoryrefoffset(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :memoryrefreplace!) && f === Core.memoryrefreplace!
+        if nargs == 6
+            return Some{Any}(Core.memoryrefreplace!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]), @lookup(frame, args[7])))
+        else
+            return Some{Any}(Core.memoryrefreplace!(getargs(args, frame)...))
+        end
     elseif @static isdefined(Core, :memoryrefset!) && f === Core.memoryrefset!
         if nargs == 4
             return Some{Any}(Core.memoryrefset!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
         else
             return Some{Any}(Core.memoryrefset!(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :memoryrefsetonce!) && f === Core.memoryrefsetonce!
+        if nargs == 5
+            return Some{Any}(Core.memoryrefsetonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6])))
+        else
+            return Some{Any}(Core.memoryrefsetonce!(getargs(args, frame)...))
+        end
+    elseif @static isdefined(Core, :memoryrefswap!) && f === Core.memoryrefswap!
+        if nargs == 4
+            return Some{Any}(Core.memoryrefswap!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
+        else
+            return Some{Any}(Core.memoryrefswap!(getargs(args, frame)...))
+        end
     elseif @static isdefined(Core, :set_binding_type!) && f === Core.set_binding_type!
         return Some{Any}(Core.set_binding_type!(getargs(args, frame)...))
     elseif f === Core.sizeof
@@ -264,6 +288,14 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         else
             return Some{Any}(modifyfield!(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :modifyglobal!) && f === modifyglobal!
+        if nargs == 4
+            return Some{Any}(modifyglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
+        elseif nargs == 5
+            return Some{Any}(modifyglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6])))
+        else
+            return Some{Any}(modifyglobal!(getargs(args, frame)...))
+        end
     elseif f === nfields
         if nargs == 1
             return Some{Any}(nfields(@lookup(frame, args[2])))
@@ -280,6 +312,16 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         else
             return Some{Any}(replacefield!(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :replaceglobal!) && f === replaceglobal!
+        if nargs == 4
+            return Some{Any}(replaceglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
+        elseif nargs == 5
+            return Some{Any}(replaceglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6])))
+        elseif nargs == 6
+            return Some{Any}(replaceglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6]), @lookup(frame, args[7])))
+        else
+            return Some{Any}(replaceglobal!(getargs(args, frame)...))
+        end
     elseif f === setfield!
         if nargs == 3
             return Some{Any}(setfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4])))
@@ -288,6 +330,16 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         else
             return Some{Any}(setfield!(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :setfieldonce!) && f === setfieldonce!
+        if nargs == 3
+            return Some{Any}(setfieldonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4])))
+        elseif nargs == 4
+            return Some{Any}(setfieldonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
+        elseif nargs == 5
+            return Some{Any}(setfieldonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6])))
+        else
+            return Some{Any}(setfieldonce!(getargs(args, frame)...))
+        end
     elseif @static isdefined(Core, :setglobal!) && f === setglobal!
         if nargs == 3
             return Some{Any}(setglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4])))
@@ -296,6 +348,16 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         else
             return Some{Any}(setglobal!(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :setglobalonce!) && f === setglobalonce!
+        if nargs == 3
+            return Some{Any}(setglobalonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4])))
+        elseif nargs == 4
+            return Some{Any}(setglobalonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
+        elseif nargs == 5
+            return Some{Any}(setglobalonce!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5]), @lookup(frame, args[6])))
+        else
+            return Some{Any}(setglobalonce!(getargs(args, frame)...))
+        end
     elseif @static isdefined(Core, :swapfield!) && f === swapfield!
         if nargs == 3
             return Some{Any}(swapfield!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4])))
@@ -304,6 +366,14 @@ function maybe_evaluate_builtin(frame, call_expr, expand::Bool)
         else
             return Some{Any}(swapfield!(getargs(args, frame)...))
         end
+    elseif @static isdefined(Core, :swapglobal!) && f === swapglobal!
+        if nargs == 3
+            return Some{Any}(swapglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4])))
+        elseif nargs == 4
+            return Some{Any}(swapglobal!(@lookup(frame, args[2]), @lookup(frame, args[3]), @lookup(frame, args[4]), @lookup(frame, args[5])))
+        else
+            return Some{Any}(swapglobal!(getargs(args, frame)...))
+        end
     elseif f === throw
         if nargs == 1
             return Some{Any}(throw(@lookup(frame, args[2])))

src/commands.jl

@@ -271,6 +271,11 @@ If `frame.pc` points to the beginning of preparatory work for calling a keyword-
 function, advance forward until the actual call.
 """
 function maybe_step_through_kwprep!(@nospecialize(recurse), frame::Frame, istoplevel::Bool=false)
+    # XXX This code just does pattern-matching based on the current state of the compiler
+    # internals, which means this is very fragile against any future changes to those
+    # internals. We really need a more general and robust solution, but achieving that
+    # would mean simplifying and unifying how "keyword function" is represented and
+    # implemented. For the time being, our best bet is to keep tweaking it as best as we can.
     pc, src = frame.pc, frame.framecode.src
     n = length(src.code)
     stmt = pc_expr(frame, pc)

src/construct.jl

@@ -88,7 +88,7 @@ end
 
 get_source(meth::Method) = Base.uncompressed_ast(meth)
 
-if Base.VERSION < v"1.10.0-DEV.873"  # julia#48766
+@static if VERSION < v"1.10.0-DEV.873"  # julia#48766
     function get_source(g::GeneratedFunctionStub, env, file, line)
         b = g(env..., g.argnames...)
         b isa CodeInfo && return b

src/interpret.jl

@@ -461,7 +461,7 @@ function coverage_visit_line!(frame::Frame)
     code.report_coverage || return
     src = code.src
     codeloc = src.codelocs[pc]
-    if codeloc != frame.last_codeloc
+    if codeloc != frame.last_codeloc && codeloc != 0
         linetable = src.linetable::Vector{Any}
         lineinfo = linetable[codeloc]::Core.LineInfoNode
         file, line = String(lineinfo.file), lineinfo.line
@@ -672,7 +672,7 @@ function handle_err(@nospecialize(recurse), frame, err)
         rethrow(err)
     end
     data.last_exception[] = err
-    pc = VERSION >= v"1.11-" ? pop!(data.exception_frames) : data.exception_frames[end] # implicit :leave after https://github.com/JuliaLang/julia/pull/52245
+    pc = @static VERSION >= v"1.11-" ? pop!(data.exception_frames) : data.exception_frames[end] # implicit :leave after https://github.com/JuliaLang/julia/pull/52245
     frame.pc = pc
     return pc
 end

src/optimize.jl

@@ -1,7 +1,7 @@
 const compiled_calls = Dict{Any,Any}()
 
 # Pre-frame-construction lookup
-function lookup_stmt(stmts, arg)
+function lookup_stmt(stmts::Vector{Any}, @nospecialize arg)
     if isa(arg, SSAValue)
         arg = stmts[arg.id]
     end
@@ -45,16 +45,20 @@ function lookup_global_refs!(ex::Expr)
     return nothing
 end
 
-function lookup_getproperties(a::Expr)
-    if a.head === :call && length(a.args) == 3 &&
-        a.args[1] isa QuoteNode && a.args[1].value === Base.getproperty &&
-        a.args[2] isa QuoteNode && a.args[2].value isa Module           &&
-        a.args[3] isa QuoteNode && a.args[3].value isa Symbol
-        return lookup_global_ref(Core.GlobalRef(a.args[2].value, a.args[3].value))
-    end
-    return a
+function lookup_getproperties(code::Vector{Any}, @nospecialize a)
+    isexpr(a, :call) || return a
+    length(a.args) == 3 || return a
+    arg1 = lookup_stmt(code, a.args[1])
+    arg1 === Base.getproperty || return a
+    arg2 = lookup_stmt(code, a.args[2])
+    arg2 isa Module || return a
+    arg3 = lookup_stmt(code, a.args[3])
+    arg3 isa Symbol || return a
+    return lookup_global_ref(GlobalRef(arg2, arg3))
 end
 
+# TODO This isn't optimization really, but necessary to bypass llvmcall and foreigncall
+
 """
     optimize!(code::CodeInfo, mod::Module)
 
@@ -82,7 +86,7 @@ function optimize!(code::CodeInfo, scope)
                 continue
             else
                 lookup_global_refs!(stmt)
-                code.code[i] = lookup_getproperties(stmt)
+                code.code[i] = lookup_getproperties(code.code, stmt)
             end
         end
     end

src/types.jl

@@ -132,7 +132,7 @@ function FrameCode(scope, src::CodeInfo; generator=false, optimize=true)
     end
     breakpoints = Vector{BreakpointState}(undef, length(src.code))
     for (i, pc_expr) in enumerate(src.code)
-        if isa(pc_expr, Expr) && is_breakpoint_expr(pc_expr)
+        if lookup_stmt(src.code, pc_expr) === __BREAK_POINT_MARKER__
             breakpoints[i] = BreakpointState()
             src.code[i] = nothing
         end

src/utils.jl

@@ -210,7 +210,7 @@ end
 
 is_generated(meth::Method) = isdefined(meth, :generator)
 
-if Base.VERSION < v"1.10.0-DEV.873"  # julia#48766
+@static if VERSION < v"1.10.0-DEV.873"  # julia#48766
     get_staged(mi::MethodInstance) = Core.Compiler.get_staged(mi)
 else
     get_staged(mi::MethodInstance) = Core.Compiler.get_staged(mi, Base.get_world_counter())
@@ -364,12 +364,25 @@ end
 getfile(frame::Frame, pc=frame.pc) = getfile(frame.framecode, pc)
 
 function codelocation(code::CodeInfo, idx::Int)
-    codeloc = codelocs(code)[idx]
-    while codeloc == 0 && (code.code[idx] === nothing || isexpr(code.code[idx], :meta)) && idx < length(code.code)
-        idx += 1
-        codeloc = codelocs(code)[idx]
-    end
-    return codeloc
+    idx′ = idx
+    # look ahead if we are on a meta line
+    while idx′ < length(code.code)
+        codeloc = codelocs(code)[idx′]
+        codeloc == 0 || return codeloc
+        ex = code.code[idx′]
+        ex === nothing || isexpr(ex, :meta) || break
+        idx′ += 1
+    end
+    idx′ = idx - 1
+    # if zero, look behind until we find where we last might have had a line
+    while idx′ > 0
+        ex = code.code[idx′]
+        codeloc = codelocs(code)[idx′]
+        codeloc == 0 || return codeloc
+        idx′ -= 1
+    end
+    # for the start of the function, return index 1
+    return 1
 end
 
 function compute_corrected_linerange(method::Method)

test/breakpoints.jl

@@ -195,7 +195,7 @@ struct Squarer end
     # Breakpoint display
     io = IOBuffer()
     frame = JuliaInterpreter.enter_call(loop_radius2, 2)
-    if VERSION < v"1.9.0-DEV.846" # https://github.com/JuliaLang/julia/pull/45069
+    @static if VERSION < v"1.9.0-DEV.846" # https://github.com/JuliaLang/julia/pull/45069
         LOC = " in $(@__MODULE__) at $(@__FILE__)"
     else
         LOC = " @ $(@__MODULE__) $(contractuser(@__FILE__))"
@@ -219,7 +219,7 @@ struct Squarer end
     end
     fr, bp = @interpret f_outer_bp(3)
     @test leaf(fr).framecode.scope.name === :g_inner_bp
-    @test bp.stmtidx == 3
+    @test bp.stmtidx == (@static VERSION >= v"1.11-" ? 4 : 3)
 
     # Breakpoints on types
     remove()