From 72f67ce82a59713ff3dd762d69d784b429f684d4 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 14 Sep 2023 20:29:02 +0000 Subject: [PATCH] implement Memory{T} as a new backend for Array{T} TODO: add more tests for Memory specifically TODO: add to julia-licm the gc_loaded TODO: make sure all tests are passing (updated as needed) TODO: write API documentation for Memory (as needed) and devdoc updates (as needed) TODO: test Memory owner relationship (avoid implied unalias copy) of Memory in staticdata TODO: can we detect ownership for dataids for more accurate range checking? TODO: add memoryowner intrinsic (since this is a hidden field) TODO: add memoryoffset intrinsic (since this is computed better with exact idiv) TODO: implement dataids for Memory aliasing check and fix dataids for Array also makes memory allocation ccalls safer, for catching Serialization and deepcopy bugs in packages. more things to do TODO: don't mutate Memory length when making a String, but set Array ref field to empty Memory TODO: add alias `AtomicMemory{T} = GenericMemory{:atomic,T}` TODO: document jl_memoryt_slice TODO: add AddrSpace::Int parameter to GenericMemory Future possible kinds of possible Memories: GenericMemory{:atomic, 0, Int} GenericMemory{:not_atomic, 0, Int} GenericMemory{:unsync, 0, Int} GenericMemory{:normal, 0, Int} GenericMemory{:default, 0, Int} GenericMemory{:racy, 0, Int} GenericMemory{:local, 0, Int} GenericMemory{:const, 0, Int} GenericMemory{:ntuple, 0, Int} GenericMemory{:value, 0, Int} cleanups finish sizehint! implimentation and cleanups cleanup remove arrayref and arrayset from julia side (still need to remove the C side), and impliment in julia remove arrayref, arrayset, arraysize, arraylen from backend cleanups fix fix fixes fixes fixes fixes --- base/Base.jl | 16 +- base/abstractarray.jl | 5 +- base/array.jl | 302 +++- base/bitarray.jl | 2 +- base/bitset.jl | 2 +- base/boot.jl | 146 +- base/cmem.jl | 4 + base/compiler/abstractinterpretation.jl | 2 +- base/compiler/compiler.jl | 1 + base/compiler/optimize.jl | 3 - .../ssair/EscapeAnalysis/EscapeAnalysis.jl | 288 +--- base/compiler/ssair/inlining.jl | 2 +- base/compiler/tfuncs.jl | 407 +++--- base/compiler/typelattice.jl | 2 +- base/compiler/utilities.jl | 2 +- base/deepcopy.jl | 35 +- base/dict.jl | 32 +- base/essentials.jl | 44 +- base/experimental.jl | 3 +- base/genericmemory.jl | 223 +++ base/iddict.jl | 10 +- base/io.jl | 6 +- base/multidimensional.jl | 9 - base/pointer.jl | 49 +- base/promotion.jl | 2 +- base/reflection.jl | 57 +- base/reshapedarray.jl | 2 +- base/secretbuffer.jl | 12 +- base/summarysize.jl | 10 +- contrib/generate_precompile.jl | 1 + doc/src/devdocs/object.md | 4 +- doc/src/manual/embedding.md | 11 +- src/Makefile | 4 +- src/abi_win32.cpp | 4 +- src/abi_x86_64.cpp | 2 +- src/aotcompile.cpp | 4 +- src/array.c | 1296 ++++------------- src/ast.c | 16 +- src/builtin_proto.h | 8 +- src/builtins.c | 180 ++- src/ccall.cpp | 74 +- src/cgutils.cpp | 775 +++++----- src/clangsa/GCChecker.cpp | 2 + src/codegen.cpp | 661 +++++---- src/common_symbols1.inc | 120 +- src/common_symbols2.inc | 411 +++--- src/datatype.c | 204 ++- src/dlload.c | 2 +- src/gc-debug.c | 9 +- src/gc-stacks.c | 6 +- src/gc.c | 165 ++- src/gc.h | 4 +- src/gen_sysimg_symtab.jl | 10 +- src/genericmemory.c | 511 +++++++ src/gf.c | 79 +- src/iddict.c | 44 +- src/init.c | 6 +- src/interpreter.c | 14 +- src/intrinsics.cpp | 10 - src/intrinsics.h | 2 - src/ircode.c | 261 ++-- src/jitlayers.cpp | 2 +- src/jl_exported_data.inc | 11 + src/jl_exported_funcs.inc | 16 - src/jlapi.c | 9 +- src/jltypes.c | 121 +- src/julia.h | 344 +++-- src/julia_internal.h | 12 +- src/llvm-alloc-opt.cpp | 11 +- src/llvm-codegen-shared.h | 2 +- src/llvm-late-gc-lowering.cpp | 162 ++- src/llvm-pass-helpers.cpp | 3 +- src/llvm-pass-helpers.h | 1 + src/method.c | 43 +- src/precompile.c | 6 +- src/precompile_utils.c | 10 +- src/processor.cpp | 2 +- src/rtutils.c | 64 +- src/runtime_intrinsics.c | 6 - src/serialize.h | 3 +- src/smallintset.c | 34 +- src/stackwalk.c | 16 +- src/staticdata.c | 392 +++-- src/staticdata_utils.c | 116 +- src/subtype.c | 4 +- src/sys.c | 12 +- src/toplevel.c | 34 +- src/typemap.c | 201 +-- stdlib/Base64/src/buffer.jl | 6 +- stdlib/Profile/test/allocs.jl | 13 +- stdlib/Serialization/src/Serialization.jl | 70 +- test/abstractarray.jl | 8 +- test/cmdlineargs.jl | 4 +- .../compiler/EscapeAnalysis/EscapeAnalysis.jl | 1 - test/compiler/codegen.jl | 13 +- test/compiler/effects.jl | 50 +- test/compiler/inference.jl | 23 +- test/compiler/inline.jl | 10 +- test/compiler/irpasses.jl | 12 +- test/compiler/ssair.jl | 2 +- test/core.jl | 136 +- test/embedding/embedding.c | 6 +- test/precompile.jl | 8 +- test/reflection.jl | 12 +- test/show.jl | 8 +- test/sorting.jl | 14 +- 106 files changed, 4593 insertions(+), 4018 deletions(-) create mode 100644 base/genericmemory.jl create mode 100644 src/genericmemory.c diff --git a/base/Base.jl b/base/Base.jl index b440fda78a90a..ada5676e19c1e 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -6,10 +6,19 @@ using Core.Intrinsics, Core.IR # to start, we're going to use a very simple definition of `include` # that doesn't require any function (except what we can get from the `Core` top-module) -const _included_files = Array{Tuple{Module,String},1}(Core.undef, 1) +# start this big so that we don't have to resize before we have defined how to grow an array +const _included_files = Array{Tuple{Module,String},1}(Core.undef, 50) +_included_files.size = (1,) function include(mod::Module, path::String) - ccall(:jl_array_grow_end, Cvoid, (Any, UInt), _included_files, UInt(1)) - Core.arrayset(true, _included_files, (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)), arraylen(_included_files)) + len = getfield(_included_files.size, 1) + memlen = _included_files.ref.mem.length + lenp1 = Core.add_int(len, 1) + if len === memlen # by the time this is true we hopefully will have defined _growend! + _growend!(_included_files, UInt(1)) + else + _included_files.size = (lenp1,) + end + Core.memoryrefset(Core.memoryref(_included_files.ref, lenp1), (mod, ccall(:jl_prepend_cwd, Any, (Any,), path)), :not_atomic, true) Core.println(path) ccall(:jl_uv_flush, Nothing, (Ptr{Nothing},), Core.io_pointer(Core.stdout)) Core.include(mod, path) @@ -186,6 +195,7 @@ include("strings/lazy.jl") # array structures include("indices.jl") +include("genericmemory.jl") include("array.jl") include("abstractarray.jl") include("subarray.jl") diff --git a/base/abstractarray.jl b/base/abstractarray.jl index d00ec4ce7dadb..5e5d0b4e741c0 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -1290,8 +1290,6 @@ function getindex(A::AbstractArray, I...) error_if_canonical_getindex(IndexStyle(A), A, I...) _getindex(IndexStyle(A), A, to_indices(A, I)...) end -# To avoid invalidations from multidimensional.jl: getindex(A::Array, i1::Union{Integer, CartesianIndex}, I::Union{Integer, CartesianIndex}...) -@propagate_inbounds getindex(A::Array, i1::Integer, I::Integer...) = A[to_indices(A, (i1, I...))...] @inline unsafe_getindex(A::AbstractArray, I...) = @inbounds getindex(A, I...) @@ -1531,7 +1529,8 @@ their component parts. A typical definition for an array that wraps a parent is `Base.dataids(C::CustomArray) = dataids(C.parent)`. """ dataids(A::AbstractArray) = (UInt(objectid(A)),) -dataids(A::Array) = (UInt(pointer(A)),) +dataids(A::Memory) = (UInt(pointer(A)),) +dataids(A::Array) = dataids(A.ref.mem) dataids(::AbstractRange) = () dataids(x) = () diff --git a/base/array.jl b/base/array.jl index 599920253d75e..f5d08bc7887c6 100644 --- a/base/array.jl +++ b/base/array.jl @@ -120,8 +120,6 @@ const DenseVecOrMat{T} = Union{DenseVector{T}, DenseMatrix{T}} ## Basic functions ## -using Core: arraysize, arrayset, const_arrayref - """ @_safeindex @@ -137,13 +135,13 @@ end function _safeindex(__module__, ex) isa(ex, Expr) || return ex if ex.head === :(=) - lhs = arrayref(true, ex.args, 1) + lhs = ex.args[1] if isa(lhs, Expr) && lhs.head === :ref # xs[i] = x - rhs = arrayref(true, ex.args, 2) - xs = arrayref(true, lhs.args, 1) + rhs = ex.args[2] + xs = lhs.args[1] args = Vector{Any}(undef, length(lhs.args)-1) for i = 2:length(lhs.args) - arrayset(true, args, _safeindex(__module__, arrayref(true, lhs.args, i)), i-1) + args[i-1] = _safeindex(__module__, lhs.args[i]) end return Expr(:call, GlobalRef(__module__, :__inbounds_setindex!), xs, _safeindex(__module__, rhs), args...) end @@ -152,7 +150,7 @@ function _safeindex(__module__, ex) end args = Vector{Any}(undef, length(ex.args)) for i = 1:length(ex.args) - arrayset(true, args, _safeindex(__module__, arrayref(true, ex.args, i)), i) + args[i] = _safeindex(__module__, ex.args[i]) end return Expr(ex.head, args...) end @@ -187,12 +185,13 @@ function vect(X...) return T[X...] end -size(a::Array, d::Integer) = arraysize(a, d isa Int ? d : convert(Int, d)) -size(a::Vector) = (arraysize(a,1),) -size(a::Matrix) = (arraysize(a,1), arraysize(a,2)) -size(a::Array{<:Any,N}) where {N} = (@inline; ntuple(M -> size(a, M), Val(N))::Dims) +function size(a::Array, d::Integer) + d < 1 && error("arraysize: dimension out of range") + return d > ndims(a) ? 1 : a.size[d] +end +size(a::Array) = a.size -asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...) +asize_from(a::Array, n) = n > ndims(a) ? () : (size(a,n), asize_from(a, n+1)...) allocatedinline(@nospecialize T::Type) = (@_total_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0)) @@ -232,6 +231,7 @@ function _unsetindex!(A::Array{T}, i::Int) where {T} end +# TODO: deprecate this (aligned_sizeof and/or elsize and/or sizeof(Some{T}) are more correct) """ Base.bitsunionsize(U::Union) -> Int @@ -252,7 +252,7 @@ function bitsunionsize(u::Union) return sz end -elsize(@nospecialize _::Type{A}) where {T,A<:Array{T}} = aligned_sizeof(T) +elsize(::Type{A}) where {T,A<:Array{T}} = aligned_sizeof(T) function elsize(::Type{Ptr{T}}) where T # this only must return something valid for values which satisfy is_valid_intrinsic_elptr(T), # which includes Any and most concrete datatypes @@ -261,15 +261,23 @@ function elsize(::Type{Ptr{T}}) where T return LLT_ALIGN(Core.sizeof(T), datatype_alignment(T)) end elsize(::Type{Union{}}, slurp...) = 0 -sizeof(a::Array) = Core.sizeof(a) + +sizeof(a::Array) = length(a) * elsize(typeof(a)) # TODO(jwn): add bitsunion bytes? function isassigned(a::Array, i::Int...) @inline @boundscheck checkbounds(Bool, a, i...) || return false - ii = (_sub2ind(size(a), i...) % UInt) - 1 - ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1 + ii = _sub2ind(size(a), i...) + return @inbounds isassigned(memoryref(a.ref, ii, false)) +end + +function isassigned(a::Vector, i::Int) # slight compiler simplification for the most common case + @inline + @boundscheck checkbounds(Bool, a, i) || return false + return @inbounds isassigned(memoryref(a.ref, i, false)) end + ## copy ## """ @@ -291,9 +299,12 @@ end function _unsafe_copyto!(dest, doffs, src, soffs, n) + @_terminates_locally_meta + # use pointer math to determine if they are deemed to alias destp = pointer(dest, doffs) srcp = pointer(src, soffs) - @inbounds if destp < srcp || destp > srcp + n + endp = pointer(src, soffs + n - 1) + @inbounds if destp < srcp || destp > endp for i = 1:n if isassigned(src, soffs + i - 1) dest[doffs + i - 1] = src[soffs + i - 1] @@ -324,6 +335,7 @@ that N is inbounds on either array. Incorrect usage may corrupt or segfault your the same manner as C. """ function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T + @_terminates_locally_meta t1 = @_gc_preserve_begin dest t2 = @_gc_preserve_begin src destp = pointer(dest, doffs) @@ -670,6 +682,25 @@ if nameof(@__MODULE__) === :Base # avoid method overwrite # constructors should make copies Array{T,N}(x::AbstractArray{S,N}) where {T,N,S} = copyto_axcheck!(Array{T,N}(undef, size(x)), x) AbstractArray{T,N}(A::AbstractArray{S,N}) where {T,N,S} = copyto_axcheck!(similar(A,T), A) + +else +# Bare Array Constructors +# from MemoryRef and size +@eval function (self::Type{Array{T,1}})(ref::MemoryRef, dims::Tuple{Int}) where {T} + length = getfield(dims, 1) + length === 0 || MemoryRef(ref, length, true) # make sure last element is inbounds + return $(Expr(:new, :self, :(ref), :(dims))) +end +@eval function (self::Type{Array{T,N}})(ref::MemoryRef, dims::NTuple{N,Int}) where {N, T} + length = Core.checked_dims(dims...) + length === 0 || MemoryRef(ref, length, true) # make sure last element is inbounds + mem = ref.mem + # offset == 0 + if ref === MemoryRef(mem) && length === mem.length + return $(Expr(:new, :self, :(ref), :(dims))) + end + throw_boundserror(ref, dims) +end end ## copying iterators to containers @@ -1016,16 +1047,19 @@ Dict{String, Int64} with 2 entries: """ function setindex! end -@eval setindex!(A::Array{T}, x, i1::Int) where {T} = - arrayset($(Expr(:boundscheck)), A, x isa T ? x : convert(T,x)::T, i1) -@eval setindex!(A::Array{T}, x, i1::Int, i2::Int, I::Int...) where {T} = - (@inline; arrayset($(Expr(:boundscheck)), A, x isa T ? x : convert(T,x)::T, i1, i2, I...)) - -__inbounds_setindex!(A::Array{T}, x, i1::Int) where {T} = - arrayset(false, A, convert(T,x)::T, i1) -__inbounds_setindex!(A::Array{T}, x, i1::Int, i2::Int, I::Int...) where {T} = - (@inline; arrayset(false, A, convert(T,x)::T, i1, i2, I...)) +function setindex!(A::Array{T}, x, i::Int) where {T} + @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, i) + memoryrefset(memoryref(A.ref, i, false), x isa T ? x : convert(T,x)::T, :not_atomic, false) + A +end +# FIXME: this is getting marked consistent?? Also this isn't being marked nothrow when it should +function __inbounds_setindex!(A::Array{T}, x::T, i::Int) where {T} + @_nothrow_meta + memoryrefset(memoryref(A.ref, i, false), x, :not_atomic, false) + A +end +__inbounds_setindex!(A::Array{T}, x, i::Int) where {T} = __inbounds_setindex!(A, convert(T,x)::T, i) # This is redundant with the abstract fallbacks but needed and helpful for bootstrap function setindex!(A::Array, X::AbstractArray, I::AbstractVector{Int}) @_propagate_inbounds_meta @@ -1063,24 +1097,188 @@ function setindex!(A::Array{T}, X::Array{T}, c::Colon) where T return A end -# efficiently grow an array +# Pick new memory size for efficiently growing an array +# TODO: This should know about the size of our GC pools +# Specifically we are wasting ~10% of memory for small arrays +# by not picking memory sizes that max out a GC pool +function overallocation(maxsize) + maxsize < 8 && return 8; + # compute maxsize = maxsize + 4*maxsize^(7/8) + maxsize/8 + # for small n, we grow faster than O(n) + # for large n, we grow at O(n/8) + # and as we reach O(memory) for memory>>1MB, + # this means we end by adding about 10% of memory each time + exp2 = sizeof(maxsize) * 8 - Core.Intrinsics.ctlz_int(maxsize) + maxsize += (1 << div(exp2 * 7, 8)) * 4 + div(maxsize, 8) + return maxsize +end + +function array_new_memory(mem::Memory{T}, newlen::Int) where T + if ccall(:jl_mem_owner, Any, (Memory{T},), mem) isa String + # if data is in a String, keep it that way + # TODO: use jl_gc_expand_string(oldstr, newlen)? + str = _string_n(newlen) + return ccall(:jl_string_to_genericmemory, Memory{T}, (Any,), str) + else + # TODO: when implimented, this should use a memory growing call + mem = Memory{T}(undef, newlen) + return mem + end +end -_growbeg!(a::Vector, delta::Integer) = - ccall(:jl_array_grow_beg, Cvoid, (Any, UInt), a, delta) -_growend!(a::Vector, delta::Integer) = - ccall(:jl_array_grow_end, Cvoid, (Any, UInt), a, delta) -_growat!(a::Vector, i::Integer, delta::Integer) = - ccall(:jl_array_grow_at, Cvoid, (Any, Int, UInt), a, i - 1, delta) +function _growbeg!(a::Vector, delta::Integer) + delta = Int(delta) + delta == 0 && return # avoid attempting to index off the end + delta >= 0 || throw(ArgumentError("grow requires delta >= 0")) + ref = a.ref + mem = ref.mem + len = length(a) + offset = memoffset(ref) + newlen = len + delta + a.size = (newlen,) + # if offset is far enough advanced to fit data in existing memory without copying + if delta <= offset + a.ref = MemoryRef(ref, 1 - delta, false) + else + @noinline (function() + memlen = length(mem) + # since we will allocate the array in the middle of the memory we need at least 2*delta extra space + # the +1 is because I didn't want to have an off by 1 error. + newmemlen = max(overallocation(memlen), len+2*delta+1) + newoffset = div(newmemlen - newlen, 2) + # If there is extra data after the end of the array we can use that space so long as there is enough + # space at the end that there won't be quadratic behavior with a mix of growth from both ends. + # Specifically, we want to ensure that we will only do this operation once before + # increasing the size of the array, and that we leave enough space at both the benining and the end. + if newoffset + newlen + 1 < memlen + newoffset = div(memlen - newlen, 2) + newmem = mem + else + newmem = array_new_memory(mem, newmemlen) + end + if len > 0 + unsafe_copyto!(newmem, newoffset+delta+1, mem, offset+1, len) + end + a.ref = MemoryRef(newmem, newoffset+1, false) + end)() + end + return +end -# efficiently delete part of an array +function _growend!(a::Vector, delta::Integer) + delta = Int(delta) + delta >= 0 || throw(ArgumentError("grow requires delta >= 0")) + ref = a.ref + mem = ref.mem + memlen = length(mem) + len = length(a) + newlen = len + delta + offset = memoffset(ref) + a.size = (newlen,) + newmemlen = offset + newlen + if memlen < newmemlen + @noinline (function() + if offset > div(5*newlen, 4) + # If the offset is far enough that we can copy without resizing + # while maintaining proportional spacing on both ends of the array + # note that this branch prevents infinite growth when doing combinations + # of push! and popfirst! (i.e. when using a Vector as a queue) + newmem = mem + newoffset = div(newlen, 8) + else + # grow either by our computed overallocation factor + # or exactly the requested size, whichever is larger + # TODO we should possibly increase the offset if the current offset is nonzero, + newmemlen2 = max(overallocation(memlen), newmemlen) + newmem = array_new_memory(mem, newmemlen2) + newoffset = offset + end + if len > 0 + unsafe_copyto!(newmem, newoffset+1, mem, offset+1, len) + end + a.ref = MemoryRef(newmem, newoffset+1, false) + end)() + end + return +end -_deletebeg!(a::Vector, delta::Integer) = - ccall(:jl_array_del_beg, Cvoid, (Any, UInt), a, delta) -_deleteend!(a::Vector, delta::Integer) = - ccall(:jl_array_del_end, Cvoid, (Any, UInt), a, delta) -_deleteat!(a::Vector, i::Integer, delta::Integer) = - ccall(:jl_array_del_at, Cvoid, (Any, Int, UInt), a, i - 1, delta) +function _growat!(a::Vector, i::Integer, delta::Integer) + delta = Int(delta) + i == 1 && return _growbeg!(a, delta) + len = length(a) + i == len + 1 && return _growend!(a, delta) + delta >= 0 || throw(ArgumentError("grow requires delta >= 0")) + 1 < i <= len || throw(BoundsError(a, i)) + ref = a.ref + mem = ref.mem + memlen = length(mem) + newlen = len + delta + offset = memoffset(ref) + a.size = (newlen,) + newmemlen = offset + newlen + + # which side would we rather grow into? + prefer_start = i <= div(len, 2) + # if offset is far enough advanced to fit data in begining of the memory + if prefer_start && delta <= offset + a.ref = MemoryRef(mem, offset-delta+1, false) + unsafe_copyto!(mem, offset-delta+1, mem, offset+1, i) + elseif !prefer_start && memlen >= newmemlen + unsafe_copyto!(mem, offset+delta+i, mem, offset+i, len-i+1) + else + # since we will allocate the array in the middle of the memory we need at least 2*delta extra space + # the +1 is because I didn't want to have an off by 1 error. + newmemlen = max(overallocation(memlen), len+2*delta+1) + newoffset = (newmemlen - newlen) ÷ 2 + newmem = array_new_memory(mem, newmemlen) + a.ref = MemoryRef(newmem, newoffset+1, false) + unsafe_copyto!(newmem, newoffset+1, mem, offset+1, i) + unsafe_copyto!(newmem, newoffset+delta+i, mem, offset+i, len-i+1) + end +end +# efficiently delete part of an array +function _deletebeg!(a::Vector, delta::Integer) + delta = Int(delta) + len = length(a) + 0 <= delta <= len || throw(ArgumentError("_deleteat! requires delta in 0:length(a)")) + for i in 1:delta + _unsetindex!(a, i) + end + newlen = len - delta + if newlen != 0 # if newlen==0 we could accidentally index past the memory + a.ref = MemoryRef(a.ref, delta + 1, false) + end + a.size = (newlen,) + return +end +function _deleteend!(a::Vector, delta::Integer) + delta = Int(delta) + len = length(a) + 0 <= delta <= len || throw(ArgumentError("_deleteat! requires delta in 0:length(a)")) + newlen = len - delta + for i in newlen+1:len + _unsetindex!(a, i) + end + a.size = (newlen,) + return +end +function _deleteat!(a::Vector, i::Integer, delta::Integer) + i = Int(i) + len = length(a) + 0 <= delta || throw(ArgumentError("_deleteat! requires delta >= 0")) + 1 <= i <= len || throw(BoundsError(a, i)) + i + delta <= len + 1 || throw(BoundsError(a, i + delta - 1)) + newa = a + if 2*i + delta <= len + unsafe_copyto!(newa, 1 + delta, a, 1, i - 1) + _deletebeg!(a, delta) + else + unsafe_copyto!(newa, i, a, i + delta, len + 1 - delta - i) + _deleteend!(a, delta) + end + return +end ## Dequeue functionality ## """ @@ -1346,7 +1544,29 @@ For types that support `sizehint!`, function sizehint! end function sizehint!(a::Vector, sz::Integer) - ccall(:jl_array_sizehint, Cvoid, (Any, UInt), a, sz) + len = length(a) + ref = a.ref + mem = ref.mem + memlen = length(mem) + offset = memoffset(ref) + sz = max(Int(sz), offset+len) + if sz <= memlen + # if we don't save at least 1/8th memlen then its not worth it to shrink + if memlen - sz <= div(memlen, 8) + return a + end + newmem = array_new_memory(mem, sz) + if len == 0 + a.ref = MemoryRef(newmem) + else + unsafe_copyto!(newmem, offset+1, mem, offset+1, len) + a.ref = MemoryRef(newmem, offset+1, false) + end + else + inc = sz - len; + _growend!(a, inc) + a.size = (len,) # undo the size change from _growend! + end a end diff --git a/base/bitarray.jl b/base/bitarray.jl index 0fd2a665f9f29..dd791679c7000 100644 --- a/base/bitarray.jl +++ b/base/bitarray.jl @@ -807,7 +807,7 @@ prepend!(B::BitVector, items) = prepend!(B, BitArray(items)) prepend!(A::Vector{Bool}, items::BitVector) = prepend!(A, Array(items)) function sizehint!(B::BitVector, sz::Integer) - ccall(:jl_array_sizehint, Cvoid, (Any, UInt), B.chunks, num_bit_chunks(sz)) + sizehint!(B.chunks, num_bit_chunks(sz)) return B end diff --git a/base/bitset.jl b/base/bitset.jl index 240be822fa263..d7c905180586a 100644 --- a/base/bitset.jl +++ b/base/bitset.jl @@ -15,7 +15,7 @@ mutable struct BitSet <: AbstractSet{Int} # 1st stored Int equals 64*offset offset::Int - BitSet() = new(resize!(Vector{UInt64}(undef, 4), 0), NO_OFFSET) + BitSet() = new(Vector{UInt64}(MemoryRef(Memory{UInt64}(undef, 4)), (0,)), NO_OFFSET) end """ diff --git a/base/boot.jl b/base/boot.jl index 637b16e04c13e..df54e5e5e44c0 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -52,8 +52,23 @@ #abstract type AbstractArray{T,N} end #abstract type DenseArray{T,N} <: AbstractArray{T,N} end +#struct GenericMemory{isatomic, T} +# length::Int +# const data::Ptr{Cvoid} +# Union{ # hidden data +# elements :: NTuple{length, T} +# owner :: Any +# } +#end + +# struct MemoryRef{isatomic, T} +# mem::Memory{isatomic, T} +# data::Ptr{Cvoid} +#end + #mutable struct Array{T,N} <: DenseArray{T,N} -## opaque +# ref::MemoryRef{:not_atomic,T} +# dims::NTuple{N,Int} #end #mutable struct Module @@ -173,8 +188,8 @@ export Tuple, Type, UnionAll, TypeVar, Union, Nothing, Cvoid, AbstractArray, DenseArray, NamedTuple, Pair, # special objects - Function, Method, - Module, Symbol, Task, Array, UndefInitializer, undef, WeakRef, VecElement, + Function, Method, Array, Memory, GenericMemory, MemoryRef, + Module, Symbol, Task, UndefInitializer, undef, WeakRef, VecElement, # numeric types Number, Real, Integer, Bool, Ref, Ptr, AbstractFloat, Float16, Float32, Float64, @@ -356,13 +371,15 @@ struct UndefKeywordError <: Exception var::Symbol end +const typemax_UInt = Intrinsics.sext_int(UInt, 0xFF) +const typemax_Int = Core.Intrinsics.udiv_int(Core.Intrinsics.sext_int(Int, 0xFF), 2) + struct MethodError <: Exception f args world::UInt MethodError(@nospecialize(f), @nospecialize(args), world::UInt) = new(f, args, world) end -const typemax_UInt = ccall(:jl_typemax_uint, Any, (Any,), UInt) MethodError(@nospecialize(f), @nospecialize(args)) = MethodError(f, args, typemax_UInt) struct AssertionError <: Exception @@ -471,34 +488,102 @@ const NTuple{N,T} = Tuple{Vararg{T,N}} ## primitive Array constructors struct UndefInitializer end const undef = UndefInitializer() + +# type and dimensionality specified +(self::Type{GenericMemory{isatomic,T}})(::UndefInitializer, m::Int) where {T,isatomic} = + if isdefined(self, :instance) && m === 0 + self.instance + else + ccall(:jl_alloc_genericmemory, Ref{GenericMemory{isatomic,T}}, (Any, Int), self, m) + end +(self::Type{GenericMemory{isatomic,T}})(::UndefInitializer, d::NTuple{1,Int}) where {T,isatomic} = self(undef, getfield(d,1)) +# empty vector constructor +(self::Type{GenericMemory{isatomic,T}})() where {T,isatomic} = self(undef, 0) +# copy constructors + +MemoryRef(mem::GenericMemory) = memoryref(mem) +MemoryRef(ref::MemoryRef, i::Int) = memoryref(ref, i, true) +MemoryRef(ref::MemoryRef, i::Int, check) = memoryref(ref, i, check) +MemoryRef(mem::GenericMemory, i::Int) = memoryref(memoryref(mem), i, true) +MemoryRef(mem::GenericMemory, i::Int, check) = memoryref(memoryref(mem), i, check) +const Memory{T} = GenericMemory{:not_atomic, T} + +# construction helpers for Array +new_as_memoryref(self::Type{MemoryRef{isatomic,T}}, m::Int) where {T,isatomic} = memoryref(fieldtype(self, :mem)(undef, m)) + +# checked-multiply intrinsic function for dimensions +_checked_mul_dims() = 1, false +_checked_mul_dims(m::Int) = m, Intrinsics.ule_int(typemax_Int, m) # equivalently: (m + 1) < 1 +function _checked_mul_dims(m::Int, n::Int) + b = Intrinsics.checked_smul_int(m, n) + a = getfield(b, 1) + ovflw = getfield(b, 2) + ovflw = Intrinsics.or_int(ovflw, Intrinsics.ule_int(typemax_Int, m)) + ovflw = Intrinsics.or_int(ovflw, Intrinsics.ule_int(typemax_Int, n)) + return a, ovflw +end +function _checked_mul_dims(m::Int, d::Int...) + @_foldable_meta # the compiler needs to know this loop terminates + a = m + i = 1 + ovflw = false + while Intrinsics.sle_int(i, nfields(d)) + di = getfield(d, i) + b = Intrinsics.checked_smul_int(a, di) + ovflw = Intrinsics.or_int(ovflw, getfield(b, 2)) + ovflw = Intrinsics.or_int(ovflw, Intrinsics.ule_int(typemax_Int, di)) + a = getfield(b, 1) + i = Intrinsics.add_int(i, 1) + end + return a, ovflw +end + +# convert a set of dims to a length, with overflow checking +checked_dims() = 1 +checked_dims(m::Int) = m # defer this check to Memory constructor instead +function checked_dims(d::Int...) + b = _checked_mul_dims(d...) + getfield(b, 2) && throw(ArgumentError("invalid Array dimensions")) + return getfield(b, 1) +end + # type and dimensionality specified, accepting dims as series of Ints -Array{T,1}(::UndefInitializer, m::Int) where {T} = - ccall(:jl_alloc_array_1d, Array{T,1}, (Any, Int), Array{T,1}, m) -Array{T,2}(::UndefInitializer, m::Int, n::Int) where {T} = - ccall(:jl_alloc_array_2d, Array{T,2}, (Any, Int, Int), Array{T,2}, m, n) -Array{T,3}(::UndefInitializer, m::Int, n::Int, o::Int) where {T} = - ccall(:jl_alloc_array_3d, Array{T,3}, (Any, Int, Int, Int), Array{T,3}, m, n, o) -Array{T,N}(::UndefInitializer, d::Vararg{Int,N}) where {T,N} = - ccall(:jl_new_array, Array{T,N}, (Any, Any), Array{T,N}, d) +eval(Core, :(function (self::Type{Array{T,1}})(::UndefInitializer, m::Int) where {T} + @noinline + mem = fieldtype(fieldtype(self, :ref), :mem)(undef, m) + return $(Expr(:new, :self, :(memoryref(mem)), :((m,)))) +end)) +eval(Core, :(function (self::Type{Array{T,2}})(::UndefInitializer, m::Int, n::Int) where {T} + @noinline + return $(Expr(:new, :self, :(new_as_memoryref(fieldtype(self, :ref), checked_dims(m, n))), :((m, n)))) +end)) +eval(Core, :(function (self::Type{Array{T,3}})(::UndefInitializer, m::Int, n::Int, o::Int) where {T} + @noinline + return $(Expr(:new, :self, :(new_as_memoryref(fieldtype(self, :ref), checked_dims(m, n, o))), :((m, n, o)))) +end)) +eval(Core, :(function (self::Type{Array{T, N}})(::UndefInitializer, d::Vararg{Int, N}) where {T, N} + @noinline + return $(Expr(:new, :self, :(new_as_memoryref(fieldtype(self, :ref), checked_dims(d...))), :d)) +end)) # type and dimensionality specified, accepting dims as tuples of Ints -Array{T,1}(::UndefInitializer, d::NTuple{1,Int}) where {T} = Array{T,1}(undef, getfield(d,1)) -Array{T,2}(::UndefInitializer, d::NTuple{2,Int}) where {T} = Array{T,2}(undef, getfield(d,1), getfield(d,2)) -Array{T,3}(::UndefInitializer, d::NTuple{3,Int}) where {T} = Array{T,3}(undef, getfield(d,1), getfield(d,2), getfield(d,3)) -Array{T,N}(::UndefInitializer, d::NTuple{N,Int}) where {T,N} = ccall(:jl_new_array, Array{T,N}, (Any, Any), Array{T,N}, d) +(self::Type{Array{T,1}})(::UndefInitializer, d::NTuple{1, Int}) where {T} = self(undef, getfield(d, 1)) +(self::Type{Array{T,2}})(::UndefInitializer, d::NTuple{2, Int}) where {T} = self(undef, getfield(d, 1), getfield(d, 2)) +(self::Type{Array{T,3}})(::UndefInitializer, d::NTuple{3, Int}) where {T} = self(undef, getfield(d, 1), getfield(d, 2), getfield(d, 3)) +(self::Type{Array{T,N}})(::UndefInitializer, d::NTuple{N, Int}) where {T, N} = self(undef, d...) # type but not dimensionality specified -Array{T}(::UndefInitializer, m::Int) where {T} = Array{T,1}(undef, m) -Array{T}(::UndefInitializer, m::Int, n::Int) where {T} = Array{T,2}(undef, m, n) -Array{T}(::UndefInitializer, m::Int, n::Int, o::Int) where {T} = Array{T,3}(undef, m, n, o) -Array{T}(::UndefInitializer, d::NTuple{N,Int}) where {T,N} = Array{T,N}(undef, d) +Array{T}(::UndefInitializer, m::Int) where {T} = Array{T, 1}(undef, m) +Array{T}(::UndefInitializer, m::Int, n::Int) where {T} = Array{T, 2}(undef, m, n) +Array{T}(::UndefInitializer, m::Int, n::Int, o::Int) where {T} = Array{T, 3}(undef, m, n, o) +Array{T}(::UndefInitializer, d::NTuple{N, Int}) where {T, N} = Array{T, N}(undef, d) # empty vector constructor -Array{T,1}() where {T} = Array{T,1}(undef, 0) +(self::Type{Array{T, 1}})() where {T} = self(undef, 0) -(Array{T,N} where T)(x::AbstractArray{S,N}) where {S,N} = Array{S,N}(x) +(Array{T, N} where T)(x::AbstractArray{S, N}) where {S, N} = Array{S, N}(x) -Array(A::AbstractArray{T,N}) where {T,N} = Array{T,N}(A) -Array{T}(A::AbstractArray{S,N}) where {T,N,S} = Array{T,N}(A) +Array(A::AbstractArray{T, N}) where {T, N} = Array{T, N}(A) +Array{T}(A::AbstractArray{S, N}) where {T, N, S} = Array{T, N}(A) -AbstractArray{T}(A::AbstractArray{S,N}) where {T,S,N} = AbstractArray{T,N}(A) +AbstractArray{T}(A::AbstractArray{S, N}) where {T, S, N} = AbstractArray{T, N}(A) # primitive Symbol constructors @@ -513,9 +598,9 @@ function Symbol(s::String) @noinline return _Symbol(ccall(:jl_string_ptr, Ptr{UInt8}, (Any,), s), sizeof(s), s) end -function Symbol(a::Array{UInt8,1}) +function Symbol(a::Array{UInt8, 1}) @noinline - return _Symbol(ccall(:jl_array_ptr, Ptr{UInt8}, (Any,), a), Intrinsics.arraylen(a), a) + return _Symbol(bitcast(Ptr{UInt8}, a.ref.ptr), getfield(a.dims, 1), a.ref.mem) end Symbol(s::Symbol) = s @@ -855,4 +940,11 @@ function _hasmethod(@nospecialize(tt)) # this function has a special tfunc return Intrinsics.not_int(ccall(:jl_gf_invoke_lookup, Any, (Any, Any, UInt), tt, nothing, world) === nothing) end + +# for backward compat +function arrayref(args...) end +function const_arrayref(args...) end +function arrayset(args...) end +function arraysize(args...) end +export arrayref, arrayset, arraysize, const_arrayref ccall(:jl_set_istopmod, Cvoid, (Any, Bool), Core, true) diff --git a/base/cmem.jl b/base/cmem.jl index 8b0b99b3a6ebd..18b4e4e16b550 100644 --- a/base/cmem.jl +++ b/base/cmem.jl @@ -10,6 +10,7 @@ Call `memcpy` from the C standard library. """ function memcpy(dst::Ptr, src::Ptr, n::Integer) + @_terminates_locally_meta ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t), dst, src, n) end @@ -23,6 +24,7 @@ Call `memmove` from the C standard library. """ function memmove(dst::Ptr, src::Ptr, n::Integer) + @_terminates_locally_meta ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t), dst, src, n) end @@ -36,6 +38,7 @@ Call `memset` from the C standard library. """ function memset(p::Ptr, val, n::Integer) + @_terminates_locally_meta ccall(:memset, Ptr{Cvoid}, (Ptr{Cvoid}, Cint, Csize_t), p, val, n) end @@ -49,5 +52,6 @@ Call `memcmp` from the C standard library. """ function memcmp(a::Ptr, b::Ptr, n::Integer) + @_terminates_locally_meta ccall(:memcmp, Cint, (Ptr{Cvoid}, Ptr{Cvoid}, Csize_t), a, b, n % Csize_t) % Int end diff --git a/base/compiler/abstractinterpretation.jl b/base/compiler/abstractinterpretation.jl index 6aae1d6b384e8..bdcc30fa76e0a 100644 --- a/base/compiler/abstractinterpretation.jl +++ b/base/compiler/abstractinterpretation.jl @@ -1673,7 +1673,7 @@ end thentype = Bottom elseif rt === Const(true) elsetype = Bottom - elseif elsetype isa Type && isdefined(typeof(c.val), :instance) # can only widen a if it is a singleton + elseif elsetype isa Type && issingletontype(typeof(c.val)) # can only widen a if it is a singleton elsetype = typesubtract(elsetype, typeof(c.val), max_union_splitting) end return ConditionalTypes(thentype, elsetype) diff --git a/base/compiler/compiler.jl b/base/compiler/compiler.jl index 31db9a5551c2b..997e6989312f3 100644 --- a/base/compiler/compiler.jl +++ b/base/compiler/compiler.jl @@ -104,6 +104,7 @@ include("strings/lazy.jl") # core array operations include("indices.jl") +include("genericmemory.jl") include("array.jl") include("abstractarray.jl") diff --git a/base/compiler/optimize.jl b/base/compiler/optimize.jl index 36ef4561fc339..847839a3aace1 100644 --- a/base/compiler/optimize.jl +++ b/base/compiler/optimize.jl @@ -1046,9 +1046,6 @@ function statement_cost(ex::Expr, line::Int, src::Union{CodeInfo, IRCode}, sptyp # tuple iteration/destructuring makes that impossible # return plus_saturate(argcost, isknowntype(extyp) ? 1 : params.inline_nonleaf_penalty) return 0 - elseif (f === Core.arrayref || f === Core.const_arrayref || f === Core.arrayset) && length(ex.args) >= 3 - atyp = argextype(ex.args[3], src, sptypes) - return isknowntype(atyp) ? 4 : error_path ? params.inline_error_path_cost : params.inline_nonleaf_penalty elseif f === typeassert && isconstType(widenconst(argextype(ex.args[3], src, sptypes))) return 1 end diff --git a/base/compiler/ssair/EscapeAnalysis/EscapeAnalysis.jl b/base/compiler/ssair/EscapeAnalysis/EscapeAnalysis.jl index 4c533367017ff..edb13f1089452 100644 --- a/base/compiler/ssair/EscapeAnalysis/EscapeAnalysis.jl +++ b/base/compiler/ssair/EscapeAnalysis/EscapeAnalysis.jl @@ -15,9 +15,7 @@ const _TOP_MOD = ccall(:jl_base_relative_to, Any, (Any,), EscapeAnalysis)::Modul # imports import ._TOP_MOD: ==, getindex, setindex! # usings -using Core: - MethodMatch, SimpleVector, - arrayref, arrayset, arraysize, ifelse, sizeof +using Core: MethodMatch, SimpleVector, ifelse, sizeof using Core.IR using ._TOP_MOD: # Base definitions @__MODULE__, @assert, @eval, @goto, @inbounds, @inline, @label, @noinline, @@ -27,8 +25,7 @@ using ._TOP_MOD: # Base definitions unwrap_unionall, !, !=, !==, &, *, +, -, :, <, <<, =>, >, |, ∈, ∉, ∩, ∪, ≠, ≤, ≥, ⊆ using Core.Compiler: # Core.Compiler specific definitions Bottom, IRCode, IR_FLAG_NOTHROW, InferenceResult, SimpleInferenceLattice, - alloc_array_ndims, argextype, array_builtin_common_typecheck, arrayset_typecheck, - check_effect_free!, fieldcount_noerror, hasintersect, intrinsic_nothrow, + argextype, check_effect_free!, fieldcount_noerror, hasintersect, intrinsic_nothrow, is_meta_expr_head, isbitstype, isexpr, println, setfield!_nothrow, singleton_type, try_compute_field, try_compute_fieldidx, widenconst, ⊑ @@ -761,39 +758,6 @@ function compute_frameinfo(ir::IRCode) leave_block = stmt.args[1]::Int leave_pc = first(ir.cfg.blocks[leave_block].stmts) push!(tryregions, idx:leave_pc) - elseif isexpr(stmt, :foreigncall) - args = stmt.args - name = args[1] - nn = normalize(name) - isa(nn, Symbol) || @goto next_stmt - ndims = alloc_array_ndims(nn) - ndims === nothing && @goto next_stmt - if ndims ≠ 0 - length(args) ≥ ndims+6 || @goto next_stmt - dims = Int[] - for i in 1:ndims - dim = argextype(args[i+6], ir) - isa(dim, Const) || @goto next_stmt - dim = dim.val - isa(dim, Int) || @goto next_stmt - push!(dims, dim) - end - else - length(args) ≥ 7 || @goto next_stmt - dims = argextype(args[7], ir) - if isa(dims, Const) - dims = dims.val - isa(dims, Tuple{Vararg{Int}}) || @goto next_stmt - dims = collect(Int, dims) - else - dims === Tuple{} || @goto next_stmt - dims = Int[] - end - end - if arrayinfo === nothing - arrayinfo = ArrayInfo() - end - arrayinfo[idx] = dims elseif arrayinfo !== nothing # TODO this super limited alias analysis is able to handle only very simple cases # this should be replaced with a proper forward dimension analysis @@ -1188,18 +1152,9 @@ function escape_foreigncall!(astate::AnalysisState, pc::Int, args::Vector{Any}) name = args[1] nn = normalize(name) if isa(nn, Symbol) - boundserror_ninds = array_resize_info(nn) - if boundserror_ninds !== nothing - boundserror, ninds = boundserror_ninds - escape_array_resize!(boundserror, ninds, astate, pc, args) - return - end if is_array_copy(nn) escape_array_copy!(astate, pc, args) return - elseif is_array_isassigned(nn) - escape_array_isassigned!(astate, pc, args) - return end # if nn === :jl_gc_add_finalizer_th # # TODO add `FinalizerEscape` ? @@ -1535,139 +1490,6 @@ function escape_builtin!(::typeof(setfield!), astate::AnalysisState, pc::Int, ar end end -function escape_builtin!(::typeof(arrayref), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) ≥ 4 || return false - # check potential thrown escapes from this arrayref call - argtypes = Any[argextype(args[i], astate.ir) for i in 2:length(args)] - boundcheckt = argtypes[1] - aryt = argtypes[2] - if !array_builtin_common_typecheck(boundcheckt, aryt, argtypes, 3) - add_thrown_escapes!(astate, pc, args, 2) - end - ary = args[3] - inbounds = isa(boundcheckt, Const) && !boundcheckt.val::Bool - inbounds || add_escape_change!(astate, ary, ThrownEscape(pc)) - # we don't track precise index information about this array and thus don't know what values - # can be referenced here: directly propagate the escape information imposed on the return - # value of this `arrayref` call to the array itself as the most conservative propagation - # but also with updated index information - estate = astate.estate - if isa(ary, SSAValue) || isa(ary, Argument) - aryinfo = estate[ary] - else - # unanalyzable object, so the return value is also unanalyzable - add_escape_change!(astate, SSAValue(pc), ⊤) - return true - end - AliasInfo = aryinfo.AliasInfo - if isa(AliasInfo, Bool) - AliasInfo && @goto conservative_propagation - # AliasInfo of this array hasn't been analyzed yet: set AliasInfo now - idx = array_nd_index(astate, ary, args[4:end]) - if isa(idx, Int) - AliasInfo = IndexableElements(IdDict{Int,AInfo}()) - @goto record_indexable_use - end - AliasInfo = Unindexable() - @goto record_unindexable_use - elseif isa(AliasInfo, IndexableElements) - idx = array_nd_index(astate, ary, args[4:end]) - if !isa(idx, Int) - AliasInfo = merge_to_unindexable(AliasInfo) - @goto record_unindexable_use - end - @label record_indexable_use - info = get!(()->AInfo(), AliasInfo.infos, idx) - push!(info, LocalUse(pc)) - add_escape_change!(astate, ary, EscapeInfo(aryinfo, AliasInfo)) # update with new AliasInfo - elseif isa(AliasInfo, Unindexable) - @label record_unindexable_use - push!(AliasInfo.info, LocalUse(pc)) - add_escape_change!(astate, ary, EscapeInfo(aryinfo, AliasInfo)) # update with new AliasInfo - else - # this object has been used as struct, but it is used as array here (thus should throw) - # update ary's element information and just handle this case conservatively - aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) - @label conservative_propagation - # at the extreme case, an element of `ary` may point to `ary` itself - # so add the alias change here as the most conservative propagation - add_alias_change!(astate, ary, SSAValue(pc)) - end - return true -end - -function escape_builtin!(::typeof(arrayset), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) ≥ 5 || return false - # check potential escapes from this arrayset call - # NOTE here we essentially only need to account for TypeError, assuming that - # UndefRefError or BoundsError don't capture any of the arguments here - argtypes = Any[argextype(args[i], astate.ir) for i in 2:length(args)] - boundcheckt = argtypes[1] - aryt = argtypes[2] - valt = argtypes[3] - if !(array_builtin_common_typecheck(boundcheckt, aryt, argtypes, 4) && - arrayset_typecheck(aryt, valt)) - add_thrown_escapes!(astate, pc, args, 2) - end - ary = args[3] - val = args[4] - inbounds = isa(boundcheckt, Const) && !boundcheckt.val::Bool - inbounds || add_escape_change!(astate, ary, ThrownEscape(pc)) - # we don't track precise index information about this array and won't record what value - # is being assigned here: directly propagate the escape information of this array to - # the value being assigned as the most conservative propagation - estate = astate.estate - if isa(ary, SSAValue) || isa(ary, Argument) - aryinfo = estate[ary] - else - # unanalyzable object (e.g. obj::GlobalRef): escape field value conservatively - add_escape_change!(astate, val, ⊤) - return true - end - AliasInfo = aryinfo.AliasInfo - if isa(AliasInfo, Bool) - AliasInfo && @goto conservative_propagation - # AliasInfo of this array hasn't been analyzed yet: set AliasInfo now - idx = array_nd_index(astate, ary, args[5:end]) - if isa(idx, Int) - AliasInfo = IndexableElements(IdDict{Int,AInfo}()) - @goto escape_indexable_def - end - AliasInfo = Unindexable() - @goto escape_unindexable_def - elseif isa(AliasInfo, IndexableElements) - idx = array_nd_index(astate, ary, args[5:end]) - if !isa(idx, Int) - AliasInfo = merge_to_unindexable(AliasInfo) - @goto escape_unindexable_def - end - @label escape_indexable_def - info = get!(()->AInfo(), AliasInfo.infos, idx) - add_alias_escapes!(astate, val, info) - push!(info, LocalDef(pc)) - add_escape_change!(astate, ary, EscapeInfo(aryinfo, AliasInfo)) # update with new AliasInfo - # propagate the escape information of this array ignoring elements information - add_escape_change!(astate, val, ignore_aliasinfo(aryinfo)) - elseif isa(AliasInfo, Unindexable) - @label escape_unindexable_def - add_alias_escapes!(astate, val, AliasInfo.info) - push!(AliasInfo.info, LocalDef(pc)) - add_escape_change!(astate, ary, EscapeInfo(aryinfo, AliasInfo)) # update with new AliasInfo - # propagate the escape information of this array ignoring elements information - add_escape_change!(astate, val, ignore_aliasinfo(aryinfo)) - else - # this object has been used as struct, but it is used as array here (thus should throw) - # update ary's element information and just handle this case conservatively - aryinfo = escape_unanalyzable_obj!(astate, ary, aryinfo) - @label conservative_propagation - add_alias_change!(astate, val, ary) - end - # also propagate escape information imposed on the return value of this `arrayset` - ssainfo = estate[SSAValue(pc)] - add_escape_change!(astate, ary, ssainfo) - return true -end - # NOTE this function models and thus should be synced with the implementation of: # size_t array_nd_index(jl_array_t *a, jl_value_t **args, size_t nidxs, ...) function array_nd_index(astate::AnalysisState, @nospecialize(ary), args::Vector{Any}, nidxs::Int = length(args)) @@ -1701,63 +1523,6 @@ function array_nd_index(astate::AnalysisState, @nospecialize(ary), args::Vector{ return i end -function escape_builtin!(::typeof(arraysize), astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) == 3 || return false - ary = args[2] - dim = args[3] - if !arraysize_typecheck(ary, dim, astate.ir) - add_escape_change!(astate, ary, ThrownEscape(pc)) - add_escape_change!(astate, dim, ThrownEscape(pc)) - end - # NOTE we may still see "arraysize: dimension out of range", but it doesn't capture anything - return true -end - -function arraysize_typecheck(@nospecialize(ary), @nospecialize(dim), ir::IRCode) - aryt = argextype(ary, ir) - aryt ⊑ Array || return false - dimt = argextype(dim, ir) - dimt ⊑ Int || return false - return true -end - -# returns nothing if this isn't array resizing operation, -# otherwise returns true if it can throw BoundsError and false if not -function array_resize_info(name::Symbol) - if name === :jl_array_grow_beg || name === :jl_array_grow_end - return false, 1 - elseif name === :jl_array_del_beg || name === :jl_array_del_end - return true, 1 - elseif name === :jl_array_grow_at || name === :jl_array_del_at - return true, 2 - else - return nothing - end -end - -# NOTE may potentially throw "cannot resize array with shared data" error, -# but just ignore it since it doesn't capture anything -function escape_array_resize!(boundserror::Bool, ninds::Int, - astate::AnalysisState, pc::Int, args::Vector{Any}) - length(args) ≥ 6+ninds || return add_fallback_changes!(astate, pc, args) - ary = args[6] - aryt = argextype(ary, astate.ir) - aryt ⊑ Array || return add_fallback_changes!(astate, pc, args) - for i in 1:ninds - ind = args[i+6] - indt = argextype(ind, astate.ir) - indt ⊑ Integer || return add_fallback_changes!(astate, pc, args) - end - if boundserror - # this array resizing can potentially throw `BoundsError`, impose it now - add_escape_change!(astate, ary, ThrownEscape(pc)) - end - # give up indexing analysis whenever we see array resizing - # (since we track array dimensions only globally) - mark_unindexable!(astate, ary) - add_liveness_changes!(astate, pc, args, 6) -end - function mark_unindexable!(astate::AnalysisState, @nospecialize(ary)) isa(ary, SSAValue) || return aryinfo = astate.estate[ary] @@ -1785,53 +1550,4 @@ function escape_array_copy!(astate::AnalysisState, pc::Int, args::Vector{Any}) add_liveness_changes!(astate, pc, args, 6) end -is_array_isassigned(name::Symbol) = name === :jl_array_isassigned - -function escape_array_isassigned!(astate::AnalysisState, pc::Int, args::Vector{Any}) - if !array_isassigned_nothrow(args, astate.ir) - add_thrown_escapes!(astate, pc, args) - end - add_liveness_changes!(astate, pc, args, 6) -end - -function array_isassigned_nothrow(args::Vector{Any}, src::IRCode) - # if !validate_foreigncall_args(args, - # :jl_array_isassigned, Cint, svec(Any,Csize_t), 0, :ccall) - # return false - # end - length(args) ≥ 7 || return false - arytype = argextype(args[6], src) - arytype ⊑ Array || return false - idxtype = argextype(args[7], src) - idxtype ⊑ Csize_t || return false - return true -end - -# # COMBAK do we want to enable this (and also backport this to Base for array allocations?) -# using Core.Compiler: Cint, svec -# function validate_foreigncall_args(args::Vector{Any}, -# name::Symbol, @nospecialize(rt), argtypes::SimpleVector, nreq::Int, convention::Symbol) -# length(args) ≥ 5 || return false -# normalize(args[1]) === name || return false -# args[2] === rt || return false -# args[3] === argtypes || return false -# args[4] === vararg || return false -# normalize(args[5]) === convention || return false -# return true -# end -# -# using Core: ImmutableArray, arrayfreeze, mutating_arrayfreeze, arraythaw -# -# escape_builtin!(::typeof(arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = -# is_safe_immutable_array_op(Array, astate, args) -# escape_builtin!(::typeof(mutating_arrayfreeze), astate::AnalysisState, pc::Int, args::Vector{Any}) = -# is_safe_immutable_array_op(Array, astate, args) -# escape_builtin!(::typeof(arraythaw), astate::AnalysisState, pc::Int, args::Vector{Any}) = -# is_safe_immutable_array_op(ImmutableArray, astate, args) -# function is_safe_immutable_array_op(@nospecialize(arytype), astate::AnalysisState, args::Vector{Any}) -# length(args) == 2 || return false -# argextype(args[2], astate.ir) ⊑ arytype || return false -# return true -# end - end # baremodule EscapeAnalysis diff --git a/base/compiler/ssair/inlining.jl b/base/compiler/ssair/inlining.jl index c681522745a9b..6a8fa23f69c66 100644 --- a/base/compiler/ssair/inlining.jl +++ b/base/compiler/ssair/inlining.jl @@ -763,7 +763,7 @@ function rewrite_apply_exprargs!(todo::Vector{Pair{Int,Any}}, ti = ti.parameters[2]::DataType # checked by `is_valid_type_for_apply_rewrite` end for p in ti.parameters - if isa(p, DataType) && isdefined(p, :instance) + if issingletontype(p) # replace singleton types with their equivalent Const object p = Const(p.instance) elseif isconstType(p) diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 2f0b6711cf995..d5e60ee6bc991 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -303,7 +303,6 @@ end add_tfunc(Core.Intrinsics.cglobal, 1, 2, cglobal_tfunc, 5) add_tfunc(Core.Intrinsics.have_fma, 1, 1, @nospecs((𝕃::AbstractLattice, x)->Bool), 1) -add_tfunc(Core.Intrinsics.arraylen, 1, 1, @nospecs((𝕃::AbstractLattice, x)->Int), 4) # builtin functions # ================= @@ -488,8 +487,8 @@ function sizeof_nothrow(@nospecialize(x)) end function _const_sizeof(@nospecialize(x)) - # Constant Vector does not have constant size - isa(x, Vector) && return Int + # Constant GenericMemory does not have constant size + isa(x, GenericMemory) && return Int size = try Core.sizeof(x) catch ex @@ -612,22 +611,6 @@ end end add_tfunc(Core._typevar, 3, 3, typevar_tfunc, 100) -@nospecs function arraysize_tfunc(𝕃::AbstractLattice, ary, dim) - hasintersect(widenconst(ary), Array) || return Bottom - hasintersect(widenconst(dim), Int) || return Bottom - return Int -end -add_tfunc(arraysize, 2, 2, arraysize_tfunc, 4) - -@nospecs function arraysize_nothrow(ary, dim) - ary ⊑ Array || return false - if isa(dim, Const) - dimval = dim.val - return isa(dimval, Int) && dimval > 0 - end - return false -end - struct MemoryOrder x::Cint end const MEMORY_ORDER_UNSPECIFIED = MemoryOrder(-2) const MEMORY_ORDER_INVALID = MemoryOrder(-1) @@ -1982,34 +1965,6 @@ function tuple_tfunc(𝕃::AbstractLattice, argtypes::Vector{Any}) return anyinfo ? PartialStruct(typ, argtypes) : typ end -@nospecs function arrayref_tfunc(𝕃::AbstractLattice, boundscheck, ary, idxs...) - return _arrayref_tfunc(𝕃, boundscheck, ary, idxs) -end -@nospecs function _arrayref_tfunc(𝕃::AbstractLattice, boundscheck, ary, @nospecialize idxs::Tuple) - isempty(idxs) && return Bottom - array_builtin_common_errorcheck(boundscheck, ary, idxs) || return Bottom - return array_elmtype(ary) -end -add_tfunc(arrayref, 3, INT_INF, arrayref_tfunc, 20) -add_tfunc(const_arrayref, 3, INT_INF, arrayref_tfunc, 20) - -@nospecs function arrayset_tfunc(𝕃::AbstractLattice, boundscheck, ary, item, idxs...) - hasintersect(widenconst(item), _arrayref_tfunc(𝕃, boundscheck, ary, idxs)) || return Bottom - return ary -end -add_tfunc(arrayset, 4, INT_INF, arrayset_tfunc, 20) - -@nospecs function array_builtin_common_errorcheck(boundscheck, ary, @nospecialize idxs::Tuple) - hasintersect(widenconst(boundscheck), Bool) || return false - hasintersect(widenconst(ary), Array) || return false - for i = 1:length(idxs) - idx = getfield(idxs, i) - idx = isvarargtype(idx) ? unwrapva(idx) : widenconst(idx) - hasintersect(idx, Int) || return false - end - return true -end - function array_elmtype(@nospecialize ary) a = widenconst(ary) if !has_free_typevars(a) && a <: Array @@ -2026,6 +1981,101 @@ function array_elmtype(@nospecialize ary) return Any end +@nospecs function genericmemory_builtin_common_errorcheck(boundscheck, mem, idx) + hasintersect(widenconst(boundscheck), Bool) || return false + hasintersect(widenconst(mem), GenericMemory) || return false + hasintersect(widenconst(idx), Int) || return false + return true +end + +function genericmemory_elemtype(@nospecialize mem) + m = widenconst(mem) + if !has_free_typevars(m) && m <: GenericMemory + m0 = m + if isa(m, UnionAll) + m = unwrap_unionall(m0) + end + if isa(m, DataType) + T = m.parameters[2] + valid_as_lattice(T) || return Bottom + return rewrap_unionall(T, m0) + end + end + return Any +end + +@nospecs function memoryrefget_tfunc(𝕃::AbstractLattice, mem, order, boundscheck) + return _memoryrefget_tfunc(𝕃, mem, order, boundscheck) +end +@nospecs function _memoryrefget_tfunc(𝕃::AbstractLattice, mem, order, boundscheck) + memoryref_builtin_common_errorcheck(mem, order, boundscheck) || return Bottom + return memoryref_elemtype(mem) +end +add_tfunc(memoryrefget, 3, 3, memoryrefget_tfunc, 20) + +@nospecs function memoryrefset_tfunc(𝕃::AbstractLattice, mem, item, order, boundscheck) + hasintersect(widenconst(item), _memoryrefget_tfunc(𝕃, mem, order, boundscheck)) || return Bottom + return mem +end +add_tfunc(memoryrefset, 4, 4, memoryrefset_tfunc, 20) + +@nospecs function memoryref_isassigned_tfunc(𝕃::AbstractLattice, mem, order, boundscheck) + return _memoryref_isassigned_tfunc(𝕃, mem, order, boundscheck) +end +@nospecs function _memoryref_isassigned_tfunc(𝕃::AbstractLattice, mem, order, boundscheck) + memoryref_builtin_common_errorcheck(mem, order, boundscheck) || return Bottom + return Bool +end +add_tfunc(memoryref_isassigned, 3, 3, memoryref_isassigned_tfunc, 20) + +@nospecs function memoryref_tfunc(𝕃::AbstractLattice, mem) + a = widenconst(mem) + if !has_free_typevars(a) + unw = unwrap_unionall(a) + if isa(unw, DataType) && unw.name === GenericMemory.body.body.name + A = unw.parameters[1] + T = unw.parameters[2] + T isa Type || T isa TypeVar || return Bottom + return rewrap_unionall(MemoryRef{A, T}, a) + end + end + return MemoryRef +end +@nospecs function memoryref_tfunc(𝕃::AbstractLattice, ref, idx) + memoryref_tfunc(𝕃, ref, idx, Const(true)) +end +@nospecs function memoryref_tfunc(𝕃::AbstractLattice, ref, idx, boundscheck) + memoryref_builtin_common_errorcheck(ref, Const(:not_atomic), boundscheck) || return Bottom + hasintersect(widenconst(idx), Int) || return Bottom + return ref +end +add_tfunc(memoryref, 1, 3, memoryref_tfunc, 1) + + +@nospecs function memoryref_builtin_common_errorcheck(mem, order, boundscheck) + hasintersect(widenconst(mem), MemoryRef) || return false + hasintersect(widenconst(order), Symbol) || return false + hasintersect(widenconst(boundscheck), Bool) || return false + return true +end + +function memoryref_elemtype(@nospecialize mem) + m = widenconst(mem) + if !has_free_typevars(m) && m <: MemoryRef + m0 = m + if isa(m, UnionAll) + m = unwrap_unionall(m0) + end + if isa(m, DataType) + T = m.parameters[2] + valid_as_lattice(T) || return Bottom + return rewrap_unionall(T, m0) + end + end + return Any +end + + @nospecs function opaque_closure_tfunc(𝕃::AbstractLattice, arg, lb, ub, source, env::Vector{Any}, linfo::MethodInstance) argt, argt_exact = instanceof_tfunc(arg) lbt, lb_exact = instanceof_tfunc(lb) @@ -2055,16 +2105,27 @@ function array_type_undefable(@nospecialize(arytype)) end end -function array_builtin_common_nothrow(argtypes::Vector{Any}, isarrayref::Bool) - first_idx_idx = isarrayref ? 3 : 4 - length(argtypes) ≥ first_idx_idx || return false +@nospecs function array_builtin_common_typecheck(boundscheck, arytype, + argtypes::Vector{Any}, first_idx_idx::Int) + (boundscheck ⊑ Bool && arytype ⊑ Array) || return false + for i = first_idx_idx:length(argtypes) + argtypes[i] ⊑ Int || return false + end + return true +end + +function genericmemory_builtin_common_nothrow(argtypes::Vector{Any}, ismemoryref::Bool) + idx_idx = ismemoryref ? 3 : 4 + length(argtypes) == idx_idx || return false boundscheck = argtypes[1] - arytype = argtypes[2] - array_builtin_common_typecheck(boundscheck, arytype, argtypes, first_idx_idx) || return false - if isarrayref + memtype = widenconst(argtypes[2]) + genericmemory_builtin_common_typecheck(boundscheck, memtype, argtypes[idx_idx]) || return false + if ismemoryref # If we could potentially throw undef ref errors, bail out now. - arytype = widenconst(arytype) - array_type_undefable(arytype) && return false + array_type_undefable(memtype) && return false + else + # Additionally check element type compatibility + memoryset_typecheck(memtype, argtypes[3]) || return false end # If we have @inbounds (first argument is false), we're allowed to assume # we don't throw bounds errors. @@ -2072,40 +2133,92 @@ function array_builtin_common_nothrow(argtypes::Vector{Any}, isarrayref::Bool) boundscheck.val::Bool || return true end # Else we can't really say anything here - # TODO: In the future we may be able to track the shapes of arrays though - # inference. + # TODO: In the future we may be able to track the minimum length though inference. return false end -@nospecs function array_builtin_common_typecheck(boundscheck, arytype, - argtypes::Vector{Any}, first_idx_idx::Int) - (boundscheck ⊑ Bool && arytype ⊑ Array) || return false - for i = first_idx_idx:length(argtypes) - argtypes[i] ⊑ Int || return false - end - return true +@nospecs function genericmemory_builtin_common_typecheck(boundscheck, memtype, idx) + return boundscheck ⊑ Bool && memtype ⊑ GenericMemory && idx ⊑ Int end -@nospecs function arrayset_typecheck(arytype, elmtype) +@nospecs function memoryset_typecheck(memtype, elemtype) # Check that we can determine the element type - arytype = widenconst(arytype) - isa(arytype, DataType) || return false - elmtype_expected = arytype.parameters[1] - isa(elmtype_expected, Type) || return false + isa(memtype, DataType) || return false + elemtype_expected = memtype.parameters[1] + isa(elemtype_expected, Type) || return false # Check that the element type is compatible with the element we're assigning - elmtype ⊑ elmtype_expected || return false + elemtype ⊑ elemtype_expected || return false return true end +function memoryref_builtin_common_nothrow(argtypes::Vector{Any}) + if length(argtypes) == 1 + memtype = widenconst(argtypes[1]) + return memtype ⊑ GenericMemory + else + if length(argtypes) == 2 + boundscheck = Const(true) + elseif length(argtypes) == 3 + boundscheck = argtypes[3] + else + return false + end + memtype = widenconst(argtypes[1]) + idx = widenconst(argtypes[2]) + idx ⊑ Int || return false + boundscheck ⊑ Bool || return false + memtype ⊑ MemoryRef || return false + # If we have @inbounds (last argument is false), we're allowed to assume + # we don't throw bounds errors. + if isa(boundscheck, Const) + boundscheck.val::Bool || return true + end + # Else we can't really say anything here + # TODO: In the future we may be able to track the minimum length though inference. + return false + end +end + +function memoryref_builtin_common_nothrow(argtypes::Vector{Any}, ismemoryref::Bool) + nargs = ismemoryref ? 3 : 4 + length(argtypes) == nargs || return false + order = argtypes[3 - ismemoryref] + boundscheck = argtypes[4 - ismemoryref] + memtype = widenconst(argtypes[1]) + memoryref_builtin_common_typecheck(boundscheck, memtype, order) || return false + if ismemoryref + # If we could potentially throw undef ref errors, bail out now. + array_type_undefable(memtype) && return false + else + # Additionally check element type compatibility + memoryset_typecheck(memtype, argtypes[2]) || return false + end + # If we have @inbounds (last argument is false), we're allowed to assume + # we don't throw bounds errors. + if isa(boundscheck, Const) + boundscheck.val::Bool || return true + end + # Else we can't really say anything here + # TODO: In the future we may be able to track the minimum length though inference. + return false +end + +@nospecs function memoryref_builtin_common_typecheck(boundscheck, memtype, order) + return boundscheck ⊑ Bool && memtype ⊑ MemoryRef && order ⊑ Symbol +end + + # Query whether the given builtin is guaranteed not to throw given the argtypes @nospecs function _builtin_nothrow(𝕃::AbstractLattice, f, argtypes::Vector{Any}, rt) ⊑ = Core.Compiler.:⊑(𝕃) - if f === arrayset - array_builtin_common_nothrow(argtypes, #=isarrayref=#false) || return false - # Additionally check element type compatibility - return arrayset_typecheck(argtypes[2], argtypes[3]) - elseif f === arrayref || f === const_arrayref - return array_builtin_common_nothrow(argtypes, #=isarrayref=#true) + if f === memoryref + return memoryref_builtin_common_nothrow(argtypes) + elseif f === memoryrefset + return memoryref_builtin_common_nothrow(argtypes, #=ismemoryref=#false) + elseif f === memoryrefget + return memoryref_builtin_common_nothrow(argtypes, #=ismemoryref=#true) + elseif f === memoryref_isassigned + return memoryref_builtin_common_nothrow(argtypes, #=ismemoryref=#true) elseif f === Core._expr length(argtypes) >= 1 || return false return argtypes[1] ⊑ Symbol @@ -2115,10 +2228,7 @@ end # the correct number of arguments. na = length(argtypes) (na ≠ 0 && isvarargtype(argtypes[end])) && return false - if f === arraysize - na == 2 || return false - return arraysize_nothrow(argtypes[1], argtypes[2]) - elseif f === Core._typevar + if f === Core._typevar na == 3 || return false return typevar_nothrow(𝕃, argtypes[1], argtypes[2], argtypes[3]) elseif f === invoke @@ -2221,9 +2331,9 @@ const _EFFECT_FREE_BUILTINS = [ isa, UnionAll, getfield, - arrayref, - arraysize, - const_arrayref, + memoryref, + memoryrefget, + memoryref_isassigned, isdefined, Core.sizeof, Core.ifelse, @@ -2244,7 +2354,6 @@ const _INACCESSIBLEMEM_BUILTINS = Any[ svec, fieldtype, isa, - isdefined, nfields, throw, tuple, @@ -2256,9 +2365,10 @@ const _INACCESSIBLEMEM_BUILTINS = Any[ ] const _ARGMEM_BUILTINS = Any[ - arrayref, - arrayset, - arraysize, + memoryref, + memoryrefget, + memoryref_isassigned, + memoryrefset, modifyfield!, replacefield!, setfield!, @@ -2406,8 +2516,6 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argin return getfield_effects(𝕃, arginfo, rt) end - # TODO taint `:noub` for `arrayref` and `arrayset` here - # if this builtin call deterministically throws, # don't bother to taint the other effects other than :nothrow: # note this is safe only if we accounted for :noub already @@ -2431,14 +2539,16 @@ function builtin_effects(𝕃::AbstractLattice, @nospecialize(f::Builtin), argin else if contains_is(_CONSISTENT_BUILTINS, f) consistent = ALWAYS_TRUE - elseif f === arrayref || f === arrayset || f === arraysize + elseif f === memoryref + consistent = ALWAYS_TRUE + elseif f === memoryrefget || f === memoryrefset || f === memoryref_isassigned consistent = CONSISTENT_IF_INACCESSIBLEMEMONLY elseif f === Core._typevar consistent = CONSISTENT_IF_NOTRETURNED else consistent = ALWAYS_FALSE end - if f === setfield! || f === arrayset + if f === setfield! || f === memoryrefset effect_free = EFFECT_FREE_IF_INACCESSIBLEMEMONLY elseif contains_is(_EFFECT_FREE_BUILTINS, f) || contains_is(_PURE_BUILTINS, f) effect_free = ALWAYS_TRUE @@ -2567,9 +2677,6 @@ function intrinsic_nothrow(f::IntrinsicFunction, argtypes::Vector{Any}) isprimitivetype(eT) || return false return argtypes[2] ⊑ eT && argtypes[3] ⊑ Int && argtypes[4] ⊑ Int end - if f === Intrinsics.arraylen - return argtypes[1] ⊑ Array - end if f === Intrinsics.bitcast ty, isexact, isconcrete = instanceof_tfunc(argtypes[1]) xty = widenconst(argtypes[2]) @@ -2607,7 +2714,6 @@ function is_pure_intrinsic_infer(f::IntrinsicFunction) return !(f === Intrinsics.pointerref || # this one is volatile f === Intrinsics.pointerset || # this one is never effect-free f === Intrinsics.llvmcall || # this one is never effect-free - f === Intrinsics.arraylen || # this one is volatile f === Intrinsics.sqrt_llvm_fast || # this one may differ at runtime (by a few ulps) f === Intrinsics.have_fma || # this one depends on the runtime environment f === Intrinsics.cglobal) # cglobal lookup answer changes at runtime @@ -2628,18 +2734,12 @@ function intrinsic_effects(f::IntrinsicFunction, argtypes::Vector{Any}) if contains_is(_INCONSISTENT_INTRINSICS, f) consistent = ALWAYS_FALSE - elseif f === arraylen - consistent = CONSISTENT_IF_INACCESSIBLEMEMONLY else consistent = ALWAYS_TRUE end effect_free = !(f === Intrinsics.pointerset) ? ALWAYS_TRUE : ALWAYS_FALSE nothrow = (isempty(argtypes) || !isvarargtype(argtypes[end])) && intrinsic_nothrow(f, argtypes) - if f === arraylen - inaccessiblememonly = INACCESSIBLEMEM_OR_ARGMEMONLY - else - inaccessiblememonly = ALWAYS_TRUE - end + inaccessiblememonly = ALWAYS_TRUE return Effects(EFFECTS_TOTAL; consistent, effect_free, nothrow, inaccessiblememonly) end @@ -2908,90 +3008,29 @@ function foreigncall_effects(@specialize(abstract_eval), e::Expr) args = e.args name = args[1] isa(name, QuoteNode) && (name = name.value) - isa(name, Symbol) || return EFFECTS_UNKNOWN - ndims = alloc_array_ndims(name) - if ndims !== nothing - if ndims ≠ 0 - return alloc_array_effects(abstract_eval, args, ndims) - else - return new_array_effects(abstract_eval, args) - end - end - if is_array_resize(name) - return array_resize_effects() + if name === :jl_alloc_genericmemory + nothrow = new_genericmemory_nothrow(abstract_eval, args) + return Effects(EFFECTS_TOTAL; consistent=CONSISTENT_IF_NOTRETURNED, nothrow) end return EFFECTS_UNKNOWN end -function is_array_resize(name::Symbol) - return name === :jl_array_grow_beg || name === :jl_array_grow_end || - name === :jl_array_del_beg || name === :jl_array_del_end || - name === :jl_array_grow_at || name === :jl_array_del_at -end - -function array_resize_effects() - return Effects(EFFECTS_TOTAL; - effect_free = EFFECT_FREE_IF_INACCESSIBLEMEMONLY, - nothrow = false, - inaccessiblememonly = INACCESSIBLEMEM_OR_ARGMEMONLY) -end - -function alloc_array_ndims(name::Symbol) - if name === :jl_alloc_array_1d - return 1 - elseif name === :jl_alloc_array_2d - return 2 - elseif name === :jl_alloc_array_3d - return 3 - elseif name === :jl_new_array - return 0 - end - return nothing -end - -function alloc_array_effects(@specialize(abstract_eval), args::Vector{Any}, ndims::Int) - nothrow = alloc_array_nothrow(abstract_eval, args, ndims) - return Effects(EFFECTS_TOTAL; consistent=CONSISTENT_IF_NOTRETURNED, nothrow) -end - -function alloc_array_nothrow(@specialize(abstract_eval), args::Vector{Any}, ndims::Int) - length(args) ≥ ndims+FOREIGNCALL_ARG_START || return false - atype = instanceof_tfunc(abstract_eval(args[FOREIGNCALL_ARG_START]))[1] - dims = Csize_t[] - for i in 1:ndims - dim = abstract_eval(args[i+FOREIGNCALL_ARG_START]) - isa(dim, Const) || return false - dimval = dim.val - isa(dimval, Int) || return false - push!(dims, reinterpret(Csize_t, dimval)) - end - return _new_array_nothrow(atype, ndims, dims) -end - -function new_array_effects(@specialize(abstract_eval), args::Vector{Any}) - nothrow = new_array_nothrow(abstract_eval, args) - return Effects(EFFECTS_TOTAL; consistent=CONSISTENT_IF_NOTRETURNED, nothrow) -end - -function new_array_nothrow(@specialize(abstract_eval), args::Vector{Any}) - length(args) ≥ FOREIGNCALL_ARG_START+1 || return false - atype = instanceof_tfunc(abstract_eval(args[FOREIGNCALL_ARG_START]))[1] - dims = abstract_eval(args[FOREIGNCALL_ARG_START+1]) - isa(dims, Const) || return dims === Tuple{} - dimsval = dims.val - isa(dimsval, Tuple{Vararg{Int}}) || return false - ndims = nfields(dimsval) - isa(ndims, Int) || return false - dims = Csize_t[reinterpret(Csize_t, dimval) for dimval in dimsval] - return _new_array_nothrow(atype, ndims, dims) -end - -function _new_array_nothrow(@nospecialize(atype), ndims::Int, dims::Vector{Csize_t}) - isa(atype, DataType) || return false - eltype = atype.parameters[1] - iskindtype(typeof(eltype)) || return false - elsz = aligned_sizeof(eltype) - return ccall(:jl_array_validate_dims, Cint, - (Ptr{Csize_t}, Ptr{Csize_t}, UInt32, Ptr{Csize_t}, Csize_t), - #=nel=#RefValue{Csize_t}(), #=tot=#RefValue{Csize_t}(), ndims, dims, elsz) == 0 +function new_genericmemory_nothrow(@nospecialize(abstract_eval), args::Vector{Any}) + length(args) ≥ 1+FOREIGNCALL_ARG_START || return false + mtype = instanceof_tfunc(abstract_eval(args[FOREIGNCALL_ARG_START]))[1] + isa(mtype, DataType) || return false + isdefined(mtype, :instance) || return false + elsz = Int(datatype_layoutsize(mtype)) + arrayelem = datatype_arrayelem(mtype) + dim = abstract_eval(args[1+FOREIGNCALL_ARG_START]) + isa(dim, Const) || return false + dimval = dim.val + isa(dimval, Int) || return false + 0 < dimval < typemax(Int) || return false + tot, ovflw = Intrinsics.checked_smul_int(dimval, elsz) + ovflw && return false + isboxed = 1; isunion = 2 + tot, ovflw = Intrinsics.checked_sadd_int(tot, arrayelem == isunion ? 1 + dimval : 1) + ovflw && return false + return true end diff --git a/base/compiler/typelattice.jl b/base/compiler/typelattice.jl index 324f2b600cc44..a5c79932401fd 100644 --- a/base/compiler/typelattice.jl +++ b/base/compiler/typelattice.jl @@ -323,7 +323,7 @@ end @nospecializeinfer function isalreadyconst(@nospecialize t) isa(t, Const) && return true - isa(t, DataType) && isdefined(t, :instance) && return true + issingletontype(t) && return true return isconstType(t) end diff --git a/base/compiler/utilities.jl b/base/compiler/utilities.jl index bdc0156efd824..a6cf08326e789 100644 --- a/base/compiler/utilities.jl +++ b/base/compiler/utilities.jl @@ -91,7 +91,7 @@ function count_const_size(@nospecialize(x), count_self::Bool = true) sz = count_self ? sizeof(dt) : 0 sz > MAX_INLINE_CONST_SIZE && return MAX_INLINE_CONST_SIZE + 1 dtfd = DataTypeFieldDesc(dt) - for i = 1:nfields(x) + for i = 1:datatype_nfields(dt) isdefined(x, i) || continue f = getfield(x, i) if !dtfd[i].isptr && datatype_pointerfree(typeof(f)) diff --git a/base/deepcopy.jl b/base/deepcopy.jl index eae8974326d06..459e2aec74148 100644 --- a/base/deepcopy.jl +++ b/base/deepcopy.jl @@ -85,30 +85,51 @@ function deepcopy_internal(@nospecialize(x), stackdict::IdDict) return y::T end -function deepcopy_internal(x::Array, stackdict::IdDict) +function deepcopy_internal(x::Memory, stackdict::IdDict) if haskey(stackdict, x) return stackdict[x]::typeof(x) end - _deepcopy_array_t(x, eltype(x), stackdict) + _deepcopy_memory_t(x, eltype(x), stackdict) end -function _deepcopy_array_t(@nospecialize(x::Array), T, stackdict::IdDict) +function _deepcopy_memory_t(@nospecialize(x::Memory), T, stackdict::IdDict) if isbitstype(T) return (stackdict[x]=copy(x)) end - dest = similar(x) + dest = typeof(x)(undef, length(x)) stackdict[x] = dest + xr = Core.memoryref(x) + dr = Core.memoryref(dest) for i = 1:length(x) - if ccall(:jl_array_isassigned, Cint, (Any, Csize_t), x, i-1) != 0 - xi = ccall(:jl_arrayref, Any, (Any, Csize_t), x, i-1) + xi = Core.memoryref(xr, i, false) + if Core.memoryref_isassigned(xi, :not_atomic, false) + xi = Core.memoryrefget(xi, :not_atomic, false) if !isbits(xi) xi = deepcopy_internal(xi, stackdict)::typeof(xi) end - ccall(:jl_arrayset, Cvoid, (Any, Any, Csize_t), dest, xi, i-1) + di = Core.memoryref(dr, i, false) + di = Core.memoryrefset(di, xi, :not_atomic, false) end end return dest end +function deepcopy_internal(x::Array{T, N}, stackdict::IdDict) where {T, N} + if haskey(stackdict, x) + return stackdict[x]::typeof(x) + end + stackdict[x] = Array{T, N}(deepcopy_internal(x.ref, stackdict), x.size) +end +function deepcopy_internal(x::MemoryRef, stackdict::IdDict) + if haskey(stackdict, x) + return stackdict[x]::typeof(x) + end + mem = getfield(x, :mem) + dest = MemoryRef(deepcopy_internal(mem, stackdict)::typeof(mem)) + i = memoffset(x) + i == 0 || (dest = Core.memoryref(dest, i + 1, true)) + return dest +end + function deepcopy_internal(x::Union{Dict,IdDict}, stackdict::IdDict) if haskey(stackdict, x) diff --git a/base/dict.jl b/base/dict.jl index 77272acf98aff..d231b44676c59 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -64,9 +64,9 @@ Dict{String, Int64} with 2 entries: """ mutable struct Dict{K,V} <: AbstractDict{K,V} # Metadata: empty => 0x00, removed => 0x7f, full => 0b1[7 most significant hash bits] - slots::Vector{UInt8} - keys::Array{K,1} - vals::Array{V,1} + slots::Memory{UInt8} + keys::Memory{K} + vals::Memory{V} ndel::Int count::Int age::UInt @@ -75,7 +75,9 @@ mutable struct Dict{K,V} <: AbstractDict{K,V} function Dict{K,V}() where V where K n = 16 - new(zeros(UInt8,n), Vector{K}(undef, n), Vector{V}(undef, n), 0, 0, 0, n, 0) + slots = Memory{UInt8}(undef,n) + fill!(slots, 0x0) + new(slots, Memory{K}(undef, n), Memory{V}(undef, n), 0, 0, 0, n, 0) end function Dict{K,V}(d::Dict{K,V}) where V where K new(copy(d.slots), copy(d.keys), copy(d.vals), d.ndel, d.count, d.age, @@ -179,17 +181,19 @@ end h.age += 1 h.idxfloor = 1 if h.count == 0 - resize!(h.slots, newsz) + # TODO: tryresize + h.slots = Memory{UInt8}(undef, newsz) fill!(h.slots, 0x0) - resize!(h.keys, newsz) - resize!(h.vals, newsz) + h.keys = Memory{K}(undef, newsz) + h.vals = Memory{V}(undef, newsz) h.ndel = 0 return h end - slots = zeros(UInt8,newsz) - keys = Vector{K}(undef, newsz) - vals = Vector{V}(undef, newsz) + slots = Memory{UInt8}(undef, newsz) + fill!(slots, 0x0) + keys = Memory{K}(undef, newsz) + vals = Memory{V}(undef, newsz) age0 = h.age count = 0 maxprobe = 0 @@ -253,10 +257,10 @@ Dict{String, Int64}() function empty!(h::Dict{K,V}) where V where K fill!(h.slots, 0x0) sz = length(h.slots) - empty!(h.keys) - empty!(h.vals) - resize!(h.keys, sz) - resize!(h.vals, sz) + for i in 1:sz + _unsetindex!(h.keys, i) + _unsetindex!(h.vals, i) + end h.ndel = 0 h.count = 0 h.age += 1 diff --git a/base/essentials.jl b/base/essentials.jl index 4323052d540da..fbce16719b22d 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1,17 +1,21 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -import Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, arrayref +import Core: CodeInfo, SimpleVector, donotdelete, compilerbarrier, memoryref, memoryrefget const Callable = Union{Function,Type} const Bottom = Union{} # Define minimal array interface here to help code used in macros: -length(a::Array) = arraylen(a) +length(a::Array{T, 1}) where {T} = getfield(a, :size)[1] +length(a::Array) = getfield(getfield(getfield(a, :ref), :mem), :length) +length(a::GenericMemory) = getfield(a, :length) # This is more complicated than it needs to be in order to get Win64 through bootstrap -eval(:(getindex(A::Array, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1))) -eval(:(getindex(A::Array, i1::Int, i2::Int, I::Int...) = (@inline; arrayref($(Expr(:boundscheck)), A, i1, i2, I...)))) +eval(:(getindex(A::Memory, i::Int) = memoryrefget(MemoryRef(A, i, $(Expr(:boundscheck))), :not_atomic, false))) +eval(:(getindex(A::MemoryRef{:not_atomic}) = memoryrefget(A, :not_atomic, $(Expr(:boundscheck))))) + +# multidimensional getindex will be defined later on ==(a::GlobalRef, b::GlobalRef) = a.mod === b.mod && a.name === b.name @@ -653,9 +657,6 @@ julia> ifelse(1 > 2, 1, 2) """ ifelse(condition::Bool, x, y) = Core.ifelse(condition, x, y) -# simple Array{Any} operations needed for bootstrap -@eval setindex!(A::Array{Any}, @nospecialize(x), i::Int) = arrayset($(Expr(:boundscheck)), A, x, i) - """ esc(e) @@ -771,6 +772,35 @@ macro goto(name::Symbol) return esc(Expr(:symbolicgoto, name)) end +# linear indexing +eval(:(function getindex(A::Array, i::Int) + #=$(Expr(:meta, Expr(:purity, + #=:consistent=#false, + #=:effect_free=#false, + #=:nothrow=#false, + #=:terminates_globally=#false, + #=:terminates_locally=#false, + #=:notaskstate=#false, + #=:inaccessiblememonly=#0x2, # set consistent_if_inaccessiblememonly + #=:noub=#false)))=# + @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, i) + ref = memoryref(A.ref, i, false) + memoryrefget(ref, :not_atomic, false) + end)) +# simple Array{Any} operations needed for bootstrap +function setindex!(A::Array{Any}, @nospecialize(x), i::Int) + @boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, i) + memoryrefset(memoryref(A.ref, i, false), x, :not_atomic, false) +end +@eval setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = memoryrefset(memoryref(memoryref(A), i, $(Expr(:boundscheck))), x, :not_atomic, $(Expr(:boundscheck))) +#@eval setindex!(A::Memory{:not_atomic,Any}, @nospecialize(x), i::Int) = memoryset($(Expr(:boundscheck)), A, x, i) +@eval setindex!(A::MemoryRef{:not_atomic,Any}, @nospecialize(x)) = memoryrefset(A, x, :not_atomic, $(Expr(:boundscheck))) + +#@eval setindex!(A::GenericMemory{true,Any}, @nospecialize(x), order::Symbol, i::Int) = memoryrefset(memoryref(memoryref(A), i, $(Expr(:boundscheck))), x, order, false) +##@eval setindex!(A::GenericMemory{true,Any}, @nospecialize(x), order::Symbol, i::Int) = memoryset($(Expr(:boundscheck)), A, x, i) +#@eval setindex!(A::MemoryRef{true,Any}, @nospecialize(x), order::Symbol) = memoryrefset(A, x, order, $(Expr(:boundscheck))) + + # SimpleVector getindex(v::SimpleVector, i::Int) = (@_foldable_meta; Core._svec_ref(v, i)) diff --git a/base/experimental.jl b/base/experimental.jl index af67d9e019378..996cda2e8782a 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -28,10 +28,11 @@ end Base.IndexStyle(::Type{<:Const}) = IndexLinear() Base.size(C::Const) = size(C.a) Base.axes(C::Const) = axes(C.a) +# TODO: fixme @eval Base.getindex(A::Const, i1::Int) = (Base.@inline; Core.const_arrayref($(Expr(:boundscheck)), A.a, i1)) @eval Base.getindex(A::Const, i1::Int, i2::Int, I::Int...) = - (Base.@inline; Core.const_arrayref($(Expr(:boundscheck)), A.a, i1, i2, I...)) + (Base.@inline; Core.const_arrayref($(Expr(:boundscheck)), A.a, i1, i2, I...)) """ @aliasscope expr diff --git a/base/genericmemory.jl b/base/genericmemory.jl new file mode 100644 index 0000000000000..3d244f72628cf --- /dev/null +++ b/base/genericmemory.jl @@ -0,0 +1,223 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +## genericmemory.jl: Managed Memory + +""" + GenericMemory{isatomic, T} <: AbstractVector{T} + +One-dimensional dense array with elements of type `T`. +""" +GenericMemory +""" + Memory{T} == GenericMemory{:not_atomic, T} + +One-dimensional dense array with elements of type `T`. +""" +Memory + +## Basic functions ## + +using Core: memoryref, memoryrefget, memoryrefset, memoryref_isassigned + +size(a::GenericMemory, d::Int) = + d < 1 ? error("dimension out of range") : + d == 1 ? length(a) : + 1 +size(a::GenericMemory, d::Integer) = size(a, convert(d, Int)) +size(a::GenericMemory) = (length(a),) + +pointer(mem::GenericMemory, i::Int) = unsafe_convert(Ptr{Cvoid}, MemoryRef(mem, i)) +function _unsetindex!(A::Memory{T}, i::Int) where {T} + @_terminates_locally_meta + @inline + @boundscheck checkbounds(A, i) + t = @_gc_preserve_begin A + p = Ptr{Ptr{Cvoid}}(pointer(A, i)) + # TODO(jwn): access datatype_layout from A instead + if !allocatedinline(T) + Intrinsics.atomic_pointerset(p, C_NULL, :monotonic) + elseif T isa DataType + if !datatype_pointerfree(T) + for j = 1:Core.sizeof(Ptr{Cvoid}):Core.sizeof(T) + Intrinsics.atomic_pointerset(p + j - 1, C_NULL, :monotonic) + end + end + end + @_gc_preserve_end t + return A +end + +elsize(@nospecialize _::Type{A}) where {T,A<:GenericMemory{<:Any,T}} = aligned_sizeof(T) +sizeof(a::GenericMemory) = Core.sizeof(a) + +# multi arg case will be overwritten later. This is needed for bootstraping +function isassigned(a::Memory, i::Int) + @inline + @boundscheck 1 <= i <= length(a) || return false + # @boundscheck (i % UInt) - 1 < length(A) || return false + return memoryref_isassigned(MemoryRef(a, i, false), :not_atomic, false) +end + +@eval isassigned(a::MemoryRef) = memoryref_isassigned(a, :not_atomic, $(Expr(:boundscheck))) + +## copy ## +unsafe_copyto!(dest::Memory, doffs, src::Memory, soffs, n) = + _unsafe_copyto!(dest, doffs, src, soffs, n) + +copy(a::T) where {T<:Memory} = ccall(:jl_genericmemory_copy, Ref{T}, (Any,), a) + +## Constructors ## + +similar(a::Memory{T}) where {T} = Memory{T}(undef, length(a)) +similar(a::Memory{T}, S::Type) where {T} = Memory{S}(undef, length(a)) +similar(a::Memory{T}, m::Int) where {T} = Memory{T}(undef, m) +similar(a::Memory, T::Type, dims::Dims{1}) = Memory{T}(undef, dims[1]) +similar(a::Memory{T}, dims::Dims{1}) where {T} = Memory{T}(undef, dims[1]) + +function fill!(a::Union{Memory{UInt8}, Memory{Int8}}, x::Integer) + t = @_gc_preserve_begin a + p = unsafe_convert(Ptr{Cvoid}, a) + T = eltype(a) + memset(p, x isa T ? x : convert(T, x), length(a)) + @_gc_preserve_end t + return a +end + +## Conversions ## + +convert(::Type{T}, a::AbstractArray) where {T<:GenericMemory} = a isa T ? a : T(a)::T + +promote_rule(a::Type{Memory{T}}, b::Type{Memory{S}}) where {T,S} = el_same(promote_type(T,S), a, b) +promote_rule(a::Type{GenericMemory{:atomic,T}}, b::Type{GenericMemory{:atomic,S}}) where {T,S} = el_same(promote_type(T,S), a, b) + +## Constructors ## + +if nameof(@__MODULE__) === :Base # avoid method overwrite +# constructors should make copies +Memory{T}(x::AbstractArray{S,1}) where {T,S} = copyto_axcheck!(Memory{T}(undef, size(x)), x) +end + +## copying iterators to containers + +## Iteration ## + +iterate(A::Memory, i=1) = (@inline; (i % UInt) - 1 < length(A) ? (@inbounds A[i], i + 1) : nothing) + +## Indexing: getindex ## + +# Faster contiguous indexing using copyto! for AbstractUnitRange and Colon +function getindex(A::Memory, I::AbstractUnitRange{<:Integer}) + @inline + @boundscheck checkbounds(A, I) + lI = length(I) + X = similar(A, axes(I)) + if lI > 0 + copyto!(X, firstindex(X), A, first(I), lI) + end + return X +end + +# getindex for carrying out logical indexing for AbstractUnitRange{Bool} as Bool <: Integer +getindex(a::Memory, r::AbstractUnitRange{Bool}) = getindex(a, to_index(r)) + +function getindex(A::Memory, c::Colon) + lI = length(A) + X = similar(A, lI) + if lI > 0 + unsafe_copyto!(X, 1, A, 1, lI) + end + return X +end + +## Indexing: setindex! ## + +@eval begin +function setindex!(A::Memory{T}, x, i1::Int) where {T} + val = x isa T ? x : convert(T,x)::T + ref = MemoryRef(A, i1, $(Expr(:boundscheck))) + memoryrefset(ref, val, :not_atomic, $(Expr(:boundscheck))) +end +end +function setindex!(A::Memory{T}, x, i1::Int, i2::Int, I::Int...) where {T} + @inline + @boundscheck (i2 == 1 && all(==(1), I)) || throw_boundserror(A, (i1, i2, I...)) + setindex!(A, x, i1) +end + +__inbounds_setindex!(A::Memory{T}, x, i1::Int) where {T} = + memoryset(false, A, convert(T,x)::T, i1) +function __inbounds_setindex!(A::Memory{T}, x, i1::Int, i2::Int, I::Int...) where {T} + @inline + @boundscheck (i2 == 1 && all(==(1), I)) || throw_boundserror(A, (i1, i2, I...)) + __inbounds_setindex(A, x, i1) +end + +# Faster contiguous setindex! with copyto! +function setindex!(A::Memory{T}, X::Memory{T}, I::AbstractUnitRange{Int}) where T + @inline + @boundscheck checkbounds(A, I) + lI = length(I) + @boundscheck setindex_shape_check(X, lI) + if lI > 0 + unsafe_copyto!(A, first(I), X, 1, lI) + end + return A +end +function setindex!(A::Memory{T}, X::Memory{T}, c::Colon) where T + @inline + lI = length(A) + @boundscheck setindex_shape_check(X, lI) + if lI > 0 + unsafe_copyto!(A, 1, X, 1, lI) + end + return A +end + +# use memcmp for cmp on byte arrays +function cmp(a::Memory{UInt8}, b::Memory{UInt8}) + ta = @_gc_preserve_begin a + tb = @_gc_preserve_begin b + pa = unsafe_convert(Ptr{Cvoid}, a) + pb = unsafe_convert(Ptr{Cvoid}, b) + c = memcmp(pa, pb, min(length(a),length(b))) + @_gc_preserve_end ta + @_gc_preserve_end tb + return c < 0 ? -1 : c > 0 ? +1 : cmp(length(a),length(b)) +end + +const BitIntegerMemory{N} = Union{map(T->Memory{T}, BitInteger_types)...} +# use memcmp for == on bit integer types +function ==(a::M, b::M) where {M <: BitIntegerMemory} + if length(a) == length(b) + ta = @_gc_preserve_begin a + tb = @_gc_preserve_begin b + pa = unsafe_convert(Ptr{Cvoid}, a) + pb = unsafe_convert(Ptr{Cvoid}, b) + c = memcmp(pa, pb, sizeof(eltype(M)) * length(a)) + @_gc_preserve_end ta + @_gc_preserve_end tb + return c == 0 + else + return false + end +end + +function findall(pred::Fix2{typeof(in),<:Union{Memory{<:Real},Real}}, x::Memory{<:Real}) + if issorted(x, Sort.Forward) && issorted(pred.x, Sort.Forward) + return _sortedfindin(x, pred.x) + else + return _findin(x, pred.x) + end +end + +# Copying subregions +function indcopy(sz::Dims, I::GenericMemory) + n = length(I) + s = sz[n] + for i = n+1:length(sz) + s *= sz[i] + end + dst = eltype(I)[_findin(I[i], i < n ? (1:sz[i]) : (1:s)) for i = 1:n] + src = eltype(I)[I[i][_findin(I[i], i < n ? (1:sz[i]) : (1:s))] for i = 1:n] + dst, src +end diff --git a/base/iddict.jl b/base/iddict.jl index 01ff213305d7b..a4d3d439f03c9 100644 --- a/base/iddict.jl +++ b/base/iddict.jl @@ -24,10 +24,10 @@ IdDict{Any, String} with 3 entries: ``` """ mutable struct IdDict{K,V} <: AbstractDict{K,V} - ht::Vector{Any} + ht::Memory{Any} count::Int ndel::Int - IdDict{K,V}() where {K, V} = new{K,V}(Vector{Any}(undef, 32), 0, 0) + IdDict{K,V}() where {K, V} = new{K,V}(Memory{Any}(undef, 32), 0, 0) function IdDict{K,V}(itr) where {K, V} d = IdDict{K,V}() @@ -69,7 +69,7 @@ end empty(d::IdDict, ::Type{K}, ::Type{V}) where {K, V} = IdDict{K,V}() function rehash!(d::IdDict, newsz = length(d.ht)%UInt) - d.ht = ccall(:jl_idtable_rehash, Vector{Any}, (Any, Csize_t), d.ht, newsz) + d.ht = ccall(:jl_idtable_rehash, Memory{Any}, (Any, Csize_t), d.ht, newsz) d end @@ -93,7 +93,7 @@ function setindex!(d::IdDict{K,V}, @nospecialize(val), @nospecialize(key)) where d.ndel = 0 end inserted = RefValue{Cint}(0) - d.ht = ccall(:jl_eqtable_put, Array{Any,1}, (Any, Any, Any, Ptr{Cint}), d.ht, key, val, inserted) + d.ht = ccall(:jl_eqtable_put, Memory{Any}, (Any, Any, Any, Ptr{Cint}), d.ht, key, val, inserted) d.count += inserted[] return d end @@ -133,7 +133,7 @@ function delete!(d::IdDict{K}, @nospecialize(key)) where K end function empty!(d::IdDict) - resize!(d.ht, 32) + d.ht = Memory{Any}(undef, 32) ht = d.ht t = @_gc_preserve_begin ht memset(unsafe_convert(Ptr{Cvoid}, ht), 0, sizeof(ht)) diff --git a/base/io.jl b/base/io.jl index be43a954c621e..45458991fe018 100644 --- a/base/io.jl +++ b/base/io.jl @@ -800,7 +800,7 @@ function write(s::IO, A::AbstractArray) return nb end -function write(s::IO, a::Array) +function write(s::IO, a::Union{Array,Memory}) if isbitstype(eltype(a)) return GC.@preserve a unsafe_write(s, pointer(a), sizeof(a)) else @@ -866,13 +866,13 @@ end read(s::IO, ::Type{Bool}) = (read(s, UInt8) != 0) read(s::IO, ::Type{Ptr{T}}) where {T} = convert(Ptr{T}, read(s, UInt)) -function read!(s::IO, a::Array{UInt8}) +function read!(s::IO, a::Union{Array{UInt8},Memory{UInt8}}) GC.@preserve a unsafe_read(s, pointer(a), sizeof(a)) return a end function read!(s::IO, a::AbstractArray{T}) where T - if isbitstype(T) && (a isa Array || a isa FastContiguousSubArray{T,<:Any,<:Array{T}}) + if isbitstype(T) && (a isa Array || a isa FastContiguousSubArray{T,<:Any,<:Array{T}} || a isa Memory) GC.@preserve a unsafe_read(s, pointer(a), sizeof(a)) else for i in eachindex(a) diff --git a/base/multidimensional.jl b/base/multidimensional.jl index 93c0ddcedd1f1..cd2d38d7981b7 100644 --- a/base/multidimensional.jl +++ b/base/multidimensional.jl @@ -690,14 +690,6 @@ end @inline checkbounds_indices(::Type{Bool}, IA::Tuple, I::Tuple{CartesianIndex,Vararg{Any}}) = checkbounds_indices(Bool, IA, (I[1].I..., tail(I)...)) -# Indexing into Array with mixtures of Integers and CartesianIndices is -# extremely performance-sensitive. While the abstract fallbacks support this, -# codegen has extra support for SIMDification that sub2ind doesn't (yet) support -@propagate_inbounds getindex(A::Array, i1::Union{Integer, CartesianIndex}, I::Union{Integer, CartesianIndex}...) = - A[to_indices(A, (i1, I...))...] -@propagate_inbounds setindex!(A::Array, v, i1::Union{Integer, CartesianIndex}, I::Union{Integer, CartesianIndex}...) = - (A[to_indices(A, (i1, I...))...] = v; A) - # Support indexing with an array of CartesianIndex{N}s # Here we try to consume N of the indices (if there are that many available) # The first two simply handle ambiguities @@ -719,7 +711,6 @@ end checkindex(Bool, IA1, I[1]) & checkbounds_indices(Bool, IArest, tail(I)) end - @inline function checkbounds_indices(::Type{Bool}, IA::Tuple{}, I::Tuple{AbstractArray{Bool,N},Vararg{Any}}) where N return checkbounds_indices(Bool, IA, (LogicalIndex(I[1]), tail(I)...)) diff --git a/base/pointer.jl b/base/pointer.jl index a47f1e38edb9b..a2ae036712cf5 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -62,10 +62,44 @@ unsafe_convert(::Type{Ptr{Int8}}, s::String) = ccall(:jl_string_ptr, Ptr{Int8}, cconvert(::Type{Ptr{UInt8}}, s::AbstractString) = String(s) cconvert(::Type{Ptr{Int8}}, s::AbstractString) = String(s) -unsafe_convert(::Type{Ptr{T}}, a::Array{T}) where {T} = ccall(:jl_array_ptr, Ptr{T}, (Any,), a) +unsafe_convert(::Type{Ptr{T}}, a::Array) where {T} = unsafe_convert(Ptr{T}, a.ref) unsafe_convert(::Type{Ptr{S}}, a::AbstractArray{T}) where {S,T} = convert(Ptr{S}, unsafe_convert(Ptr{T}, a)) unsafe_convert(::Type{Ptr{T}}, a::AbstractArray{T}) where {T} = error("conversion to pointer not defined for $(typeof(a))") +unsafe_convert(::Type{Ptr{Cvoid}}, a::GenericMemory{T}) where {T} = getfield(a, :data) +unsafe_convert(::Type{Ptr{T}}, a::GenericMemory) where {T} = convert(Ptr{T}, getfield(a, :data)) + +function unsafe_convert(::Type{Ptr{Cvoid}}, a::MemoryRef{<:Any,T}) where {T} + mem = getfield(a, :mem) + offset = getfield(a, :ptr) + MemT = typeof(mem) + arrayelem = datatype_arrayelem(MemT) + elsz = datatype_layoutsize(MemT) + isboxed = 1; isunion = 2 + if arrayelem == isunion || elsz == 0 + offset = UInt(offset) * elsz + offset += unsafe_convert(Ptr{Cvoid}, mem) + end + return offset +end +unsafe_convert(::Type{Ptr{T}}, a::MemoryRef) where {T} = convert(Ptr{T}, unsafe_convert(Ptr{Cvoid}, a)) + +function memoffset(a::MemoryRef{<:Any,T}) where {T} # n.b. offset + 1 = index + mem = getfield(a, :mem) + offset = getfield(a, :ptr) + MemT = typeof(mem) + arrayelem = datatype_arrayelem(MemT) + elsz = datatype_layoutsize(MemT) + isboxed = 1; isunion = 2 + if arrayelem != isunion && elsz != 0 + offset -= unsafe_convert(Ptr{Cvoid}, mem) + offset ÷= elsz + else + offset = UInt(offset) + end + return offset % Int +end + # unsafe pointer to array conversions """ unsafe_wrap(Array, pointer::Ptr{T}, dims; own = false) @@ -92,10 +126,21 @@ function unsafe_wrap(::Union{Type{Array},Type{Array{T}},Type{Array{T,1}}}, ccall(:jl_ptr_to_array_1d, Array{T,1}, (Any, Ptr{Cvoid}, Csize_t, Cint), Array{T,1}, p, d, own) end -unsafe_wrap(Atype::Union{Type{Array},Type{Array{T}},Type{Array{T,N}}}, +function unsafe_wrap(::Union{Type{GenericMemory{isatomic}},Type{GenericMemory{isatomic,T}}}, + p::Ptr{T}, dims::Tuple{Int}; own::Bool = false) where {isatomic,T} + ccall(:jl_ptr_to_genericmemory, Ref{GenericMemory{isatomic,T}}, + (Any, Ptr{Cvoid}, Csize_t, Cint), Memory{isatomic,T}, p, dim[1], own) +end +function unsafe_wrap(::Union{Type{GenericMemory{isatomic}},Type{GenericMemory{isatomic,T}}}, + p::Ptr{T}, d::Integer; own::Bool = false) where {isatomic,T} + ccall(:jl_ptr_to_genericmemory, Ref{GenericMemory{isatomic,T}}, + (Any, Ptr{Cvoid}, Csize_t, Cint), GenericMemory{isatomic,T}, p, d, own) +end +unsafe_wrap(Atype::Union{Type{Array},Type{Array{T}},Type{Array{T,N}},Type{GenericMemory{isatomic}},Type{GenericMemory{isatomic,T}}} where {isatomic}, p::Ptr{T}, dims::NTuple{N,<:Integer}; own::Bool = false) where {T,N} = unsafe_wrap(Atype, p, convert(Tuple{Vararg{Int}}, dims), own = own) + """ unsafe_load(p::Ptr{T}, i::Integer=1) unsafe_load(p::Ptr{T}, order::Symbol) diff --git a/base/promotion.jl b/base/promotion.jl index bc081b15c7a31..a1438267763ae 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -120,7 +120,7 @@ function typejoin(@nospecialize(a), @nospecialize(b)) aprimary = aprimary::UnionAll # pushfirst!(vars, aprimary.var) _growbeg!(vars, 1) - arrayset(false, vars, aprimary.var, 1) + vars[1] = aprimary.var aprimary = aprimary.body end end diff --git a/base/reflection.jl b/base/reflection.jl index 6f4ce493ff7ae..f15d0b5634850 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -427,13 +427,15 @@ struct DataTypeLayout flags::UInt16 # haspadding : 1; # fielddesc_type : 2; + # arrayelem_isboxed : 1; end """ Base.datatype_alignment(dt::DataType) -> Int Memory allocation minimum alignment for instances of this type. -Can be called on any `isconcretetype`. +Can be called on any `isconcretetype`, although for Memory it will give the +alignment of the elements, not the whole object. """ function datatype_alignment(dt::DataType) @_foldable_meta @@ -475,7 +477,8 @@ gc_alignment(T::Type) = gc_alignment(Core.sizeof(T)) Base.datatype_haspadding(dt::DataType) -> Bool Return whether the fields of instances of this type are packed in memory, -with no intervening padding bytes. +with no intervening padding bits (defined as bits whose value does not uniquely +impact the egal test when applied to the struct fields). Can be called on any `isconcretetype`. """ function datatype_haspadding(dt::DataType) @@ -486,9 +489,10 @@ function datatype_haspadding(dt::DataType) end """ - Base.datatype_nfields(dt::DataType) -> Bool + Base.datatype_nfields(dt::DataType) -> UInt32 -Return the number of fields known to this datatype's layout. +Return the number of fields known to this datatype's layout. This may be +different from the number of actual fields of the type for opaque types. Can be called on any `isconcretetype`. """ function datatype_nfields(dt::DataType) @@ -526,6 +530,31 @@ function datatype_fielddesc_type(dt::DataType) return (flags >> 1) & 3 end +""" + Base.datatype_arrayelem(dt::DataType) -> Int + +Return the behavior of the trailing array types allocations. +Can be called on any `isconcretetype`, but only meaningful on `Memory`. + +0 = inlinealloc +1 = isboxed +2 = isbitsunion +""" +function datatype_arrayelem(dt::DataType) + @_foldable_meta + dt.layout == C_NULL && throw(UndefRefError()) + flags = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).flags + return (flags >> 3) & 3 +end + +function datatype_layoutsize(dt::DataType) + @_foldable_meta + dt.layout == C_NULL && throw(UndefRefError()) + size = unsafe_load(convert(Ptr{DataTypeLayout}, dt.layout)).size + return size % Int +end + + # For type stability, we only expose a single struct that describes everything struct FieldDesc isforeign::Bool @@ -552,7 +581,7 @@ end function getindex(dtfd::DataTypeFieldDesc, i::Int) layout_ptr = convert(Ptr{DataTypeLayout}, dtfd.dt.layout) - fd_ptr = layout_ptr + sizeof(DataTypeLayout) + fd_ptr = layout_ptr + Core.sizeof(DataTypeLayout) layout = unsafe_load(layout_ptr) fielddesc_type = (layout.flags >> 1) & 3 nfields = layout.nfields @@ -827,10 +856,10 @@ end Base.issingletontype(T) Determine whether type `T` has exactly one possible instance; for example, a -struct type with no fields. -If `T` is not a type, then return `false`. +struct type with no fields except other singleton values. +If `T` is not a concrete type, then return `false`. """ -issingletontype(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && isdefined(t, :instance)) +issingletontype(@nospecialize(t)) = (@_total_meta; isa(t, DataType) && isdefined(t, :instance) && datatype_layoutsize(t) == 0 && datatype_pointerfree(t)) """ typeintersect(T::Type, S::Type) @@ -1191,11 +1220,11 @@ function visit(f, mt::Core.MethodTable) nothing end function visit(f, mc::Core.TypeMapLevel) - function avisit(f, e::Array{Any,1}) + function avisit(f, e::Memory{Any}) for i in 2:2:length(e) isassigned(e, i) || continue ei = e[i] - if ei isa Vector{Any} + if ei isa Memory{Any} for j in 2:2:length(ei) isassigned(ei, j) || continue visit(f, ei[j]) @@ -1206,16 +1235,16 @@ function visit(f, mc::Core.TypeMapLevel) end end if mc.targ !== nothing - avisit(f, mc.targ::Vector{Any}) + avisit(f, mc.targ::Memory{Any}) end if mc.arg1 !== nothing - avisit(f, mc.arg1::Vector{Any}) + avisit(f, mc.arg1::Memory{Any}) end if mc.tname !== nothing - avisit(f, mc.tname::Vector{Any}) + avisit(f, mc.tname::Memory{Any}) end if mc.name1 !== nothing - avisit(f, mc.name1::Vector{Any}) + avisit(f, mc.name1::Memory{Any}) end mc.list !== nothing && visit(f, mc.list) mc.any !== nothing && visit(f, mc.any) diff --git a/base/reshapedarray.jl b/base/reshapedarray.jl index bcb47a9359392..a6c420255dc70 100644 --- a/base/reshapedarray.jl +++ b/base/reshapedarray.jl @@ -48,7 +48,7 @@ function reshape(a::Array{T,M}, dims::NTuple{N,Int}) where {T,N,M} if N == M && dims == size(a) return a end - ccall(:jl_reshape_array, Array{T,N}, (Any, Any, Any), Array{T,N}, a, dims) + Array{T, N}(a.ref, dims) end """ diff --git a/base/secretbuffer.jl b/base/secretbuffer.jl index eedfd8cbe84c1..f1cc2dcee3dc4 100644 --- a/base/secretbuffer.jl +++ b/base/secretbuffer.jl @@ -29,12 +29,12 @@ true ``` """ mutable struct SecretBuffer <: IO - data::Vector{UInt8} + data::Memory{UInt8} size::Int ptr::Int function SecretBuffer(; sizehint=128) - s = new(Vector{UInt8}(undef, sizehint), 0, 1) + s = new(Memory{UInt8}(undef, sizehint), 0, 1) finalizer(final_shred!, s) return s end @@ -49,7 +49,7 @@ Strings are bad at keeping secrets because they are unable to be securely zeroed or destroyed. Therefore, avoid using this constructor with secret data. Instead of starting with a string, either construct the `SecretBuffer` incrementally with `SecretBuffer()` and [`write`](@ref), or use a `Vector{UInt8}` with -the `Base.SecretBuffer!(::Vector{UInt8})` constructor. +the `Base.SecretBuffer!(::AbstractVector{UInt8})` constructor. """ SecretBuffer(str::AbstractString) = SecretBuffer(String(str)) function SecretBuffer(str::String) @@ -68,7 +68,7 @@ convert(::Type{SecretBuffer}, s::AbstractString) = SecretBuffer(String(s)) Initialize a new `SecretBuffer` from `data`, securely zeroing `data` afterwards. """ -function SecretBuffer!(d::Vector{UInt8}) +function SecretBuffer!(d::AbstractVector{UInt8}) len = length(d) s = SecretBuffer(sizehint=len) for i in 1:len @@ -106,7 +106,7 @@ show(io::IO, s::SecretBuffer) = print(io, "SecretBuffer(\"*******\")") ==(s1::SecretBuffer, s2::SecretBuffer) = (s1.ptr == s2.ptr) && (s1.size == s2.size) && (UInt8(0) == _bufcmp(s1.data, s2.data, min(s1.size, s2.size))) # Also attempt a constant time buffer comparison algorithm — the length of the secret might be # inferred by a timing attack, but not its values. -@noinline function _bufcmp(data1::Vector{UInt8}, data2::Vector{UInt8}, sz::Int) +@noinline function _bufcmp(data1::Memory{UInt8}, data2::Memory{UInt8}, sz::Int) res = UInt8(0) for i = 1:sz res |= xor(data1[i], data2[i]) @@ -121,7 +121,7 @@ hash(s::SecretBuffer, h::UInt) = hash(_sb_hash, h) function write(io::SecretBuffer, b::UInt8) if io.ptr > length(io.data) # We need to resize! the array: do this manually to ensure no copies are left behind - newdata = Vector{UInt8}(undef, (io.size+16)*2) + newdata = Memory{UInt8}(undef, (io.size+16)*2) copyto!(newdata, io.data) securezero!(io.data) io.data = newdata diff --git a/base/summarysize.jl b/base/summarysize.jl index 9bbae187cab12..6fb051a4d8632 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -49,9 +49,9 @@ function summarysize(obj; if isassigned(x, i) val = x[i] end - elseif isa(x, Array) + elseif isa(x, GenericMemory) nf = length(x) - if ccall(:jl_array_isassigned, Cint, (Any, UInt), x, i - 1) != 0 + if @inbounds @inline isassigned(x, i) val = x[i] end else @@ -126,14 +126,14 @@ function (ss::SummarySize)(obj::Core.TypeName) return Core.sizeof(obj) + (isdefined(obj, :mt) ? ss(obj.mt) : 0) end -function (ss::SummarySize)(obj::Array) +function (ss::SummarySize)(obj::GenericMemory) haskey(ss.seen, obj) ? (return 0) : (ss.seen[obj] = true) - headersize = 4*sizeof(Int) + 8 + max(0, ndims(obj)-2)*sizeof(Int) + headersize = 2*sizeof(Int) size::Int = headersize datakey = unsafe_convert(Ptr{Cvoid}, obj) if !haskey(ss.seen, datakey) ss.seen[datakey] = true - dsize = Core.sizeof(obj) + dsize = sizeof(obj) T = eltype(obj) if isbitsunion(T) # add 1 union selector byte for each element diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index fea4ca6bc1fe3..02600b88d5efd 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -445,6 +445,7 @@ generate_precompile_statements() = try # Make sure `ansi_enablecursor` is printe yield() # Make clock spinning print_state("step3" => string("R$n_succeeded", failed > 0 ? " ($failed failed)" : "")) catch ex + @show backtrace() # See #28808 @warn "Failed to precompile expression" form=statement exception=ex _module=nothing _file=nothing _line=0 end diff --git a/doc/src/devdocs/object.md b/doc/src/devdocs/object.md index caba6c3f12190..8dbca1a37254e 100644 --- a/doc/src/devdocs/object.md +++ b/doc/src/devdocs/object.md @@ -163,10 +163,8 @@ Arrays: ```c jl_array_t *jl_new_array(jl_value_t *atype, jl_tuple_t *dims); -jl_array_t *jl_new_arrayv(jl_value_t *atype, ...); jl_array_t *jl_alloc_array_1d(jl_value_t *atype, size_t nr); -jl_array_t *jl_alloc_array_2d(jl_value_t *atype, size_t nr, size_t nc); -jl_array_t *jl_alloc_array_3d(jl_value_t *atype, size_t nr, size_t nc, size_t z); +jl_array_t *jl_alloc_array_nd(jl_value_t *atype, size_t *dims, size_t ndims); jl_array_t *jl_alloc_vec_any(size_t n); ``` diff --git a/doc/src/manual/embedding.md b/doc/src/manual/embedding.md index 2b6e48c533849..60a768788bbb7 100644 --- a/doc/src/manual/embedding.md +++ b/doc/src/manual/embedding.md @@ -436,7 +436,7 @@ For example: ```c jl_array_t *some_array = ...; // e.g. a Vector{Any} -void **data = (void**)jl_array_data(some_array); +void **data = jl_array_data(some_array, void*); jl_value_t *some_value = ...; data[0] = some_value; jl_gc_wb(some_array, some_value); @@ -487,13 +487,13 @@ referenced. In order to access the data of `x`, we can use `jl_array_data`: ```c -double *xData = (double*)jl_array_data(x); +double *xData = jl_array_data(x, double); ``` Now we can fill the array: ```c -for(size_t i=0; ilayout->nfields > 0) { + if (dt->layout->nfields > 0 || dt->layout->npointers) { ab.addByValAttr(Ty); return true; } @@ -63,7 +63,7 @@ Type *preferred_llvm_type(jl_datatype_t *dt, bool isret, LLVMContext &ctx) const { // Arguments are either scalar or passed by value // rewrite integer sized (non-sret) struct to the corresponding integer - if (!dt->layout->nfields) + if (!dt->layout->nfields && !dt->layout->npointers) return NULL; return Type::getIntNTy(ctx, jl_datatype_nbits(dt)); } diff --git a/src/abi_x86_64.cpp b/src/abi_x86_64.cpp index c3d12417e6de8..f78f5bfa3d3d5 100644 --- a/src/abi_x86_64.cpp +++ b/src/abi_x86_64.cpp @@ -147,7 +147,7 @@ void classifyType(Classification& accum, jl_datatype_t *dt, uint64_t offset) con accum.addField(offset, Sse); } // Other struct types - else if (jl_datatype_size(dt) <= 16 && dt->layout) { + else if (jl_datatype_size(dt) <= 16 && dt->layout && !jl_is_layout_opaque(dt->layout)) { size_t i; for (i = 0; i < jl_datatype_nfields(dt); ++i) { jl_value_t *ty = jl_field_type(dt, i); diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 3a54e2729ff5f..12fcc898d9d53 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -268,7 +268,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm { JL_TIMING(NATIVE_AOT, NATIVE_Create); ++CreateNativeCalls; - CreateNativeMax.updateMax(jl_array_len(methods)); + CreateNativeMax.updateMax(jl_array_nrows(methods)); if (cgparams == NULL) cgparams = &jl_default_cgparams; jl_native_code_desc_t *data = new jl_native_code_desc_t; @@ -316,7 +316,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm if (policy != CompilationPolicy::Default && params.world == jl_typeinf_world) continue; size_t i, l; - for (i = 0, l = jl_array_len(methods); i < l; i++) { + for (i = 0, l = jl_array_nrows(methods); i < l; i++) { // each item in this list is either a MethodInstance indicating something // to compile, or an svec(rettype, sig) describing a C-callable alias to create. jl_value_t *item = jl_array_ptr_ref(methods, i); diff --git a/src/array.c b/src/array.c index 65efbb7e9df19..bbec0180ef78f 100644 --- a/src/array.c +++ b/src/array.c @@ -16,41 +16,13 @@ extern "C" { #endif -// similar to MEMDEBUG's purpose, this ensures there is a garbage byte after a -// most UInt8 arrays (even String-backed) so that almost any C string -// processing on it is guaranteed to run too far and return some garbage -// answers to warn the user of their bug. -#ifdef NDEBUG -#define JL_ARRAY_MEMDEBUG_TERMINATOR 0 +#if defined(_P64) && defined(UINT128MAX) +typedef __uint128_t wideint_t; #else -#define JL_ARRAY_MEMDEBUG_TERMINATOR 1 +typedef uint64_t wideint_t; #endif -#define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes) - -static inline void arrayassign_safe(int hasptr, jl_value_t *parent, char *dst, const jl_value_t *src, size_t nb) JL_NOTSAFEPOINT -{ - // array can assume more alignment than a field would normally have - assert(nb >= jl_datatype_size(jl_typeof(src))); // nb might move some undefined bits, but we should be okay with that - if (hasptr) { - size_t nptr = nb / sizeof(void*); - memmove_refs((void**)dst, (void* const*)src, nptr); - jl_gc_multi_wb(parent, src); - } - else { - switch (nb) { - case 0: break; - case 1: *(uint8_t*)dst = *(uint8_t*)src; break; - case 2: *(uint16_t*)dst = *(uint16_t*)src; break; - case 4: *(uint32_t*)dst = *(uint32_t*)src; break; - case 8: *(uint64_t*)dst = *(uint64_t*)src; break; - case 16: - memcpy(jl_assume_aligned(dst, 16), jl_assume_aligned(src, 16), 16); - break; - default: memcpy(dst, src, nb); - } - } -} +#define MAXINTVAL (((size_t)-1)>>1) static inline void memmove_safe(int hasptr, char *dst, const char *src, size_t nb) JL_NOTSAFEPOINT { @@ -60,30 +32,6 @@ static inline void memmove_safe(int hasptr, char *dst, const char *src, size_t n memmove(dst, src, nb); } -// array constructors --------------------------------------------------------- -JL_DLLEXPORT char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT -{ - assert(jl_array_isbitsunion(a)); - return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset; -} - -STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT -{ - if (a->flags.how == 3) { - a = (jl_array_t*)jl_array_data_owner(a); - assert(jl_is_string(a) || a->flags.how != 3); - } - return (jl_value_t*)a; -} - -#if defined(_P64) && defined(UINT128MAX) -typedef __uint128_t wideint_t; -#else -typedef uint64_t wideint_t; -#endif - -#define MAXINTVAL (((size_t)-1)>>1) - JL_DLLEXPORT int jl_array_validate_dims(size_t *nel, size_t *tot, uint32_t ndims, size_t *dims, size_t elsz) { size_t i; @@ -103,123 +51,12 @@ JL_DLLEXPORT int jl_array_validate_dims(size_t *nel, size_t *tot, uint32_t ndims return 0; } -static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims, - int8_t isunboxed, int8_t hasptr, int8_t isunion, int8_t zeroinit, size_t elsz) -{ - jl_task_t *ct = jl_current_task; - size_t i, tot, nel; - void *data; - jl_array_t *a; - assert(isunboxed || elsz == sizeof(void*)); - assert(atype == NULL || isunion == jl_is_uniontype(jl_tparam0(atype))); - int validated = jl_array_validate_dims(&nel, &tot, ndims, dims, elsz); - if (validated == 1) - jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); - else if (validated == 2) - jl_error("invalid Array size"); - if (isunboxed) { - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isunion) { - // extra byte for all julia allocated byte arrays - tot++; - } - if (isunion) { - // an extra byte for each isbits union array element, stored after a->maxsize - tot += nel; - } - } - - int ndimwords = jl_array_ndimwords(ndims); - int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t); - if (tot <= ARRAY_INLINE_NBYTES) { - // align data area - if (tot >= ARRAY_CACHE_ALIGN_THRESHOLD) - tsz = JL_ARRAY_ALIGN(tsz, JL_CACHE_BYTE_ALIGNMENT); - else if (isunboxed && elsz >= 4) - tsz = JL_ARRAY_ALIGN(tsz, JL_SMALL_BYTE_ALIGNMENT); - size_t doffs = tsz; - tsz += tot; - // jl_array_t is large enough that objects will always be aligned 16 - a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); - assert(((size_t)a & 15) == 0); - // No allocation or safepoint allowed after this - a->flags.how = 0; - data = (char*)a + doffs; - } - else { - data = jl_gc_managed_malloc(tot); - // Allocate the Array **after** allocating the data - // to make sure the array is still young - a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); - // No allocation or safepoint allowed after this - a->flags.how = 2; - jl_gc_track_malloced_array(ct->ptls, a); - } - a->flags.pooled = tsz <= GC_MAX_SZCLASS; - - if (zeroinit) - memset(data, 0, tot); - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isunion) { - ((char*)data)[tot - 1] = 0xfe; - msan_allocated_memory(&((char*)data)[tot - 1], 1); - } - a->data = data; - a->length = nel; - a->flags.ndims = ndims; - a->flags.ptrarray = !isunboxed; - a->flags.hasptr = hasptr; - a->elsize = elsz; - a->flags.isshared = 0; - a->flags.isaligned = 1; - a->offset = 0; - if (ndims == 1) { - a->nrows = nel; - a->maxsize = nel; - } - else if (a->flags.ndims != ndims) { - jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); - } - else { - size_t *adims = &a->nrows; - for (i = 0; i < ndims; i++) - adims[i] = dims[i]; - } - - return a; -} - -static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *dims) -{ - jl_value_t *eltype = jl_tparam0(atype); - size_t elsz = 0, al = 0; - if (!jl_is_kind(jl_typeof(eltype))) - jl_type_error_rt("Array", "element type", (jl_value_t*)jl_type_type, eltype); - int isunboxed = jl_islayout_inline(eltype, &elsz, &al); - int isunion = jl_is_uniontype(eltype); - int hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0); - if (!isunboxed) { - elsz = sizeof(void*); - al = elsz; - } - else { - elsz = LLT_ALIGN(elsz, al); - } - int zi = !isunboxed || hasptr || isunion || (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->zeroinit); - - return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, zi, elsz); -} - -jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims, - int isunboxed, int hasptr, int isunion, int elsz) -{ - return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, 0, (size_t)elsz); -} - #ifndef JL_NDEBUG static inline int is_ntuple_long(jl_value_t *v) { if (!jl_is_tuple(v)) return 0; - jl_value_t *tt = jl_typeof(v); + jl_value_t *tt = (jl_value_t*)jl_typetagof(v); size_t i, nfields = jl_nparams(tt); for (i = 0; i < nfields; i++) { if (jl_tparam(tt, i) != (jl_value_t*)jl_long_type) { @@ -230,140 +67,111 @@ static inline int is_ntuple_long(jl_value_t *v) } #endif -JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, - jl_value_t *_dims) +JL_UNUSED static size_t overallocation(size_t maxsize) { - jl_task_t *ct = jl_current_task; - assert(jl_types_equal(jl_tparam0(jl_typeof(data)), jl_tparam0(atype))); + if (maxsize < 8) + return 8; + // compute maxsize = maxsize + 4*maxsize^(7/8) + maxsize/8 + // for small n, we grow faster than O(n) + // for large n, we grow at O(n/8) + // and as we reach O(memory) for memory>>1MB, + // this means we end by adding about 10% of memory each time + int exp2 = sizeof(maxsize) * 8 - +#ifdef _P64 + __builtin_clzll(maxsize); +#else + __builtin_clz(maxsize); +#endif + maxsize += ((size_t)1 << (exp2 * 7 / 8)) * 4 + maxsize / 8; + return maxsize; +} - size_t ndims = jl_nfields(_dims); - assert(is_ntuple_long(_dims)); - size_t *dims = (size_t*)_dims; - int ndimwords = jl_array_ndimwords(ndims); - int tsz = sizeof(jl_array_t) + ndimwords * sizeof(size_t) + sizeof(void*); - jl_array_t *a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); - // No allocation or safepoint allowed after this - // copy data (except dims) from the old object - a->flags.pooled = tsz <= GC_MAX_SZCLASS; - a->flags.ndims = ndims; - a->offset = 0; - a->data = NULL; - a->flags.isaligned = data->flags.isaligned; - a->elsize = data->elsize; - a->flags.ptrarray = data->flags.ptrarray; - a->flags.hasptr = data->flags.hasptr; +#define jl_array_hasptr(a) (((jl_datatype_t*)jl_typetagof((a)->ref.mem))->layout->first_ptr >= 0) +#define jl_array_ptrarray(a) (((jl_datatype_t*)jl_typetagof((a)->ref.mem))->layout->arrayelem_isboxed) +#define jl_array_elsize(a) (((jl_datatype_t*)jl_typetagof((a)->ref.mem))->layout->size) - // if data is itself a shared wrapper, - // owner should point back to the original array - jl_array_t *owner = (jl_array_t*)jl_array_owner(data); - jl_array_data_owner(a) = (jl_value_t*)owner; +JL_DLLEXPORT char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT +{ + assert(jl_genericmemory_isbitsunion(a->ref.mem)); + return jl_genericmemory_typetagdata(a->ref.mem) + (uintptr_t)a->ref.data; +} - a->flags.how = 3; - a->data = data->data; - a->flags.isshared = 1; - data->flags.isshared = 1; +STATIC_INLINE jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *dims) +{ + jl_task_t *ct = jl_current_task; + size_t i, tot, nel; + jl_array_t *a; + int invalid = jl_array_validate_dims(&nel, &tot, ndims, dims, 1); + if (invalid) + jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); + jl_value_t *mtype = jl_field_type_concrete((jl_datatype_t*)jl_field_type_concrete((jl_datatype_t*)atype, 0), 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + // extra byte for all julia allocated byte vectors + jl_genericmemory_t *mem = jl_alloc_genericmemory(mtype, nel); + JL_GC_PUSH1(&mem); + int ndimwords = ndims; + int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t); + a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); + a->ref.mem = mem; + if (layout->arrayelem_isunion || layout->size == 0) + a->ref.data = 0; + else + a->ref.data = mem->data; if (ndims == 1) { - size_t l = dims[0]; - a->length = l; - a->nrows = l; - a->maxsize = l; + a->dimsize[0] = nel; } - else if (a->flags.ndims != ndims) { + else if (jl_array_ndims(a) != ndims) { jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); } else { - size_t *adims = &a->nrows; - size_t l = 1; - wideint_t prod; - for (size_t i = 0; i < ndims; i++) { - adims[i] = dims[i]; - prod = (wideint_t)l * (wideint_t)adims[i]; - if (prod > (wideint_t) MAXINTVAL) - jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); - l = prod; - } - a->length = l; + for (i = 0; i < ndims; i++) + a->dimsize[i] = dims[i]; } - + JL_GC_POP(); return a; } +jl_genericmemory_t *_new_genericmemory_(jl_value_t *mtype, size_t nel, int8_t isunion, int8_t zeroinit, size_t elsz); + +JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str); + JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str) { jl_task_t *ct = jl_current_task; - jl_array_t *a; - - int ndimwords = jl_array_ndimwords(1); - int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t) + sizeof(void*); - a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, jl_array_uint8_type); - a->flags.pooled = tsz <= GC_MAX_SZCLASS; - a->flags.ndims = 1; - a->offset = 0; - a->data = jl_string_data(str); - a->flags.isaligned = 0; - a->elsize = 1; - a->flags.ptrarray = 0; - a->flags.hasptr = 0; - jl_array_data_owner(a) = str; - a->flags.how = 3; - a->flags.isshared = 1; - size_t l = jl_string_len(str); - a->length = l; - a->nrows = a->maxsize = l; + jl_genericmemory_t *mem = jl_string_to_genericmemory(str); + JL_GC_PUSH1(&mem); + int ndimwords = 1; + int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t); + jl_array_t *a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, jl_array_uint8_type); + a->ref.mem = mem; + a->ref.data = mem->data; + a->dimsize[0] = mem->length; + JL_GC_POP(); return a; } -// own_buffer != 0 iff GC should call free() on this pointer eventually JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer) { jl_task_t *ct = jl_current_task; jl_array_t *a; - jl_value_t *eltype = jl_tparam0(atype); - - int isunboxed = jl_stored_inline(eltype); - if (isunboxed && jl_is_uniontype(eltype)) - jl_exceptionf(jl_argumenterror_type, - "unsafe_wrap: unspecified layout for union element type"); - size_t elsz; - unsigned align; - if (isunboxed) { - elsz = jl_datatype_size(eltype); - align = jl_datatype_align(eltype); - } - else { - align = elsz = sizeof(void*); - } - if (((uintptr_t)data) & ((align > JL_HEAP_ALIGNMENT ? JL_HEAP_ALIGNMENT : align) - 1)) - jl_exceptionf(jl_argumenterror_type, - "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); - - int ndimwords = jl_array_ndimwords(1); - int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t); + if (nel > MAXINTVAL) + jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); + jl_value_t *mtype = jl_field_type_concrete((jl_datatype_t*)jl_field_type_concrete((jl_datatype_t*)atype, 0), 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + jl_genericmemory_t *mem = jl_ptr_to_genericmemory(mtype, data, nel, own_buffer); + JL_GC_PUSH1(&mem); + int ndims = 1; + int tsz = sizeof(jl_array_t) + ndims*sizeof(size_t); a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); - // No allocation or safepoint allowed after this - a->flags.pooled = tsz <= GC_MAX_SZCLASS; - a->data = data; - a->length = nel; - a->elsize = LLT_ALIGN(elsz, align); - a->flags.ptrarray = !isunboxed; - a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0); - a->flags.ndims = 1; - a->flags.isshared = 1; - a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers - if (own_buffer) { - a->flags.how = 2; - jl_gc_track_malloced_array(ct->ptls, a); - jl_gc_count_allocd(nel*elsz); - } - else { - a->flags.how = 0; - } - - a->nrows = nel; - a->maxsize = nel; - a->offset = 0; + a->ref.mem = mem; + if (layout->arrayelem_isunion || layout->size == 0) + a->ref.data = 0; + else + a->ref.data = mem->data; + a->dimsize[0] = nel; + JL_GC_POP(); return a; } @@ -372,7 +180,6 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, { jl_task_t *ct = jl_current_task; size_t nel = 1; - jl_array_t *a; size_t ndims = jl_nfields(_dims); wideint_t prod; assert(is_ntuple_long(_dims)); @@ -385,160 +192,98 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, } if (__unlikely(ndims == 1)) return jl_ptr_to_array_1d(atype, data, nel, own_buffer); - jl_value_t *eltype = jl_tparam0(atype); - - int isunboxed = jl_stored_inline(eltype); - if (isunboxed && jl_is_uniontype(eltype)) - jl_exceptionf(jl_argumenterror_type, - "unsafe_wrap: unspecified layout for union element type"); - size_t elsz; - unsigned align; - if (isunboxed) { - elsz = jl_datatype_size(eltype); - align = jl_datatype_align(eltype); - } - else { - align = elsz = sizeof(void*); - } - if (((uintptr_t)data) & ((align > JL_HEAP_ALIGNMENT ? JL_HEAP_ALIGNMENT : align) - 1)) - jl_exceptionf(jl_argumenterror_type, - "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); - - int ndimwords = jl_array_ndimwords(ndims); + jl_value_t *mtype = jl_field_type_concrete((jl_datatype_t*)jl_field_type_concrete((jl_datatype_t*)atype, 0), 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + jl_genericmemory_t *mem = jl_ptr_to_genericmemory(mtype, data, nel, own_buffer); + JL_GC_PUSH1(&mem); + int ndimwords = ndims; int tsz = sizeof(jl_array_t) + ndimwords*sizeof(size_t); - a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); - // No allocation or safepoint allowed after this - a->flags.pooled = tsz <= GC_MAX_SZCLASS; - a->data = data; - a->length = nel; - a->elsize = LLT_ALIGN(elsz, align); - a->flags.ptrarray = !isunboxed; - a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0); - a->flags.ndims = ndims; - a->offset = 0; - a->flags.isshared = 1; - a->flags.isaligned = 0; - if (own_buffer) { - a->flags.how = 2; - jl_gc_track_malloced_array(ct->ptls, a); - jl_gc_count_allocd(nel*elsz); - } - else { - a->flags.how = 0; - } - + jl_array_t *a = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, atype); + a->ref.mem = mem; + if (layout->arrayelem_isunion || layout->size == 0) + a->ref.data = 0; + else + a->ref.data = mem->data; assert(ndims != 1); // handled above - if (a->flags.ndims != ndims) + if (jl_array_ndims(a) != ndims) jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions"); - memcpy(&a->nrows, dims, ndims * sizeof(size_t)); - return a; -} - -JL_DLLEXPORT jl_array_t *jl_new_array(jl_value_t *atype, jl_value_t *_dims) -{ - size_t ndims = jl_nfields(_dims); - assert(is_ntuple_long(_dims)); - return _new_array(atype, ndims, (size_t*)_dims); -} - -JL_DLLEXPORT jl_array_t *jl_alloc_array_1d(jl_value_t *atype, size_t nr) -{ - return _new_array(atype, 1, &nr); -} - -JL_DLLEXPORT jl_array_t *jl_alloc_array_2d(jl_value_t *atype, size_t nr, - size_t nc) -{ - size_t d[2] = {nr, nc}; - return _new_array(atype, 2, &d[0]); -} - -JL_DLLEXPORT jl_array_t *jl_alloc_array_3d(jl_value_t *atype, size_t nr, - size_t nc, size_t z) -{ - size_t d[3] = {nr, nc, z}; - return _new_array(atype, 3, &d[0]); -} - -JL_DLLEXPORT jl_array_t *jl_pchar_to_array(const char *str, size_t len) -{ - jl_array_t *a = jl_alloc_array_1d(jl_array_uint8_type, len); - memcpy(a->data, str, len); + memcpy(&a->dimsize, dims, ndims * sizeof(size_t)); + JL_GC_POP(); return a; } JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a) { - size_t len = jl_array_len(a); + size_t len = jl_array_nrows(a); // only for Vector if (len == 0) { // this may seem like purely an optimization (which it also is), but it // also ensures that calling `String(a)` doesn't corrupt a previous // string also created the same way, where `a = StringVector(_)`. return jl_an_empty_string; } - if (a->flags.how == 3 && a->offset == 0 && a->elsize == 1 && - (jl_array_ndims(a) != 1 || - ((a->maxsize + sizeof(void*) + 1 <= GC_MAX_SZCLASS) == (len + sizeof(void*) + 1 <= GC_MAX_SZCLASS)))) { - jl_value_t *o = jl_array_data_owner(a); - if (jl_is_string(o)) { - a->flags.isshared = 1; - *(size_t*)o = len; - a->nrows = 0; - a->length = 0; - a->maxsize = 0; - if (jl_string_data(o)[len] != '\0') - jl_string_data(o)[len] = '\0'; - return o; - } - } - a->nrows = 0; - a->length = 0; - a->maxsize = 0; - return jl_pchar_to_string((const char*)jl_array_data(a), len); + jl_value_t *str; + if (a->ref.data == a->ref.mem->data) + str = jl_genericmemory_to_string(a->ref.mem, len); + else + str = jl_pchar_to_string(jl_array_data(a, char), len); + a->ref.mem->length = 0; // TODO(jwn): unsound? + a->dimsize[0] = 0; + return str; } -JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len) -{ - if (len == 0) - return jl_an_empty_string; - size_t sz = sizeof(size_t) + len + 1; // add space for trailing \nul protector and size - if (sz < len) // overflow - jl_throw(jl_memory_exception); - jl_task_t *ct = jl_current_task; - jl_value_t *s; - jl_ptls_t ptls = ct->ptls; - const size_t allocsz = sz + sizeof(jl_taggedvalue_t); - if (sz <= GC_MAX_SZCLASS) { - int pool_id = jl_gc_szclass_align8(allocsz); - jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; - int osize = jl_gc_sizeclasses[pool_id]; - // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in - // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.) - s = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize); +// Resize the buffer to a max size of `newlen` +// The buffer can either be newly allocated or realloc'd, the return +// value is 1 if a new buffer is allocated and 0 if it is realloc'd. +// the caller needs to take care of moving the data from the old buffer +// to the new one if necessary. +// When this function returns, the `->data` pointer always points to +// the **beginning** of the new buffer. +static int array_resize_buffer(jl_array_t *a, size_t newlen) +{ + // n.b. ndims == 1 + jl_value_t *mtype = (jl_value_t*)jl_typetagof(a->ref.mem); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + int newbuf = 1; + jl_genericmemory_t *mem; + if (jl_is_string(jl_array_owner(a))) { + // if data is in a String, keep it that way + jl_value_t *str = jl_alloc_string(newlen); + // TODO: use jl_gc_expand_string(oldstr, newlen)? + JL_GC_PUSH1(&str); + mem = jl_string_to_genericmemory(str); + a->ref.mem = mem; + a->ref.data = mem->data; + jl_gc_wb(a, mem); + JL_GC_POP(); } else { - if (allocsz < sz) // overflow in adding offs, size was "negative" - jl_throw(jl_memory_exception); - s = jl_gc_big_alloc_noinline(ptls, allocsz); + mem = jl_alloc_genericmemory(mtype, newlen); + a->ref.mem = mem; + if (layout->arrayelem_isunion || layout->size == 0) + a->ref.data = 0; + else + a->ref.data = mem->data; + jl_gc_wb(a, mem); } - jl_set_typetagof(s, jl_string_tag, 0); - maybe_record_alloc_to_profile(s, len, jl_string_type); - *(size_t*)s = len; - jl_string_data(s)[len] = 0; - return s; + return newbuf; } -JL_DLLEXPORT jl_value_t *jl_pchar_to_string(const char *str, size_t len) +JL_DLLEXPORT jl_array_t *jl_alloc_array_1d(jl_value_t *atype, size_t nr) { - jl_value_t *s = jl_alloc_string(len); - if (len > 0) - memcpy(jl_string_data(s), str, len); - return s; + return _new_array(atype, 1, &nr); } -JL_DLLEXPORT jl_value_t *jl_cstr_to_string(const char *str) +JL_DLLEXPORT jl_array_t *jl_alloc_array_nd(jl_value_t *atype, size_t *dims, size_t ndims) { - return jl_pchar_to_string(str, strlen(str)); + return _new_array(atype, ndims, dims); +} + +JL_DLLEXPORT jl_array_t *jl_pchar_to_array(const char *str, size_t len) +{ + jl_array_t *a = jl_alloc_array_1d(jl_array_uint8_type, len); + assert(jl_array_data(a, char)); + memcpy(jl_array_data(a, char), str, len); + return a; } JL_DLLEXPORT jl_array_t *jl_alloc_vec_any(size_t n) @@ -555,653 +300,83 @@ JL_DLLEXPORT jl_value_t *jl_apply_array_type(jl_value_t *type, size_t dim) return ret; } -// array primitives ----------------------------------------------------------- - -JL_DLLEXPORT jl_value_t *jl_ptrarrayref(jl_array_t *a JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT -{ - assert(i < jl_array_len(a)); - assert(a->flags.ptrarray); - jl_value_t *elt = jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)a->data) + i); - if (elt == NULL) - jl_throw(jl_undefref_exception); - return elt; -} - - -JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i) -{ - if (a->flags.ptrarray) - return jl_ptrarrayref(a, i); - assert(i < jl_array_len(a)); - jl_value_t *eltype = (jl_value_t*)jl_tparam0(jl_typeof(a)); - if (jl_is_uniontype(eltype)) { - // isbits union selector bytes are always stored directly after the last array element - uint8_t sel = jl_array_typetagdata(a)[i]; - eltype = jl_nth_union_component(eltype, sel); - if (jl_is_datatype_singleton((jl_datatype_t*)eltype)) - return ((jl_datatype_t*)eltype)->instance; - } - jl_value_t *r = undefref_check((jl_datatype_t*)eltype, jl_new_bits(eltype, &((char*)a->data)[i * a->elsize])); - if (__unlikely(r == NULL)) - jl_throw(jl_undefref_exception); - return r; -} - -JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i) -{ - if (a->flags.ptrarray) { - return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)jl_array_data(a)) + i) != NULL; - } - else if (a->flags.hasptr) { - jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a)); - assert(eltype->layout->first_ptr >= 0); - jl_value_t **elem = (jl_value_t**)((char*)a->data + i * a->elsize); - return elem[eltype->layout->first_ptr] != NULL; - } - return 1; -} - -JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i) -{ - assert(i < jl_array_len(a)); - jl_value_t *eltype = jl_tparam0(jl_typeof(a)); - if (eltype != (jl_value_t*)jl_any_type) { - JL_GC_PUSH1(&rhs); - if (!jl_isa(rhs, eltype)) - jl_type_error("arrayset", eltype, rhs); - JL_GC_POP(); - } - if (!a->flags.ptrarray) { - int hasptr; - if (jl_is_uniontype(eltype)) { - uint8_t *psel = &((uint8_t*)jl_array_typetagdata(a))[i]; - unsigned nth = 0; - if (!jl_find_union_component(eltype, jl_typeof(rhs), &nth)) - assert(0 && "invalid arrayset to isbits union"); - *psel = nth; - if (jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(rhs))) - return; - hasptr = 0; - } - else { - hasptr = a->flags.hasptr; - } - arrayassign_safe(hasptr, jl_array_owner(a), &((char*)a->data)[i * a->elsize], rhs, a->elsize); - } - else { - jl_atomic_store_release(((_Atomic(jl_value_t*)*)a->data) + i, rhs); - jl_gc_wb(jl_array_owner(a), rhs); - } -} - -JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i) -{ - if (i >= jl_array_len(a)) - jl_bounds_error_int((jl_value_t*)a, i + 1); - if (a->flags.ptrarray) - jl_atomic_store_relaxed(((_Atomic(jl_value_t*)*)a->data) + i, NULL); - else if (a->flags.hasptr) { - size_t elsize = a->elsize; - jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0); - memset((char*)a->data + elsize * i, 0, elsize); - } -} - -// at this size and bigger, allocate resized array data with malloc directly -// instead of managing them separately as gc objects -#define MALLOC_THRESH 1048576 - -// Resize the buffer to a max size of `newlen` -// The buffer can either be newly allocated or realloc'd, the return -// value is 1 if a new buffer is allocated and 0 if it is realloc'd. -// the caller needs to take care of moving the data from the old buffer -// to the new one if necessary. -// When this function returns, the `->data` pointer always points to -// the **beginning** of the new buffer. -static int NOINLINE array_resize_buffer(jl_array_t *a, size_t newlen) -{ - jl_task_t *ct = jl_current_task; - assert(!a->flags.isshared || a->flags.how == 3); - size_t elsz = a->elsize; - size_t nbytes = newlen * elsz; - size_t oldnbytes = a->maxsize * elsz; - size_t oldoffsnb = a->offset * elsz; - size_t oldlen = a->nrows; - int isbitsunion = jl_array_isbitsunion(a); - assert(nbytes >= oldnbytes); - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isbitsunion) { - nbytes++; - oldnbytes++; - } - if (isbitsunion) { - nbytes += newlen; - oldnbytes += a->maxsize; - } - int newbuf = 0; - if (a->flags.how == 2) { - // already malloc'd - use realloc - char *olddata = (char*)a->data - oldoffsnb; - a->data = jl_gc_managed_realloc(olddata, nbytes, oldnbytes, - a->flags.isaligned, (jl_value_t*)a); - } - else if (a->flags.how == 3 && jl_is_string(jl_array_data_owner(a)) && !isbitsunion) { - // if data is in a String, keep it that way - jl_value_t *s; - if (a->flags.isshared) { - s = jl_alloc_string(nbytes - (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1)); - newbuf = 1; - } - else { - s = jl_gc_realloc_string(jl_array_data_owner(a), nbytes - (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1)); - } - jl_array_data_owner(a) = s; - jl_gc_wb(a, s); - a->data = jl_string_data(s); - } - else { - newbuf = 1; - if (nbytes >= MALLOC_THRESH) { - a->data = jl_gc_managed_malloc(nbytes); - jl_gc_track_malloced_array(ct->ptls, a); - a->flags.how = 2; - a->flags.isaligned = 1; - } - else { - a->data = jl_gc_alloc_buf(ct->ptls, nbytes); - a->flags.how = 1; - jl_gc_wb_buf(a, a->data, nbytes); - } - } - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isbitsunion) { - memset((char*)a->data + oldnbytes - 1, 0xfe, nbytes - oldnbytes + 1); - msan_allocated_memory((char*)a->data + oldnbytes - 1, nbytes - oldnbytes + 1); - } - (void)oldlen; - assert(oldlen == a->nrows && - "Race condition detected: recursive resizing on the same array."); - a->flags.isshared = 0; - a->maxsize = newlen; - return newbuf; -} - -static void NOINLINE array_try_unshare(jl_array_t *a) -{ - if (a->flags.isshared) { - if (a->flags.how != 3) - jl_error("cannot resize array with shared data"); - // allow resizing when data is shared with a String - if (jl_is_string(jl_array_data_owner(a))) - return; - assert(a->offset == 0); - size_t len = a->maxsize; - size_t nbytes = len * a->elsize; - if (jl_array_isbitsunion(a)) { - nbytes += len; - } - char *olddata = (char*)a->data; - int newbuf = array_resize_buffer(a, len); - assert(newbuf); - (void)newbuf; - memcpy(a->data, olddata, nbytes); - } -} - -size_t overallocation(size_t maxsize) -{ - if (maxsize < 8) - return 8; - // compute maxsize = maxsize + 4*maxsize^(7/8) + maxsize/8 - // for small n, we grow faster than O(n) - // for large n, we grow at O(n/8) - // and as we reach O(memory) for memory>>1MB, - // this means we end by adding about 10% of memory each time - int exp2 = sizeof(maxsize) * 8 - -#ifdef _P64 - __builtin_clzll(maxsize); -#else - __builtin_clz(maxsize); -#endif - maxsize += ((size_t)1 << (exp2 * 7 / 8)) * 4 + maxsize / 8; - return maxsize; -} - -STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc, - size_t n) +JL_DLLEXPORT void jl_array_grow_end(jl_array_t *a, size_t inc) { - // designed to handle the case of growing and shrinking at both ends - if (__unlikely(a->flags.isshared)) { - if (a->flags.how != 3) - jl_error("cannot resize array with shared data"); - if (inc == 0) { - // If inc > 0, it will always trigger the slow path and unshare the - // buffer - array_try_unshare(a); - return; - } - } - size_t newnrows = n + inc; - size_t elsz = a->elsize; - size_t nbinc = inc * elsz; - char *data = (char*)a->data; - char *newdata; + size_t n = jl_array_nrows(a); + size_t elsz = jl_array_elsize(a); + char *data = jl_array_data(a,char); char *typetagdata; - char *newtypetagdata = NULL; - int isbitsunion = jl_array_isbitsunion(a); + char *newtypetagdata; + int isbitsunion = jl_genericmemory_isbitsunion(a->ref.mem); if (isbitsunion) typetagdata = jl_array_typetagdata(a); - if (a->offset >= inc) { - // already have enough space in a->offset - newdata = data - nbinc; - a->offset -= inc; - if (isbitsunion) newtypetagdata = typetagdata - inc; - if (idx > 0) { - // inserting new elements after 1st element - memmove_safe(a->flags.hasptr, newdata, data, idx * elsz); - if (isbitsunion) { - memmove(newtypetagdata, typetagdata, idx); - memset(newtypetagdata + idx, 0, inc); - } - } - } - else { - // not enough room for requested growth from existing a->offset - size_t oldoffset = a->offset; - size_t oldoffsnb = oldoffset * elsz; - size_t oldmaxsize = a->maxsize; - size_t nb1 = idx * elsz; - if (inc > (a->maxsize - n) / 2 - (a->maxsize - n) / 20) { - // not enough room for requested growth from end of array - size_t newlen = inc * 2; - while (n + 2 * inc > newlen - a->offset) - newlen *= 2; - size_t newmaxsize = overallocation(a->maxsize); - if (newlen < newmaxsize) - newlen = newmaxsize; - size_t newoffset = (newlen - newnrows) / 2; - if (!array_resize_buffer(a, newlen)) { - data = (char*)a->data + oldoffsnb; - } - newdata = (char*)a->data + newoffset * elsz; - if (isbitsunion) { - typetagdata = data + (oldmaxsize - oldoffset) * elsz + oldoffset; - newtypetagdata = newdata + (a->maxsize - newoffset) * elsz + newoffset; - memmove(newtypetagdata, typetagdata, idx); - memset(newtypetagdata + idx, 0, inc); - memmove(newtypetagdata + idx + inc, typetagdata + idx, n - idx); - } - // We could use memcpy if resizing allocates a new buffer, - // hopefully it's not a particularly important optimization. - if (idx > 0 && newdata < data) { - memmove_safe(a->flags.hasptr, newdata, data, nb1); - } - memmove_safe(a->flags.hasptr, newdata + nbinc + nb1, data + nb1, n * elsz - nb1); - if (idx > 0 && newdata > data) { - memmove_safe(a->flags.hasptr, newdata, data, nb1); - } - a->offset = newoffset; - } - else { - // use extra space between a->nrows & a->maxsize - a->offset = (a->maxsize - newnrows) / 2; - newdata = data - oldoffsnb + a->offset * elsz; - if (isbitsunion) newtypetagdata = newdata + (a->maxsize - a->offset) * elsz + a->offset; - if (idx > 0 && newdata < data) { - memmove_safe(a->flags.hasptr, newdata, data, nb1); - if (isbitsunion) { - memmove(newtypetagdata, typetagdata, idx); - memset(newtypetagdata + idx, 0, inc); - } - } - memmove_safe(a->flags.hasptr, newdata + nbinc + nb1, data + nb1, n * elsz - nb1); - if (isbitsunion) memmove(newtypetagdata + idx + inc, typetagdata + idx, n - idx); - if (idx > 0 && newdata > data) { - memmove_safe(a->flags.hasptr, newdata, data, nb1); - if (isbitsunion) { - memmove(newtypetagdata, typetagdata, idx); - memset(newtypetagdata + idx, 0, inc); - } - } - } - } - a->length = newnrows; - a->nrows = newnrows; - a->data = newdata; - if (jl_is_array_zeroinit(a)) { - memset(newdata + idx * elsz, 0, nbinc); - } - if (newtypetagdata) { - memset(newtypetagdata + idx, 0, inc); + if (!isbitsunion && elsz == 0) { + a->dimsize[0] = n + inc; + return; } -} - -STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx, - size_t inc, size_t n) -{ - // optimized for the case of only growing and shrinking at the end - if (__unlikely(a->flags.isshared)) { - if (a->flags.how != 3) - jl_error("cannot resize array with shared data"); - if (inc == 0) { - // If inc > 0, it will always trigger the slow path and unshare the - // buffer - array_try_unshare(a); - return; - } + size_t oldoffset = (data - (char*)a->ref.mem->data) / elsz; + if (isbitsunion) { + oldoffset = (size_t)data; + data = (char*)a->ref.mem->data + oldoffset * elsz; } - size_t elsz = a->elsize; - char *data = (char*)a->data; - char *typetagdata; - char *newtypetagdata; - int isbitsunion = jl_array_isbitsunion(a); - if (isbitsunion) typetagdata = jl_array_typetagdata(a); - int has_gap = n > idx; - size_t reqmaxsize = a->offset + n + inc; - if (__unlikely(reqmaxsize > a->maxsize)) { - size_t nb1 = idx * elsz; - size_t nbinc = inc * elsz; + size_t oldmaxsize = a->ref.mem->length; + size_t reqmaxsize = oldoffset + n + inc; + if (__unlikely(reqmaxsize > oldmaxsize)) { // grow either by our computed overallocation factor or exactly the requested size, // whichever is larger - size_t newmaxsize = overallocation(a->maxsize); + size_t newmaxsize = overallocation(oldmaxsize); if (newmaxsize < reqmaxsize) newmaxsize = reqmaxsize; - size_t oldmaxsize = a->maxsize; int newbuf = array_resize_buffer(a, newmaxsize); - char *newdata = (char*)a->data + a->offset * elsz; - if (isbitsunion) newtypetagdata = newdata + (a->maxsize - a->offset) * elsz + a->offset; + char *newdata = (char*)a->ref.mem->data + oldoffset * elsz; + if (isbitsunion) { + newtypetagdata = newdata + (newmaxsize - oldoffset) * elsz + oldoffset; + } if (newbuf) { - memcpy(newdata, data, nb1); + memcpy(newdata, data, n * elsz); if (isbitsunion) { - memcpy(newtypetagdata, typetagdata, idx); - if (has_gap) memcpy(newtypetagdata + idx + inc, typetagdata + idx, n - idx); - memset(newtypetagdata + idx, 0, inc); + memcpy(newtypetagdata, typetagdata, n); + memset(newtypetagdata+n, 0, inc); } - if (has_gap) memcpy(newdata + nb1 + nbinc, data + nb1, n * elsz - nb1); } else { if (isbitsunion) { - typetagdata = newdata + (oldmaxsize - a->offset) * elsz + a->offset; - if (has_gap) memmove(newtypetagdata + idx + inc, typetagdata + idx, n - idx); - memmove(newtypetagdata, typetagdata, idx); - memset(newtypetagdata + idx, 0, inc); + typetagdata = newdata + (newmaxsize - oldoffset) * elsz + oldoffset; + memmove(newtypetagdata, typetagdata, n); + memset(newtypetagdata+n, 0, inc); } - if (has_gap) memmove_safe(a->flags.hasptr, newdata + nb1 + nbinc, newdata + nb1, n * elsz - nb1); } - a->data = data = newdata; - } - else if (has_gap) { - if (isbitsunion) { - memmove(typetagdata + idx + inc, typetagdata + idx, n - idx); - memset(typetagdata + idx, 0, inc); - } - size_t nb1 = idx * elsz; - memmove_safe(a->flags.hasptr, data + nb1 + inc * elsz, data + nb1, n * elsz - nb1); - } - else { - // there was enough room for requested growth already in a->maxsize + data = newdata; if (isbitsunion) - memset(typetagdata + idx, 0, inc); + a->ref.data = (void*)oldoffset; + else + a->ref.data = data; } size_t newnrows = n + inc; - a->length = newnrows; - a->nrows = newnrows; - if (jl_is_array_zeroinit(a)) { - memset(data + idx * elsz, 0, inc * elsz); - } -} - -JL_DLLEXPORT void jl_array_grow_at(jl_array_t *a, ssize_t idx, size_t inc) -{ - // No need to explicitly unshare. - // Shared arrays are guaranteed to trigger the slow path for growing. - size_t n = jl_array_nrows(a); - if (idx < 0 || idx > n) - jl_bounds_error_int((jl_value_t*)a, idx + 1); - if (idx + 1 < n / 2) { - jl_array_grow_at_beg(a, idx, inc, n); - } - else { - jl_array_grow_at_end(a, idx, inc, n); - } -} - -JL_DLLEXPORT void jl_array_grow_end(jl_array_t *a, size_t inc) -{ - size_t n = jl_array_nrows(a); - jl_array_grow_at_end(a, n, inc, n); -} - -JL_DLLEXPORT void jl_array_grow_beg(jl_array_t *a, size_t inc) -{ - size_t n = jl_array_nrows(a); - jl_array_grow_at_beg(a, 0, inc, n); -} - -STATIC_INLINE void jl_array_shrink(jl_array_t *a, size_t dec) -{ - //if we don't manage this array return - if (a->flags.how == 0) return; - - size_t elsz = a->elsize; - size_t newbytes = (a->maxsize - dec) * a->elsize; - size_t oldnbytes = (a->maxsize) * a->elsize; - int isbitsunion = jl_array_isbitsunion(a); - if (isbitsunion) { - newbytes += a->maxsize - dec; - oldnbytes += a->maxsize; - } - - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isbitsunion) { - newbytes++; - oldnbytes++; - } - char *originalptr = ((char*) a->data) - a->offset * a->elsize; - if (a->flags.how == 1) { - //this is a julia-allocated buffer that needs to be marked - char *typetagdata; - char *newtypetagdata; - if (isbitsunion) { - typetagdata = (char*)malloc_s(a->nrows); - memcpy(typetagdata, jl_array_typetagdata(a), a->nrows); - } - jl_task_t *ct = jl_current_task; - char *originaldata = (char*) a->data - a->offset * a->elsize; - char *newdata = (char*)jl_gc_alloc_buf(ct->ptls, newbytes); - jl_gc_wb_buf(a, newdata, newbytes); - a->maxsize -= dec; - if (isbitsunion) { - newtypetagdata = jl_array_typetagdata(a); - memcpy(newtypetagdata, typetagdata, a->nrows); - free(typetagdata); - } - memcpy(newdata, originaldata, newbytes); - a->data = newdata + a->offset * elsz; - } - else if (a->flags.how == 2) { - //malloc-allocated pointer this array object manages - char *typetagdata; - char *newtypetagdata; - if (isbitsunion) { - typetagdata = (char*)malloc_s(a->nrows); - memcpy(typetagdata, jl_array_typetagdata(a), a->nrows); - } - size_t oldoffsnb = a->offset * elsz; - a->data = ((char*)jl_gc_managed_realloc(originalptr, newbytes, oldnbytes, - a->flags.isaligned, (jl_value_t*) a)) + oldoffsnb; - a->maxsize -= dec; - if (isbitsunion) { - newtypetagdata = jl_array_typetagdata(a); - memcpy(newtypetagdata, typetagdata, a->nrows); - free(typetagdata); - } - } - else if (a->flags.how == 3) { - //this has has a pointer to the object that owns the data - } -} - -static size_t jl_array_limit_offset(jl_array_t *a, size_t offset) -{ - // make sure offset doesn't grow forever due to deleting at beginning - // and growing at end - if (offset >= 13 * a->maxsize / 20) - offset = 17 * (a->maxsize - a->nrows) / 100; -#ifdef _P64 - while (offset > (size_t)UINT32_MAX) { - offset /= 2; - } -#endif - return offset; -} - -STATIC_INLINE void jl_array_del_at_beg(jl_array_t *a, size_t idx, size_t dec, - size_t n) -{ - // no error checking - // assume inbounds, assume unshared - size_t elsz = a->elsize; - size_t offset = a->offset; - int isbitsunion = jl_array_isbitsunion(a); - offset += dec; - a->length = n - dec; - a->nrows = n - dec; - size_t newoffs = jl_array_limit_offset(a, offset); - assert(newoffs <= offset); - size_t nbdec = dec * elsz; - if (__unlikely(newoffs != offset) || idx > 0) { - char *olddata = (char*)a->data; - char *newdata = olddata - (a->offset - newoffs) * elsz; - char *typetagdata = NULL; - char *newtypetagdata = NULL; - if (isbitsunion) { - typetagdata = jl_array_typetagdata(a); - newtypetagdata = typetagdata - (a->offset - newoffs); - } - - size_t nb1 = idx * elsz; // size in bytes of the first block - size_t nbtotal = a->nrows * elsz; // size in bytes of the new array - // Implicit '\0' for byte arrays - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isbitsunion) - nbtotal++; - if (idx > 0) { - memmove_safe(a->flags.hasptr, newdata, olddata, nb1); - if (isbitsunion) memmove(newtypetagdata, typetagdata, idx); - } - // Move the rest of the data if the offset changed - if (newoffs != offset) { - memmove_safe(a->flags.hasptr, newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1); - if (isbitsunion) memmove(newtypetagdata + idx, typetagdata + idx + dec, a->nrows - idx); - } - a->data = newdata; - } - else { - char *data = (char*)a->data; - a->data = data + nbdec; - } - a->offset = newoffs; -} - -STATIC_INLINE void jl_array_del_at_end(jl_array_t *a, size_t idx, size_t dec, - size_t n) -{ - // no error checking - // assume inbounds, assume unshared - char *data = (char*)a->data; - size_t elsz = a->elsize; - int isbitsunion = jl_array_isbitsunion(a); - size_t last = idx + dec; - if (n > last) { - memmove_safe(a->flags.hasptr, data + idx * elsz, data + last * elsz, (n - last) * elsz); - if (isbitsunion) { - char *typetagdata = jl_array_typetagdata(a); - memmove(typetagdata + idx, typetagdata + last, n - last); - } - } - n -= dec; - if (JL_ARRAY_MEMDEBUG_TERMINATOR && elsz == 1 && !isbitsunion) { - data[n] = 0xfe; - msan_allocated_memory(&data[n], 1); - } - a->nrows = n; - a->length = n; -} - -JL_DLLEXPORT void jl_array_del_at(jl_array_t *a, ssize_t idx, size_t dec) -{ - size_t n = jl_array_nrows(a); - size_t last = idx + dec; - if (__unlikely(idx < 0)) - jl_bounds_error_int((jl_value_t*)a, idx + 1); - if (__unlikely(last > n)) - jl_bounds_error_int((jl_value_t*)a, last); - // The unsharing needs to happen before we modify the buffer - if (__unlikely(a->flags.isshared)) - array_try_unshare(a); - if (idx < n - last) { - jl_array_del_at_beg(a, idx, dec, n); - } - else { - jl_array_del_at_end(a, idx, dec, n); - } -} - -JL_DLLEXPORT void jl_array_del_beg(jl_array_t *a, size_t dec) -{ - size_t n = jl_array_nrows(a); - if (__unlikely(dec > n)) - jl_bounds_error_int((jl_value_t*)a, dec); - if (__unlikely(a->flags.isshared)) - array_try_unshare(a); - if (dec == 0) - return; - jl_array_del_at_beg(a, 0, dec, n); + a->dimsize[0] = newnrows; } JL_DLLEXPORT void jl_array_del_end(jl_array_t *a, size_t dec) { + // assume inbounds, assume unshared size_t n = jl_array_nrows(a); if (__unlikely(n < dec)) jl_bounds_error_int((jl_value_t*)a, 0); - if (__unlikely(a->flags.isshared)) - array_try_unshare(a); - if (dec == 0) + // TODO(jwn): unsharing? + if (__unlikely(dec == 0)) + return; + char *data = jl_array_data(a,char); + size_t elsz = jl_array_elsize(a); + if (jl_genericmemory_isbitsunion(a->ref.mem)) + data = (char*)a->ref.mem->data + (size_t)data; + else if (__unlikely(elsz == 0)) { + a->dimsize[0] = n - dec; return; - jl_array_del_at_end(a, n - dec, dec, n); -} - -JL_DLLEXPORT void jl_array_sizehint(jl_array_t *a, size_t sz) -{ - size_t n = jl_array_nrows(a); - - size_t min = a->offset + a->length; - sz = (sz < min) ? min : sz; - - if (sz <= a->maxsize) { - size_t dec = a->maxsize - sz; - //if we don't save at least an eighth of maxsize then its not worth it to shrink - if (dec <= a->maxsize / 8) return; - jl_array_shrink(a, dec); - } - else { - size_t inc = sz - n; - jl_array_grow_end(a, inc); - - a->nrows = n; - a->length = n; } -} - -JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary) -{ - size_t elsz = ary->elsize; - size_t len = jl_array_len(ary); - int isunion = jl_is_uniontype(jl_tparam0(jl_typeof(ary))); - jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary), - &ary->nrows, !ary->flags.ptrarray, - ary->flags.hasptr, isunion, 0, elsz); - memcpy(new_ary->data, ary->data, len * elsz); - // ensure isbits union arrays copy their selector bytes correctly - if (jl_array_isbitsunion(ary)) - memcpy(jl_array_typetagdata(new_ary), jl_array_typetagdata(ary), len); - return new_ary; + n -= dec; + // don't leave behind deleted data + if (jl_is_genericmemory_zeroinit(a->ref.mem)) + memset(data + n * elsz, 0, elsz * dec); + a->dimsize[0] = n; } // Copy element by element until we hit a young object, at which point @@ -1246,7 +421,7 @@ static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner, JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT { - assert(dest->flags.ptrarray && src->flags.ptrarray); + assert(jl_array_ptrarray(dest) && jl_array_ptrarray(src)); jl_value_t *owner = jl_array_owner(dest); // Destination is old and doesn't refer to any young object if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) { @@ -1290,11 +465,80 @@ JL_DLLEXPORT void jl_array_ptr_1d_append(jl_array_t *a, jl_array_t *a2) } } -JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a) JL_NOTSAFEPOINT +JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary) { - return jl_array_data_owner(a); + size_t elsz = jl_array_elsize(ary); + size_t len = jl_array_len(ary); + jl_task_t *ct = jl_current_task; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(ary->ref.mem))->layout; + int ndims = jl_array_ndims(ary); + jl_genericmemory_t *mem = _new_genericmemory_((jl_value_t*)jl_typetagof(ary->ref.mem), len, layout->arrayelem_isunion, 0, (size_t)layout->size); + // ensure isbits union arrays copy their selector bytes correctly + if (layout->arrayelem_isunion) { + memcpy(mem->data, (char*)ary->ref.mem->data + (size_t)jl_array_data_(ary) * elsz, len * elsz); + memcpy(jl_genericmemory_typetagdata(mem), jl_array_typetagdata(ary), len); + } + else { + memcpy(mem->data, jl_array_data_(ary), len * elsz); + } + JL_GC_PUSH1(&mem); + int tsz = sizeof(jl_array_t) + ndims*sizeof(size_t); + jl_array_t *new_ary = (jl_array_t*)jl_gc_alloc(ct->ptls, tsz, (jl_value_t*)jl_typetagof(ary)); + new_ary->ref.mem = mem; + if (layout->arrayelem_isunion || layout->size == 0) + new_ary->ref.data = 0; + else + new_ary->ref.data = mem->data; + memcpy(&new_ary->dimsize, &ary->dimsize, ndims * sizeof(size_t)); + JL_GC_POP(); + return new_ary; } +JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len) +{ + if (len == 0) + return jl_an_empty_string; + size_t sz = sizeof(size_t) + len + 1; // add space for trailing \nul protector and size + if (sz < len) // overflow + jl_throw(jl_memory_exception); + jl_task_t *ct = jl_current_task; + jl_value_t *s; + jl_ptls_t ptls = ct->ptls; + const size_t allocsz = sz + sizeof(jl_taggedvalue_t); + if (sz <= GC_MAX_SZCLASS) { + int pool_id = jl_gc_szclass_align8(allocsz); + jl_gc_pool_t *p = &ptls->heap.norm_pools[pool_id]; + int osize = jl_gc_sizeclasses[pool_id]; + // We call `jl_gc_pool_alloc_noinline` instead of `jl_gc_pool_alloc` to avoid double-counting in + // the Allocations Profiler. (See https://github.com/JuliaLang/julia/pull/43868 for more details.) + s = jl_gc_pool_alloc_noinline(ptls, (char*)p - (char*)ptls, osize); + } + else { + if (allocsz < sz) // overflow in adding offs, size was "negative" + jl_throw(jl_memory_exception); + s = jl_gc_big_alloc_noinline(ptls, allocsz); + } + jl_set_typetagof(s, jl_string_tag, 0); + maybe_record_alloc_to_profile(s, len, jl_string_type); + *(size_t*)s = len; + jl_string_data(s)[len] = 0; + return s; +} + +JL_DLLEXPORT jl_value_t *jl_pchar_to_string(const char *str, size_t len) +{ + jl_value_t *s = jl_alloc_string(len); + if (len > 0) + memcpy(jl_string_data(s), str, len); + return s; +} + +JL_DLLEXPORT jl_value_t *jl_cstr_to_string(const char *str) +{ + return jl_pchar_to_string(str, strlen(str)); +} + + #ifdef __cplusplus } #endif diff --git a/src/ast.c b/src/ast.c index 600b1a229ea80..89ee3f70bf2fd 100644 --- a/src/ast.c +++ b/src/ast.c @@ -650,9 +650,9 @@ static value_t julia_to_scm(fl_context_t *fl_ctx, jl_value_t *v) static void array_to_list(fl_context_t *fl_ctx, jl_array_t *a, value_t *pv, int check_valid) { value_t temp; - for(long i=jl_array_len(a)-1; i >= 0; i--) { + for (long i = jl_array_nrows(a) - 1; i >= 0; i--) { *pv = fl_cons(fl_ctx, fl_ctx->NIL, *pv); - temp = julia_to_scm_(fl_ctx, jl_array_ptr_ref(a,i), check_valid); + temp = julia_to_scm_(fl_ctx, jl_array_ptr_ref(a, i), check_valid); // note: must be separate statement car_(*pv) = temp; } @@ -880,7 +880,7 @@ JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr) JL_GC_PUSH2(&new_ci, &new_code); new_ci = jl_copy_code_info(new_ci); new_code = jl_array_copy(new_ci->code); - size_t clen = jl_array_len(new_code); + size_t clen = jl_array_nrows(new_code); for (int i = 0; i < clen; ++i) { jl_array_ptr_set(new_code, i, jl_copy_ast( jl_array_ptr_ref(new_code, i) @@ -913,7 +913,7 @@ JL_DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr) } if (jl_is_expr(expr)) { jl_expr_t *e = (jl_expr_t*)expr; - size_t i, l = jl_array_len(e->args); + size_t i, l = jl_array_nrows(e->args); jl_expr_t *ne = jl_exprn(e->head, l); JL_GC_PUSH2(&ne, &expr); for (i = 0; i < l; i++) { @@ -991,11 +991,11 @@ JL_DLLEXPORT int jl_operator_precedence(char *sym) int jl_has_meta(jl_array_t *body, jl_sym_t *sym) JL_NOTSAFEPOINT { - size_t i, l = jl_array_len(body); + size_t i, l = jl_array_nrows(body); for (i = 0; i < l; i++) { jl_expr_t *stmt = (jl_expr_t*)jl_array_ptr_ref(body, i); if (jl_is_expr((jl_value_t*)stmt) && stmt->head == jl_meta_sym) { - size_t i, l = jl_array_len(stmt->args); + size_t i, l = jl_array_nrows(stmt->args); for (i = 0; i < l; i++) if (jl_array_ptr_ref(stmt->args, i) == (jl_value_t*)sym) return 1; @@ -1060,7 +1060,7 @@ static jl_value_t *jl_invoke_julia_macro(jl_array_t *args, jl_module_t *inmodule { jl_task_t *ct = jl_current_task; JL_TIMING(MACRO_INVOCATION, MACRO_INVOCATION); - size_t nargs = jl_array_len(args) + 1; + size_t nargs = jl_array_nrows(args) + 1; JL_NARGSV("macrocall", 3); // macro name, location, and module jl_value_t **margs; JL_GC_PUSHARGS(margs, nargs); @@ -1199,7 +1199,7 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str } size_t i; - for (i = 0; i < jl_array_len(e->args); i++) { + for (i = 0; i < jl_array_nrows(e->args); i++) { jl_value_t *a = jl_array_ptr_ref(e->args, i); jl_value_t *a2 = jl_expand_macros(a, inmodule, macroctx, onelevel, world, throw_load_error); if (a != a2) diff --git a/src/builtin_proto.h b/src/builtin_proto.h index 64e3fbd1af366..ef9188c054529 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -25,14 +25,14 @@ DECLARE_BUILTIN(applicable); DECLARE_BUILTIN(_apply_iterate); DECLARE_BUILTIN(_apply_pure); DECLARE_BUILTIN(apply_type); -DECLARE_BUILTIN(arrayref); -DECLARE_BUILTIN(arrayset); -DECLARE_BUILTIN(arraysize); +DECLARE_BUILTIN(memoryref); +DECLARE_BUILTIN(memoryrefget); +DECLARE_BUILTIN(memoryrefset); +DECLARE_BUILTIN(memoryref_isassigned); DECLARE_BUILTIN(_call_in_world); DECLARE_BUILTIN(_call_in_world_total); DECLARE_BUILTIN(_call_latest); DECLARE_BUILTIN(replacefield); -DECLARE_BUILTIN(const_arrayref); DECLARE_BUILTIN(_expr); DECLARE_BUILTIN(fieldtype); DECLARE_BUILTIN(getfield); diff --git a/src/builtins.c b/src/builtins.c index 2defeb4b3d833..3296fff042844 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -511,21 +511,24 @@ JL_CALLABLE(jl_f_sizeof) } if (jl_is_datatype(x)) { jl_datatype_t *dx = (jl_datatype_t*)x; - if (dx->layout == NULL) { + if (!jl_struct_try_layout(dx)) { if (dx->name->abstract) jl_errorf("Abstract type %s does not have a definite size.", jl_symbol_name(dx->name->name)); else jl_errorf("Argument is an incomplete %s type and does not have a definite size.", jl_symbol_name(dx->name->name)); } - if (jl_is_layout_opaque(dx->layout)) + if (jl_is_layout_opaque(dx->layout)) // includes all GenericMemory{isatomic,T} jl_errorf("Type %s does not have a definite size.", jl_symbol_name(dx->name->name)); return jl_box_long(jl_datatype_size(x)); } if (x == jl_bottom_type) jl_error("The empty type does not have a definite size since it does not have instances."); +#if 0 // TODO(jwn): is this breaking? if (jl_is_array(x)) { - return jl_box_long(jl_array_len(x) * ((jl_array_t*)x)->elsize); + return jl_box_long(jl_array_len(x) * jl_datatype_size((jl_datatype_t*)jl_typeof(((jl_array_t*)x)->ref.mem))); + } +#endif if (jl_is_string(x)) return jl_box_long(jl_string_len(x)); if (jl_is_symbol(x)) @@ -535,7 +538,10 @@ JL_CALLABLE(jl_f_sizeof) jl_datatype_t *dt = (jl_datatype_t*)jl_typeof(x); assert(jl_is_datatype(dt)); assert(!dt->name->abstract); - return jl_box_long(jl_datatype_size(dt)); + size_t sz = jl_datatype_size(dt); + if (jl_is_genericmemory(x)) // TODO(jwn): add bitsunion bytes? + sz *= ((jl_genericmemory_t*)x)->length; + return jl_box_long(sz); } JL_CALLABLE(jl_f_issubtype) @@ -607,6 +613,12 @@ STATIC_INLINE void _grow_to(jl_value_t **root, jl_value_t ***oldargs, jl_svec_t *n_alloc = newalloc; } + +static jl_value_t *jl_arrayref(jl_array_t *a, size_t i) +{ + return jl_memoryrefget(jl_memoryrefindex(a->ref, i)); +} + static jl_value_t *do_apply(jl_value_t **args, uint32_t nargs, jl_value_t *iterate) { jl_function_t *f = args[0]; @@ -715,7 +727,9 @@ static jl_value_t *do_apply(jl_value_t **args, uint32_t nargs, jl_value_t *itera precount = (precount > al) ? precount - al : 0; _grow_to(&roots[0], &newargs, &arg_heap, &n_alloc, n + precount + al, extra); assert(newargs != NULL); // inform GCChecker that we didn't write a NULL here - if (aai->flags.ptrarray) { + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(aai->ref.mem))->layout; + if (layout->arrayelem_isboxed) + { for (j = 0; j < al; j++) { jl_value_t *arg = jl_array_ptr_ref(aai, j); // apply with array splatting may have embedded NULL value (#11772) @@ -1463,70 +1477,96 @@ JL_CALLABLE(jl_f__typevar) return (jl_value_t *)jl_new_typevar((jl_sym_t*)args[0], args[1], args[2]); } -// arrays --------------------------------------------------------------------- +// genericmemory --------------------------------------------------------------------- -JL_CALLABLE(jl_f_arraysize) +JL_CALLABLE(jl_f_memoryref) { - JL_NARGS(arraysize, 2, 2); - JL_TYPECHK(arraysize, array, args[0]); - jl_array_t *a = (jl_array_t*)args[0]; - size_t nd = jl_array_ndims(a); - JL_TYPECHK(arraysize, long, args[1]); - int dno = jl_unbox_long(args[1]); - if (dno < 1) - jl_error("arraysize: dimension out of range"); - if (dno > nd) - return jl_box_long(1); - return jl_box_long((&a->nrows)[dno-1]); -} - -static size_t array_nd_index(jl_array_t *a, jl_value_t **args, size_t nidxs, - const char *fname) -{ - size_t i = 0; - size_t k, stride = 1; - size_t nd = jl_array_ndims(a); - for (k = 0; k < nidxs; k++) { - if (!jl_is_long(args[k])) - jl_type_error(fname, (jl_value_t*)jl_long_type, args[k]); - size_t ii = jl_unbox_long(args[k]) - 1; - i += ii * stride; - size_t d = (k >= nd) ? 1 : jl_array_dim(a, k); - if (k < nidxs - 1 && ii >= d) - jl_bounds_error_v((jl_value_t*)a, args, nidxs); - stride *= d; + JL_NARGS(memoryref, 1, 3); + if (nargs == 1) { + JL_TYPECHK(memoryref, genericmemory, args[0]); + jl_genericmemory_t *m = (jl_genericmemory_t*)args[0]; + jl_value_t *typ = jl_apply_type2((jl_value_t*)jl_genericmemoryref_type, jl_tparam0(jl_typetagof(m)), jl_tparam1(jl_typetagof(m))); + JL_GC_PROMISE_ROOTED(typ); // it is a concrete type + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->arrayelem_isunion || layout->size == 0) + return (jl_value_t*)jl_new_memoryref(typ, m, 0); + return (jl_value_t*)jl_new_memoryref(typ, m, m->data); } - for (; k < nd; k++) - stride *= jl_array_dim(a, k); - if (i >= stride) - jl_bounds_error_v((jl_value_t*)a, args, nidxs); - return i; -} - -JL_CALLABLE(jl_f_arrayref) -{ - JL_NARGSV(arrayref, 3); - JL_TYPECHK(arrayref, bool, args[0]); - JL_TYPECHK(arrayref, array, args[1]); - jl_array_t *a = (jl_array_t*)args[1]; - size_t i = array_nd_index(a, &args[2], nargs - 2, "arrayref"); - return jl_arrayref(a, i); -} - -JL_CALLABLE(jl_f_const_arrayref) -{ - return jl_f_arrayref(F, args, nargs); + else { + JL_TYPECHK(memoryref, genericmemoryref, args[0]); + JL_TYPECHK(memoryref, long, args[1]); + if (nargs == 3) + JL_TYPECHK(memoryref, bool, args[2]); + jl_genericmemoryref_t *m = (jl_genericmemoryref_t*)args[0]; + size_t i = jl_unbox_long(args[1]) - 1; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m->mem))->layout; + char *data = (char*)m->data; + if (layout->arrayelem_isboxed) { + if (((data - (char*)m->mem->data) / sizeof(jl_value_t*)) + i >= m->mem->length) + jl_bounds_error((jl_value_t*)m, args[1]); + data += sizeof(jl_value_t*) * i; + } + else if (layout->arrayelem_isunion || layout->size == 0) { + if ((size_t)data + i >= m->mem->length) + jl_bounds_error((jl_value_t*)m, args[1]); + data += i; + } + else { + if (((data - (char*)m->mem->data) / layout->size) + i >= m->mem->length) + jl_bounds_error((jl_value_t*)m, args[1]); + data += layout->size * i; + } + return (jl_value_t*)jl_new_memoryref((jl_value_t*)jl_typetagof(m), m->mem, data); + } +} + +JL_CALLABLE(jl_f_memoryrefget) +{ + JL_NARGS(memoryrefget, 3, 3); + JL_TYPECHK(memoryrefget, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefget, symbol, args[1]); + JL_TYPECHK(memoryrefget, bool, args[2]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); + if (isatomic == jl_false) + if (args[1] != (jl_value_t*)jl_not_atomic_sym) + jl_atomic_error("memoryrefget!: non-atomic memory cannot be accessed atomically"); + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + return jl_memoryrefget(m); +} + +JL_CALLABLE(jl_f_memoryrefset) +{ + JL_NARGS(memoryrefset, 4, 4); + JL_TYPECHK(memoryrefset, genericmemoryref, args[0]); + JL_TYPECHK(memoryrefset, symbol, args[2]); + JL_TYPECHK(memoryrefset, bool, args[3]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); + if (isatomic == jl_false) + if (args[2] != (jl_value_t*)jl_not_atomic_sym) + jl_atomic_error("memoryrefset!: non-atomic memory cannot be written atomically"); + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + jl_memoryrefset(m, args[1]); + return args[0]; } -JL_CALLABLE(jl_f_arrayset) +JL_CALLABLE(jl_f_memoryref_isassigned) { - JL_NARGSV(arrayset, 4); - JL_TYPECHK(arrayset, bool, args[0]); - JL_TYPECHK(arrayset, array, args[1]); - jl_array_t *a = (jl_array_t*)args[1]; - size_t i = array_nd_index(a, &args[3], nargs - 3, "arrayset"); - jl_arrayset(a, args[2], i); - return args[1]; + JL_NARGS(memoryref_isassigned, 3, 3); + JL_TYPECHK(memoryref_isassigned, genericmemoryref, args[0]); + JL_TYPECHK(memoryref_isassigned, symbol, args[1]); + JL_TYPECHK(memoryref_isassigned, bool, args[2]); + jl_genericmemoryref_t m = *(jl_genericmemoryref_t*)args[0]; + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); + if (isatomic == jl_false) + if (args[1] != (jl_value_t*)jl_not_atomic_sym) + jl_atomic_error("memoryref_isassigned!: non-atomic memory cannot be accessed atomically"); + if (m.mem->length == 0) + return jl_false; + return jl_memoryref_isassigned(m); } // type definition ------------------------------------------------------------ @@ -1719,7 +1759,7 @@ static int references_name(jl_value_t *p, jl_typename_t *name, int affects_layou jl_datatype_t *dp = (jl_datatype_t*)p; if (affects_layout && dp->name == name) return 1; - affects_layout = ((jl_datatype_t*)jl_unwrap_unionall(dp->name->wrapper))->layout == NULL; + affects_layout = jl_is_genericmemory_type(dp) || ((jl_datatype_t*)jl_unwrap_unionall(dp->name->wrapper))->layout == NULL; // and even if it has a layout, the fields themselves might trigger layouts if they use tparam i // rather than checking this for each field, we just assume it applies if (!affects_layout && freevars && jl_field_names(dp) != jl_emptysvec) { @@ -2024,11 +2064,11 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin_func("get_binding_type", jl_f_get_binding_type); add_builtin_func("set_binding_type!", jl_f_set_binding_type); - // array primitives - jl_builtin_arrayref = add_builtin_func("arrayref", jl_f_arrayref); - jl_builtin_const_arrayref = add_builtin_func("const_arrayref", jl_f_arrayref); - jl_builtin_arrayset = add_builtin_func("arrayset", jl_f_arrayset); - jl_builtin_arraysize = add_builtin_func("arraysize", jl_f_arraysize); + // memory primitives + jl_builtin_memoryref = add_builtin_func("memoryref", jl_f_memoryref); + jl_builtin_memoryrefget = add_builtin_func("memoryrefget", jl_f_memoryrefget); + jl_builtin_memoryrefset = add_builtin_func("memoryrefset", jl_f_memoryrefset); + jl_builtin_memoryref_isassigned = add_builtin_func("memoryref_isassigned", jl_f_memoryref_isassigned); // method table utils jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable); @@ -2101,6 +2141,8 @@ void jl_init_primitives(void) JL_GC_DISABLED add_builtin("AbstractArray", (jl_value_t*)jl_abstractarray_type); add_builtin("DenseArray", (jl_value_t*)jl_densearray_type); add_builtin("Array", (jl_value_t*)jl_array_type); + add_builtin("GenericMemory", (jl_value_t*)jl_genericmemory_type); + add_builtin("MemoryRef", (jl_value_t*)jl_genericmemoryref_type); add_builtin("Expr", (jl_value_t*)jl_expr_type); add_builtin("LineNumberNode", (jl_value_t*)jl_linenumbernode_type); diff --git a/src/ccall.cpp b/src/ccall.cpp index 1add621edde28..a3f3380c90b16 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -29,7 +29,6 @@ TRANSFORMED_CCALL_STAT(jl_sigatomic_begin); TRANSFORMED_CCALL_STAT(jl_sigatomic_end); TRANSFORMED_CCALL_STAT(jl_svec_len); TRANSFORMED_CCALL_STAT(jl_svec_ref); -TRANSFORMED_CCALL_STAT(jl_array_isassigned); TRANSFORMED_CCALL_STAT(jl_string_ptr); TRANSFORMED_CCALL_STAT(jl_symbol_name); TRANSFORMED_CCALL_STAT(memcpy); @@ -746,7 +745,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar jl_value_t *ir_arg = args[1]; JL_GC_PUSH4(&ir, &rt, &at, &entry); if (jl_is_ssavalue(ir_arg)) - ir_arg = jl_arrayref((jl_array_t*)ctx.source->code, ((jl_ssavalue_t*)ir_arg)->id - 1); + ir_arg = jl_array_ptr_ref((jl_array_t*)ctx.source->code, ((jl_ssavalue_t*)ir_arg)->id - 1); ir = static_eval(ctx, ir_arg); if (!ir) { emit_error(ctx, "error statically evaluating llvm IR argument"); @@ -754,7 +753,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar return jl_cgval_t(); } if (jl_is_ssavalue(args[2]) && !jl_is_long(ctx.source->ssavaluetypes)) { - jl_value_t *rtt = jl_arrayref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[2])->id - 1); + jl_value_t *rtt = jl_array_ptr_ref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[2])->id - 1); if (jl_is_type_type(rtt)) rt = jl_tparam0(rtt); } @@ -767,7 +766,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar } } if (jl_is_ssavalue(args[3]) && !jl_is_long(ctx.source->ssavaluetypes)) { - jl_value_t *att = jl_arrayref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[3])->id - 1); + jl_value_t *att = jl_array_ptr_ref((jl_array_t*)ctx.source->ssavaluetypes, ((jl_ssavalue_t*)args[3])->id - 1); if (jl_is_type_type(att)) at = jl_tparam0(att); } @@ -911,7 +910,7 @@ static jl_cgval_t emit_llvmcall(jl_codectx_t &ctx, jl_value_t **args, size_t nar } else { auto Buf = MemoryBuffer::getMemBuffer( - StringRef((char *)jl_array_data(ir), jl_array_len(ir)), "llvmcall", + StringRef(jl_array_data(ir, char), jl_array_nrows(ir)), "llvmcall", /*RequiresNullTerminator*/ false); Expected> ModuleOrErr = parseBitcodeFile(*Buf, ctx.builder.getContext()); @@ -1140,7 +1139,8 @@ std::string generate_func_sig(const char *fname) } Type *pat; - if (!jl_is_datatype(tti) || ((jl_datatype_t*)tti)->layout == NULL || jl_is_layout_opaque(((jl_datatype_t*)tti)->layout)) { + // n.b. `Array` used as argument type just passes a julia object reference + if (!jl_is_datatype(tti) || ((jl_datatype_t*)tti)->layout == NULL || jl_is_array_type(tti) || jl_is_layout_opaque(((jl_datatype_t*)tti)->layout)) { tti = (jl_value_t*)jl_voidpointer_type; // passed as pointer } @@ -1271,7 +1271,7 @@ static const std::string verify_ccall_sig(jl_value_t *&rt, jl_value_t *at, JL_TYPECHK(ccall, type, rt); JL_TYPECHK(ccall, simplevector, at); - if (rt == (jl_value_t*)jl_any_type || jl_is_array_type(rt) || + if (rt == (jl_value_t*)jl_any_type || jl_is_array_type(rt) || jl_is_genericmemory_type(rt) || (jl_is_datatype(rt) && ((jl_datatype_t*)rt)->layout != NULL && jl_is_layout_opaque(((jl_datatype_t*)rt)->layout))) { // n.b. `Array` used as return type just returns a julia object reference @@ -1349,11 +1349,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) return jl_cgval_t(); } - auto ccallarg = [=] (size_t i) { - assert(i < nccallargs && i + fc_args_start <= nargs); - return args[fc_args_start + i]; - }; - auto _is_libjulia_func = [&] (uintptr_t ptr, StringRef name) { if ((uintptr_t)fptr == ptr) return true; @@ -1380,7 +1375,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) SmallVector argv(nccallargs); for (size_t i = 0; i < nccallargs; i++) { // Julia (expression) value of current parameter - jl_value_t *argi = ccallarg(i); + assert(i < nccallargs && i + fc_args_start <= nargs); + jl_value_t *argi = args[fc_args_start + i]; argv[i] = emit_expr(ctx, argi); } @@ -1458,16 +1454,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) // some special functions bool isVa = nreqargs > 0; (void)isVa; // prevent compiler warning - if (is_libjulia_func(jl_array_ptr)) { - ++CCALL_STAT(jl_array_ptr); - assert(lrt == ctx.types().T_size); - assert(!isVa && !llvmcall && nccallargs == 1); - const jl_cgval_t &ary = argv[0]; - JL_GC_POP(); - return mark_or_box_ccall_result(ctx, ctx.builder.CreatePtrToInt(emit_unsafe_arrayptr(ctx, ary), lrt), - retboxed, rt, unionall, static_rt); - } - else if (is_libjulia_func(jl_value_ptr)) { + if (is_libjulia_func(jl_value_ptr)) { ++CCALL_STAT(jl_value_ptr); assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_size); assert(!isVa && !llvmcall && nccallargs == 1); @@ -1743,47 +1730,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) JL_GC_POP(); return mark_or_box_ccall_result(ctx, load, retboxed, rt, unionall, static_rt); } - else if (is_libjulia_func(jl_array_isassigned) && - argv[1].typ == (jl_value_t*)jl_ulong_type) { - ++CCALL_STAT(jl_array_isassigned); - assert(!isVa && !llvmcall && nccallargs == 2); - jl_value_t *aryex = ccallarg(0); - const jl_cgval_t &aryv = argv[0]; - const jl_cgval_t &idxv = argv[1]; - jl_datatype_t *arydt = (jl_datatype_t*)jl_unwrap_unionall(aryv.typ); - if (jl_is_array_type(arydt)) { - jl_value_t *ety = jl_tparam0(arydt); - bool ptrarray = !jl_stored_inline(ety); - if (!ptrarray && !jl_type_hasptr(ety)) { - JL_GC_POP(); - return mark_or_box_ccall_result(ctx, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 1), - false, rt, unionall, static_rt); - } - else if (!jl_has_free_typevars(ety)) { - Value *idx = emit_unbox(ctx, ctx.types().T_size, idxv, (jl_value_t*)jl_ulong_type); - Value *arrayptr = emit_bitcast(ctx, emit_arrayptr(ctx, aryv, aryex), ctx.types().T_pprjlvalue); - if (!ptrarray) { - size_t elsz = jl_datatype_size(ety); - unsigned align = jl_datatype_align(ety); - size_t stride = LLT_ALIGN(elsz, align) / sizeof(jl_value_t*); - if (stride != 1) - idx = ctx.builder.CreateMul(idx, ConstantInt::get(ctx.types().T_size, stride)); - idx = ctx.builder.CreateAdd(idx, ConstantInt::get(ctx.types().T_size, ((jl_datatype_t*)ety)->layout->first_ptr)); - setName(ctx.emission_context, idx, "array_idx"); - } - Value *slot_addr = ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, arrayptr, idx); - setName(ctx.emission_context, slot_addr, "array_slot_addr"); - LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, slot_addr, Align(sizeof(void*))); - setName(ctx.emission_context, load, "array_slot"); - load->setAtomic(AtomicOrdering::Unordered); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_ptrarraybuf); - ai.decorateInst(load); - Value *res = ctx.builder.CreateZExt(ctx.builder.CreateICmpNE(load, Constant::getNullValue(ctx.types().T_prjlvalue)), getInt32Ty(ctx.builder.getContext())); - JL_GC_POP(); - return mark_or_box_ccall_result(ctx, res, retboxed, rt, unionall, static_rt); - } - } - } else if (is_libjulia_func(jl_string_ptr)) { ++CCALL_STAT(jl_string_ptr); assert(lrt == ctx.types().T_size); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index fd9da27d78752..b3b413b91cd1d 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -28,13 +28,8 @@ STATISTIC(EmittedGetfieldKnowns, "Number of known getfield calls emitted"); STATISTIC(EmittedSetfield, "Number of setfield calls emitted"); STATISTIC(EmittedUnionLoads, "Number of union loads emitted"); STATISTIC(EmittedVarargsLength, "Number of varargs length calls emitted"); -STATISTIC(EmittedArraysize, "Number of arraysize calls emitted"); -STATISTIC(EmittedArraylen, "Number of array length calls emitted"); -STATISTIC(EmittedArrayptr, "Number of array data pointer loads emitted"); -STATISTIC(EmittedArrayflags, "Number of arrayflags calls emitted"); -STATISTIC(EmittedArrayNDims, "Number of array ndims calls emitted"); +STATISTIC(EmittedArrayptr, "Number of array ptr calls emitted"); STATISTIC(EmittedArrayElsize, "Number of array elsize calls emitted"); -STATISTIC(EmittedArrayOffset, "Number of array offset calls emitted"); STATISTIC(EmittedArrayNdIndex, "Number of array nd index calls emitted"); STATISTIC(EmittedBoxes, "Number of box operations emitted"); STATISTIC(EmittedCPointerChecks, "Number of C pointer checks emitted"); @@ -130,7 +125,7 @@ static Value *stringConstPtr( Value *zero = ConstantInt::get(Type::getInt32Ty(irbuilder.getContext()), 0); Value *Args[] = { zero, zero }; auto gep = irbuilder.CreateInBoundsGEP(gv->getValueType(), - // Addrspacecast in case globals are in non-0 AS + // AddrSpaceCast in case globals are in non-0 AS irbuilder.CreateAddrSpaceCast(gv, gv->getValueType()->getPointerTo(0)), Args); setName(emission_context, gep, "string_const_ptr"); @@ -213,7 +208,7 @@ static DIType *_julia_type_to_di(jl_codegen_params_t *ctx, jl_debugcache_t &debu uint64_t SizeInBits = jl_datatype_nbits(jdt); ditype = dbuilder->createBasicType(tname, SizeInBits, llvm::dwarf::DW_ATE_unsigned); } - else if (jl_is_structtype(jt) && !jl_is_layout_opaque(jdt->layout)) { + else if (jl_is_structtype(jt) && !jl_is_layout_opaque(jdt->layout) && !jl_is_array_type(jdt)) { size_t ntypes = jl_datatype_nfields(jdt); std::vector Elements(ntypes); for (unsigned i = 0; i < ntypes; i++) { @@ -686,6 +681,34 @@ static unsigned jl_field_align(jl_datatype_t *dt, size_t i) return std::min({al, (unsigned)jl_datatype_align(dt), (unsigned)JL_HEAP_ALIGNMENT}); } +static llvm::StructType* get_jlmemoryref(llvm::LLVMContext &C, unsigned AS) { + return llvm::StructType::get(C, { + llvm::PointerType::get(llvm::Type::getInt8Ty(C), AS), + JuliaType::get_prjlvalue_ty(C), + }); +} +static llvm::StructType* get_jlmemoryboxedref(llvm::LLVMContext &C, unsigned AS) { + return llvm::StructType::get(C, { + llvm::PointerType::get(JuliaType::get_prjlvalue_ty(C), AS), + JuliaType::get_prjlvalue_ty(C), + }); +} +static llvm::StructType* get_jlmemoryunionref(llvm::LLVMContext &C, llvm::Type *T_size) { + return llvm::StructType::get(C, { + T_size, // offset + JuliaType::get_prjlvalue_ty(C), + }); +} +static StructType *get_memoryref_type(LLVMContext &ctxt, Type *T_size, bool isboxed, bool isunion, bool isghost, unsigned AS) +{ + // TODO: try to remove this fairly weird special case + if (isboxed) + return get_jlmemoryboxedref(ctxt, AS); + if (isunion || isghost) + return get_jlmemoryunionref(ctxt, T_size); + return get_jlmemoryref(ctxt, AS); +} + static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, LLVMContext &ctxt, jl_value_t *jt, bool *isboxed, bool llvmcall) { // this function converts a Julia Type into the equivalent LLVM struct @@ -697,7 +720,16 @@ static Type *_julia_struct_to_llvm(jl_codegen_params_t *ctx, LLVMContext &ctxt, if (jl_is_primitivetype(jt)) return bitstype_to_llvm(jt, ctxt, llvmcall); jl_datatype_t *jst = (jl_datatype_t*)jt; - if (jl_is_structtype(jt) && !(jst->layout && jl_is_layout_opaque(jst->layout))) { + if (jl_is_structtype(jt) && !(jst->layout && jl_is_layout_opaque(jst->layout)) && !jl_is_array_type(jst) && !jl_is_genericmemory_type(jst)) { + if (jl_is_genericmemoryref_type(jst)) { + jl_value_t *mty_dt = jl_field_type_concrete(jst, 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + bool isghost = layout->size == 0; + Type *T_size = bitstype_to_llvm((jl_value_t*)jl_long_type, ctxt); + return get_memoryref_type(ctxt, T_size, isboxed, isunion, isghost, 0); + } bool isTuple = jl_is_tuple_type(jt); jl_svec_t *ftypes = jl_get_fieldtypes(jst); size_t i, ntypes = jl_svec_len(ftypes); @@ -1031,20 +1063,6 @@ static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst emit_memcpy_llvm(ctx, dst, dst_ai, data_pointer(ctx, src), src_ai, sz, align, is_volatile); } -static LoadInst *emit_nthptr_recast(jl_codectx_t &ctx, Value *v, Value *idx, MDNode *tbaa, Type *type) -{ - // p = (jl_value_t**)v; *(type*)&p[n] - Value *vptr = ctx.builder.CreateInBoundsGEP( - ctx.types().T_prjlvalue, - emit_bitcast(ctx, maybe_decay_tracked(ctx, v), ctx.types().T_pprjlvalue), - idx); - setName(ctx.emission_context, vptr, "arraysize_ptr"); - LoadInst *load = ctx.builder.CreateLoad(type, emit_bitcast(ctx, vptr, PointerType::get(type, 0))); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); - ai.decorateInst(load); - return load; -} - static Value *emit_tagfrom(jl_codectx_t &ctx, jl_datatype_t *dt) { if (dt->smalltag) @@ -1737,7 +1755,6 @@ static bool bounds_check_enabled(jl_codectx_t &ctx, jl_value_t *inbounds) { static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_value_t *ty, Value *i, Value *len, jl_value_t *boundscheck) { Value *im1 = ctx.builder.CreateSub(i, ConstantInt::get(ctx.types().T_size, 1)); -#if CHECK_BOUNDS==1 if (bounds_check_enabled(ctx, boundscheck)) { ++EmittedBoundschecks; Value *ok = ctx.builder.CreateICmpULT(im1, len); @@ -1773,10 +1790,18 @@ static Value *emit_bounds_check(jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_v ctx.f->getBasicBlockList().push_back(passBB); ctx.builder.SetInsertPoint(passBB); } -#endif return im1; } +static Value *CreateSimplifiedExtractValue(jl_codectx_t &ctx, Value *Agg, ArrayRef Idxs) +{ + // aka IRBuilder + SimplifyQuery SQ(jl_Module->getDataLayout()); // not actually used, but required by API + if (Value *Inst = simplifyExtractValueInst(Agg, Idxs, SQ)) + return Inst; + return ctx.builder.CreateExtractValue(Agg, Idxs); +} + static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt); static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile=false); @@ -2487,6 +2512,29 @@ static jl_cgval_t emit_unionload(jl_codectx_t &ctx, Value *addr, Value *ptindex, return mark_julia_slot(fsz > 0 ? addr : nullptr, jfty, tindex, tbaa); } +static MDNode *best_field_tbaa(jl_codectx_t &ctx, const jl_cgval_t &strct, jl_datatype_t *jt, unsigned idx, size_t byte_offset) +{ + auto tbaa = strct.tbaa; + if (tbaa == ctx.tbaa().tbaa_datatype) + if (byte_offset != offsetof(jl_datatype_t, types)) + return ctx.tbaa().tbaa_const; + if (tbaa == ctx.tbaa().tbaa_array) { + if (jl_is_genericmemory_type(jt)) { + if (idx == 0) + return ctx.tbaa().tbaa_memorylen; + if (idx == 1) + return ctx.tbaa().tbaa_memoryptr; + } + else if (jl_is_array_type(jt)) { + if (idx == 0) + return ctx.tbaa().tbaa_arrayptr; + if (idx == 1) + return ctx.tbaa().tbaa_arraysize; + } + } + return tbaa; +} + // If `nullcheck` is not NULL and a pointer NULL check is necessary // store the pointer to be checked in `*nullcheck` instead of checking it static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &strct, @@ -2519,10 +2567,8 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st size_t nfields = jl_datatype_nfields(jt); bool maybe_null = idx >= nfields - (unsigned)jt->name->n_uninitialized; size_t byte_offset = jl_field_offset(jt, idx); - auto tbaa = strct.tbaa; - if (tbaa == ctx.tbaa().tbaa_datatype && byte_offset != offsetof(jl_datatype_t, types)) - tbaa = ctx.tbaa().tbaa_const; if (strct.ispointer()) { + auto tbaa = best_field_tbaa(ctx, strct, jt, idx, byte_offset); Value *staddr = data_pointer(ctx, strct); bool isboxed; Type *lt = julia_type_to_llvm(ctx, (jl_value_t*)jt, &isboxed); @@ -2693,404 +2739,107 @@ static Value *emit_n_varargs(jl_codectx_t &ctx) #endif } -static bool arraytype_constdim(jl_value_t *ty, size_t *dim) +static Value *emit_genericmemoryelsize(jl_codectx_t &ctx, Value *v, jl_value_t *typ) { - if (jl_is_array_type(ty) && jl_is_long(jl_tparam1(ty))) { - *dim = jl_unbox_long(jl_tparam1(ty)); - return true; + ++EmittedArrayElsize; + jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(typ); + if (jl_is_datatype(sty) && !jl_has_free_typevars((jl_value_t*)sty) && sty->layout) { + if (jl_is_array_type(sty)) + sty = (jl_datatype_t*)jl_field_type_concrete(sty, 0); + if (jl_is_genericmemoryref_type(sty)) + sty = (jl_datatype_t*)jl_field_type_concrete(sty, 1); + return ConstantInt::get(ctx.types().T_size, sty->layout->size); + } + else { + v = emit_bitcast(ctx, v, ctx.types().T_prjlvalue); + Value *t = emit_typeof(ctx, v, false, false, true); + Value *elsize = emit_datatype_size(ctx, t); + return ctx.builder.CreateZExt(elsize, ctx.types().T_size); } - return false; -} - -static bool arraytype_constshape(jl_value_t *ty) -{ - size_t dim; - if (!arraytype_constdim(ty, &dim)) - return false; - return dim != 1; } -static bool arraytype_constelsize(jl_datatype_t *ty, size_t *elsz) +static ssize_t genericmemoryype_constelsize(jl_value_t *typ) { - assert(jl_is_array_type(ty)); - jl_value_t *ety = jl_tparam0(ty); - if (jl_has_free_typevars(ety)) - return false; - // `jl_islayout_inline` requires `*elsz` and `al` to be initialized. - size_t al = 0; - *elsz = 0; - int union_max = jl_islayout_inline(ety, elsz, &al); - bool isboxed = (union_max == 0); - if (isboxed) { - *elsz = sizeof(void*); - } - else if (jl_is_primitivetype(ety)) { - // Primitive types should use the array element size, but - // this can be different from the type's size - *elsz = LLT_ALIGN(*elsz, al); + jl_datatype_t *sty = (jl_datatype_t*)jl_unwrap_unionall(typ); + if (jl_is_datatype(sty) && !jl_has_free_typevars((jl_value_t*)sty) && sty->layout) { + if (jl_is_array_type(sty)) + sty = (jl_datatype_t*)jl_field_type_concrete(sty, 0); + if (jl_is_genericmemoryref_type(sty)) + sty = (jl_datatype_t*)jl_field_type_concrete(sty, 1); + return sty->layout->size; } - return true; + return -1; } -static intptr_t arraytype_maxsize(jl_value_t *ty) +static intptr_t genericmemoryype_maxsize(jl_value_t *ty) { - if (!jl_is_array_type(ty)) - return INTPTR_MAX; - size_t elsz; - if (arraytype_constelsize((jl_datatype_t*)ty, &elsz) || elsz == 0) + ssize_t elsz = genericmemoryype_constelsize(ty); + if (elsz <= 0) return INTPTR_MAX; return INTPTR_MAX / elsz; } -static Value *emit_arraylen(jl_codectx_t &ctx, const jl_cgval_t &tinfo); - -static Value *emit_arraysize(jl_codectx_t &ctx, const jl_cgval_t &tinfo, Value *dim) +static Value *emit_genericmemorylen(jl_codectx_t &ctx, Value *addr, jl_value_t *typ) { - size_t ndim; - MDNode *tbaa = ctx.tbaa().tbaa_arraysize; - if (arraytype_constdim(tinfo.typ, &ndim)) { - if (ndim == 0) - return ConstantInt::get(ctx.types().T_size, 1); - if (ndim == 1) { - if (auto d = dyn_cast(dim)) { - if (d->getZExtValue() == 1) { - return emit_arraylen(ctx, tinfo); - } - } - } - if (ndim > 1) { - if (tinfo.constant && isa(dim)) { - auto n = cast(dim)->getZExtValue() - 1; - return ConstantInt::get(ctx.types().T_size, jl_array_dim(tinfo.constant, n)); - } - tbaa = ctx.tbaa().tbaa_const; - } - } - ++EmittedArraysize; - Value *t = boxed(ctx, tinfo); - int o = offsetof(jl_array_t, nrows) / sizeof(void*) - 1; - auto load = emit_nthptr_recast(ctx, - t, - ctx.builder.CreateAdd(dim, ConstantInt::get(dim->getType(), o)), - tbaa, ctx.types().T_size); - setName(ctx.emission_context, load, "arraysize"); + addr = emit_bitcast(ctx, decay_derived(ctx, addr), ctx.types().T_jlgenericmemory->getPointerTo()), + addr = ctx.builder.CreateStructGEP(ctx.types().T_jlgenericmemory, addr, 0); + LoadInst *LI = ctx.builder.CreateAlignedLoad(ctx.types().T_jlgenericmemory->getElementType(0), addr, Align(sizeof(size_t))); + jl_aliasinfo_t aliasinfo_mem = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_memorylen); + aliasinfo_mem.decorateInst(LI); MDBuilder MDB(ctx.builder.getContext()); - auto rng = MDB.createRange(Constant::getNullValue(ctx.types().T_size), ConstantInt::get(ctx.types().T_size, arraytype_maxsize(tinfo.typ))); - load->setMetadata(LLVMContext::MD_range, rng); - return load; -} - -static Value *emit_arraysize(jl_codectx_t &ctx, const jl_cgval_t &tinfo, int dim) -{ - return emit_arraysize(ctx, tinfo, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), dim)); -} - -static Value *emit_vectormaxsize(jl_codectx_t &ctx, const jl_cgval_t &ary) -{ - return emit_arraysize(ctx, ary, 2); // maxsize aliases ncols in memory layout for vector -} - -static Value *emit_arraylen_prim(jl_codectx_t &ctx, const jl_cgval_t &tinfo) -{ - size_t ndim; - jl_value_t *ty = tinfo.typ; - MDNode *tbaa = ctx.tbaa().tbaa_arraylen; - if (arraytype_constdim(ty, &ndim)) { - if (ndim == 0) - return ConstantInt::get(ctx.types().T_size, 1); - if (ndim != 1) { - if (tinfo.constant) - return ConstantInt::get(ctx.types().T_size, jl_array_len(tinfo.constant)); - tbaa = ctx.tbaa().tbaa_const; - } - } - ++EmittedArraylen; - Value *t = boxed(ctx, tinfo); - Value *addr = ctx.builder.CreateStructGEP(ctx.types().T_jlarray, - emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), - 1); //index (not offset) of length field in ctx.types().T_pjlarray - if (tinfo.V) - setName(ctx.emission_context, addr, tinfo.V->getName() + ".length_ptr"); - else - setName(ctx.emission_context, addr, ".length_ptr"); - LoadInst *len = ctx.builder.CreateAlignedLoad(ctx.types().T_size, addr, ctx.types().alignof_ptr); - if (tinfo.V) - setName(ctx.emission_context, len, tinfo.V->getName() + ".length"); - else - setName(ctx.emission_context, len, ".length"); - len->setOrdering(AtomicOrdering::NotAtomic); - MDBuilder MDB(ctx.builder.getContext()); - auto rng = MDB.createRange(Constant::getNullValue(ctx.types().T_size), ConstantInt::get(ctx.types().T_size, arraytype_maxsize(tinfo.typ))); - len->setMetadata(LLVMContext::MD_range, rng); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); - return ai.decorateInst(len); -} - -static Value *emit_arraylen(jl_codectx_t &ctx, const jl_cgval_t &tinfo) -{ - return emit_arraylen_prim(ctx, tinfo); + auto rng = MDB.createRange(Constant::getNullValue(ctx.types().T_size), ConstantInt::get(ctx.types().T_size, genericmemoryype_maxsize(typ))); + LI->setMetadata(LLVMContext::MD_range, rng); + return LI; } -static Value *emit_arrayptr_internal(jl_codectx_t &ctx, const jl_cgval_t &tinfo, Value *t, unsigned AS, bool isboxed) +static Value *emit_genericmemoryptr(jl_codectx_t &ctx, Value *mem, bool isboxed, bool isunion, bool isghost, unsigned AS) { ++EmittedArrayptr; - auto asarray = emit_bitcast(ctx, t, ctx.types().T_pjlarray); - Value *addr = ctx.builder.CreateStructGEP(ctx.types().T_jlarray, - asarray, 0); - if (addr != asarray) { - if (tinfo.V) - setName(ctx.emission_context, addr, tinfo.V->getName() + ".data_ptr"); - else - setName(ctx.emission_context, addr, ".data_ptr"); - } - // Normally allocated array of 0 dimension always have a inline pointer. - // However, we can't rely on that here since arrays can also be constructed from C pointers. - PointerType *PT = cast(addr->getType()); - PointerType *PPT = cast(ctx.types().T_jlarray->getElementType(0)); - PointerType *LoadT = PPT; - - if (isboxed) { - LoadT = PointerType::get(ctx.types().T_prjlvalue, AS); - } - else if (AS != PPT->getAddressSpace()) { - LoadT = PointerType::getWithSamePointeeType(PPT, AS); - } - if (LoadT != PPT) { - const auto Ty = PointerType::get(LoadT, PT->getAddressSpace()); - addr = ctx.builder.CreateBitCast(addr, Ty); - } - - LoadInst *LI = ctx.builder.CreateAlignedLoad(LoadT, addr, Align(sizeof(char *))); - if (tinfo.V) - setName(ctx.emission_context, LI, tinfo.V->getName() + ".data"); - else - setName(ctx.emission_context, LI, ".data"); + PointerType *PT = cast(mem->getType()); + assert(PT == ctx.types().T_prjlvalue); + Value *addr = emit_bitcast(ctx, mem, ctx.types().T_jlgenericmemory->getPointerTo(PT->getAddressSpace())); + addr = ctx.builder.CreateStructGEP(ctx.types().T_jlgenericmemory, addr, 1); + setName(ctx.emission_context, addr, ".data_ptr"); + PointerType *PPT = cast(ctx.types().T_jlgenericmemory->getElementType(1)); + LoadInst *LI = ctx.builder.CreateAlignedLoad(PPT, addr, Align(sizeof(char*))); LI->setOrdering(AtomicOrdering::NotAtomic); LI->setMetadata(LLVMContext::MD_nonnull, MDNode::get(ctx.builder.getContext(), None)); - jl_aliasinfo_t aliasinfo = jl_aliasinfo_t::fromTBAA(ctx, arraytype_constshape(tinfo.typ) ? ctx.tbaa().tbaa_const : ctx.tbaa().tbaa_arrayptr); + jl_aliasinfo_t aliasinfo = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); aliasinfo.decorateInst(LI); - return LI; -} - -static Value *emit_arrayptr(jl_codectx_t &ctx, const jl_cgval_t &tinfo, bool isboxed = false) -{ - Value *t = boxed(ctx, tinfo); - return emit_arrayptr_internal(ctx, tinfo, decay_derived(ctx, t), AddressSpace::Loaded, isboxed); -} - -static Value *emit_unsafe_arrayptr(jl_codectx_t &ctx, const jl_cgval_t &tinfo, bool isboxed = false) -{ - Value *t = boxed(ctx, tinfo); - t = emit_pointer_from_objref(ctx, decay_derived(ctx, t)); - return emit_arrayptr_internal(ctx, tinfo, t, 0, isboxed); -} - -static Value *emit_arrayptr(jl_codectx_t &ctx, const jl_cgval_t &tinfo, jl_value_t *ex, bool isboxed = false) -{ - return emit_arrayptr(ctx, tinfo, isboxed); -} - -static Value *emit_arraysize(jl_codectx_t &ctx, const jl_cgval_t &tinfo, jl_value_t *ex, int dim) -{ - return emit_arraysize(ctx, tinfo, dim); -} - -static Value *emit_arrayflags(jl_codectx_t &ctx, const jl_cgval_t &tinfo) -{ - ++EmittedArrayflags; - Value *t = boxed(ctx, tinfo); - int arrayflag_field = 2; - Value *addr = ctx.builder.CreateStructGEP( - ctx.types().T_jlarray, - emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), - arrayflag_field); - if (tinfo.V) - setName(ctx.emission_context, addr, tinfo.V->getName() + ".flags_ptr"); - else - setName(ctx.emission_context, addr, ".flags_ptr"); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayflags); - auto flags = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), addr, Align(sizeof(int16_t)))); - if (tinfo.V) - setName(ctx.emission_context, flags, tinfo.V->getName() + ".flags"); - else - setName(ctx.emission_context, flags, ".flags"); - return flags; -} - -static Value *emit_arrayndims(jl_codectx_t &ctx, const jl_cgval_t &ary) -{ - ++EmittedArrayNDims; - Value *flags = emit_arrayflags(ctx, ary); - auto name = flags->getName(); - cast(flags)->setMetadata(LLVMContext::MD_invariant_load, MDNode::get(ctx.builder.getContext(), None)); - flags = ctx.builder.CreateLShr(flags, 2); - flags = ctx.builder.CreateAnd(flags, 0x1FF); // (1<<9) - 1 - setName(ctx.emission_context, flags, name + ".ndims"); - return flags; -} - -static Value *emit_arrayelsize(jl_codectx_t &ctx, const jl_cgval_t &tinfo) -{ - ++EmittedArrayElsize; - Value *t = boxed(ctx, tinfo); - int elsize_field = 3; - Value *addr = ctx.builder.CreateStructGEP(ctx.types().T_jlarray, - emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), - elsize_field); - if (tinfo.V) - setName(ctx.emission_context, addr, tinfo.V->getName() + ".elsize_ptr"); - else - setName(ctx.emission_context, addr, ".elsize_ptr"); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - auto elsize = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), addr, Align(sizeof(int16_t)))); - if (tinfo.V) - setName(ctx.emission_context, elsize, tinfo.V->getName() + ".elsize"); - else - setName(ctx.emission_context, elsize, ".elsize"); - return elsize; -} - -static Value *emit_arrayoffset(jl_codectx_t &ctx, const jl_cgval_t &tinfo, int nd) -{ - ++EmittedArrayOffset; - if (nd != -1 && nd != 1) // only Vector can have an offset - return ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0); - Value *t = boxed(ctx, tinfo); - int offset_field = 4; - - Value *addr = ctx.builder.CreateStructGEP( - ctx.types().T_jlarray, - emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), - offset_field); - if (tinfo.V) - setName(ctx.emission_context, addr, tinfo.V->getName() + ".offset_ptr"); - else - setName(ctx.emission_context, addr, ".offset_ptr"); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayoffset); - auto offset = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), addr, Align(sizeof(int32_t)))); - if (tinfo.V) - setName(ctx.emission_context, offset, tinfo.V->getName() + ".offset"); - else - setName(ctx.emission_context, offset, ".offset"); - return offset; + Value *ptr = LI; + if (AS) { + assert(AS == AddressSpace::Loaded); + ptr = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, ptr }); + } + if (!isboxed) + ptr = ctx.builder.CreateBitCast(ptr, PointerType::get(getInt8Ty(ctx.builder.getContext()), AS)); + return ptr; } -// Returns the size of the array represented by `tinfo` for the given dimension `dim` if -// `dim` is a valid dimension, otherwise returns constant one. -static Value *emit_arraysize_for_unsafe_dim(jl_codectx_t &ctx, - const jl_cgval_t &tinfo, jl_value_t *ex, size_t dim, size_t nd) +static Value *emit_genericmemoryptr(jl_codectx_t &ctx, const jl_cgval_t &mem, bool isboxed, bool isunion, bool isghost, unsigned AS) { - return dim > nd ? ConstantInt::get(ctx.types().T_size, 1) : emit_arraysize(ctx, tinfo, ex, dim); + return emit_genericmemoryptr(ctx, boxed(ctx, mem), isboxed, isunion, isghost, AS); } -// `nd == -1` means the dimension is unknown. -static Value *emit_array_nd_index( - jl_codectx_t &ctx, const jl_cgval_t &ainfo, jl_value_t *ex, ssize_t nd, - const jl_cgval_t *argv, size_t nidxs, jl_value_t *inbounds) +static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t) { - ++EmittedArrayNdIndex; - Value *a = boxed(ctx, ainfo); - Value *i = Constant::getNullValue(ctx.types().T_size); - Value *stride = ConstantInt::get(ctx.types().T_size, 1); -#if CHECK_BOUNDS==1 - bool bc = bounds_check_enabled(ctx, inbounds); - BasicBlock *failBB = NULL, *endBB = NULL; - if (bc) { - failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); - endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); - } -#endif - SmallVector idxs(nidxs); - for (size_t k = 0; k < nidxs; k++) { - idxs[k] = emit_unbox(ctx, ctx.types().T_size, argv[k], (jl_value_t*)jl_long_type); // type asserted by caller - } - Value *ii = NULL; - for (size_t k = 0; k < nidxs; k++) { - ii = ctx.builder.CreateSub(idxs[k], ConstantInt::get(ctx.types().T_size, 1)); - i = ctx.builder.CreateAdd(i, ctx.builder.CreateMul(ii, stride)); - if (k < nidxs - 1) { - assert(nd >= 0); - Value *d = emit_arraysize_for_unsafe_dim(ctx, ainfo, ex, k + 1, nd); -#if CHECK_BOUNDS==1 - if (bc) { - BasicBlock *okBB = BasicBlock::Create(ctx.builder.getContext(), "ib"); - // if !(i < d) goto error - auto bc = ctx.builder.CreateICmpULT(ii, d); - setName(ctx.emission_context, bc, "inbounds"); - ctx.builder.CreateCondBr(bc, okBB, failBB); - ctx.f->getBasicBlockList().push_back(okBB); - ctx.builder.SetInsertPoint(okBB); - } -#endif - stride = ctx.builder.CreateMul(stride, d); - setName(ctx.emission_context, stride, "stride"); - } - } -#if CHECK_BOUNDS==1 - if (bc) { - // We have already emitted a bounds check for each index except for - // the last one which we therefore have to do here. - if (nidxs == 1) { - // Linear indexing: Check against the entire linear span of the array - Value *alen = emit_arraylen(ctx, ainfo); - auto bc = ctx.builder.CreateICmpULT(i, alen); - setName(ctx.emission_context, bc, "inbounds"); - ctx.builder.CreateCondBr(bc, endBB, failBB); - } else if (nidxs >= (size_t)nd){ - // No dimensions were omitted; just check the last remaining index - assert(nd >= 0); - Value *last_index = ii; - Value *last_dimension = emit_arraysize_for_unsafe_dim(ctx, ainfo, ex, nidxs, nd); - auto bc = ctx.builder.CreateICmpULT(last_index, last_dimension); - setName(ctx.emission_context, bc, "inbounds"); - ctx.builder.CreateCondBr(bc, endBB, failBB); - } else { - // There were fewer indices than dimensions; check the last remaining index - BasicBlock *checktrailingdimsBB = BasicBlock::Create(ctx.builder.getContext(), "dimsib"); - assert(nd >= 0); - Value *last_index = ii; - Value *last_dimension = emit_arraysize_for_unsafe_dim(ctx, ainfo, ex, nidxs, nd); - auto bc = ctx.builder.CreateICmpULT(last_index, last_dimension); - setName(ctx.emission_context, bc, "inbounds"); - ctx.builder.CreateCondBr(bc, checktrailingdimsBB, failBB); - ctx.f->getBasicBlockList().push_back(checktrailingdimsBB); - ctx.builder.SetInsertPoint(checktrailingdimsBB); - // And then also make sure that all dimensions that weren't explicitly - // indexed into have size 1 - for (size_t k = nidxs+1; k < (size_t)nd; k++) { - BasicBlock *dimsokBB = BasicBlock::Create(ctx.builder.getContext(), "dimsok"); - Value *dim = emit_arraysize_for_unsafe_dim(ctx, ainfo, ex, k, nd); - auto bc = ctx.builder.CreateICmpEQ(dim, ConstantInt::get(ctx.types().T_size, 1)); - setName(ctx.emission_context, bc, "inbounds"); - ctx.builder.CreateCondBr(bc, dimsokBB, failBB); - ctx.f->getBasicBlockList().push_back(dimsokBB); - ctx.builder.SetInsertPoint(dimsokBB); - } - Value *dim = emit_arraysize_for_unsafe_dim(ctx, ainfo, ex, nd, nd); - auto bc2 = ctx.builder.CreateICmpEQ(dim, ConstantInt::get(ctx.types().T_size, 1)); - setName(ctx.emission_context, bc2, "inbounds"); - ctx.builder.CreateCondBr(bc2, endBB, failBB); - } - - ctx.f->getBasicBlockList().push_back(failBB); - ctx.builder.SetInsertPoint(failBB); - // CreateAlloca is OK here since we are on an error branch - Value *tmp = ctx.builder.CreateAlloca(ctx.types().T_size, ConstantInt::get(ctx.types().T_size, nidxs)); - setName(ctx.emission_context, tmp, "errorbox"); - for (size_t k = 0; k < nidxs; k++) { - ctx.builder.CreateAlignedStore(idxs[k], ctx.builder.CreateInBoundsGEP(ctx.types().T_size, tmp, ConstantInt::get(ctx.types().T_size, k)), ctx.types().alignof_ptr); - } - ctx.builder.CreateCall(prepare_call(jlboundserrorv_func), - { mark_callee_rooted(ctx, a), tmp, ConstantInt::get(ctx.types().T_size, nidxs) }); - ctx.builder.CreateUnreachable(); - - ctx.f->getBasicBlockList().push_back(endBB); - ctx.builder.SetInsertPoint(endBB); - } -#endif - - return i; + Value *m = emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_jlgenericmemory->getPointerTo(0)); + Value *addr = ctx.builder.CreateStructGEP(ctx.types().T_jlgenericmemory, m, 1); + Type *T_data = ctx.types().T_jlgenericmemory->getElementType(1); + LoadInst *LI = ctx.builder.CreateAlignedLoad(T_data, addr, Align(sizeof(char*))); + LI->setOrdering(AtomicOrdering::NotAtomic); + LI->setMetadata(LLVMContext::MD_nonnull, MDNode::get(ctx.builder.getContext(), None)); + jl_aliasinfo_t aliasinfo_mem = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_memoryown); + aliasinfo_mem.decorateInst(LI); + addr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_jlgenericmemory, m, 1); + Value *foreign = ctx.builder.CreateICmpNE(addr, emit_bitcast(ctx, decay_derived(ctx, LI), addr->getType())); + return emit_guarded_test(ctx, foreign, t, [&] { + LoadInst *owner = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, emit_bitcast(ctx, addr, ctx.types().T_pprjlvalue), Align(sizeof(void*))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + ai.decorateInst(owner); + return ctx.builder.CreateSelect(ctx.builder.CreateIsNull(owner), t, owner); + }); } // --- boxing --- @@ -3165,7 +2914,7 @@ static jl_value_t *static_constant_instance(const llvm::DataLayout &DL, Constant nargs = CDS->getNumElements(); else return NULL; - assert(nargs > 0 && jst->instance == NULL); + assert(nargs > 0 && !jl_is_datatype_singleton(jst)); if (nargs != jl_datatype_nfields(jst)) return NULL; @@ -3268,15 +3017,14 @@ static Value *_boxed_special(jl_codectx_t &ctx, const jl_cgval_t &vinfo, Type *t else if (jb == jl_char_type) box = call_with_attrs(ctx, box_char_func, as_value(ctx, t, vinfo)); else if (jb == jl_ssavalue_type) { - unsigned zero = 0; Value *v = as_value(ctx, t, vinfo); assert(v->getType() == ctx.emission_context.llvmtypes[jl_ssavalue_type]); - v = ctx.builder.CreateExtractValue(v, makeArrayRef(&zero, 1)); + v = ctx.builder.CreateExtractValue(v, 0); box = call_with_attrs(ctx, box_ssavalue_func, v); } else if (!jb->name->abstract && jl_datatype_nbits(jb) == 0) { // singleton - assert(jb->instance != NULL); + assert(jl_is_datatype_singleton(jb)); return track_pjlvalue(ctx, literal_pointer_val(ctx, jb->instance)); } if (box) { @@ -3765,6 +3513,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, ++EmittedSetfield; assert(strct.ispointer()); size_t byte_offset = jl_field_offset(sty, idx0); + auto tbaa = best_field_tbaa(ctx, strct, sty, idx0, byte_offset); Value *addr = data_pointer(ctx, strct); if (byte_offset > 0) { addr = ctx.builder.CreateInBoundsGEP( @@ -3797,7 +3546,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, } jl_cgval_t oldval = rhs; if (!issetfield) - oldval = emit_unionload(ctx, addr, ptindex, jfty, fsz, al, strct.tbaa, true, union_max, ctx.tbaa().tbaa_unionselbyte); + oldval = emit_unionload(ctx, addr, ptindex, jfty, fsz, al, tbaa, true, union_max, ctx.tbaa().tbaa_unionselbyte); Value *Success = NULL; BasicBlock *DoneBB = NULL; if (isreplacefield || ismodifyfield) { @@ -3820,7 +3569,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, if (needlock) emit_lockstate_value(ctx, strct, true); cmp = oldval; - oldval = emit_unionload(ctx, addr, ptindex, jfty, fsz, al, strct.tbaa, true, union_max, ctx.tbaa().tbaa_unionselbyte); + oldval = emit_unionload(ctx, addr, ptindex, jfty, fsz, al, tbaa, true, union_max, ctx.tbaa().tbaa_unionselbyte); } BasicBlock *XchgBB = BasicBlock::Create(ctx.builder.getContext(), "xchg", ctx.f); DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg", ctx.f); @@ -3834,7 +3583,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); // copy data if (!rhs.isghost) { - emit_unionmove(ctx, addr, strct.tbaa, rhs, nullptr); + emit_unionmove(ctx, addr, tbaa, rhs, nullptr); } if (isreplacefield || ismodifyfield) { ctx.builder.CreateBr(DoneBB); @@ -3866,7 +3615,7 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, bool isboxed = jl_field_isptr(sty, idx0); size_t nfields = jl_datatype_nfields(sty); bool maybe_null = idx0 >= nfields - (unsigned)sty->name->n_uninitialized; - return typed_store(ctx, addr, NULL, rhs, cmp, jfty, strct.tbaa, nullptr, + return typed_store(ctx, addr, NULL, rhs, cmp, jfty, tbaa, nullptr, wb ? boxed(ctx, strct) : nullptr, isboxed, Order, FailOrder, align, needlock, issetfield, isreplacefield, isswapfield, ismodifyfield, maybe_null, modifyop, fname); @@ -4153,6 +3902,222 @@ static int compare_cgparams(const jl_cgparams_t *a, const jl_cgparams_t *b) } #endif +static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, Value *mem, Value *data, bool isboxed, bool isunion, bool isghost, jl_value_t *typ) +{ + //jl_cgval_t argv[] = { + // mark_julia_type(ctx, mem, true, jl_any_type), + // mark_julia_type(ctx, data, false, jl_voidpointer_type) + //}; + //return emit_new_struct(ctx, typ, 3, argv); + Value *ref = Constant::getNullValue(get_memoryref_type(ctx.builder.getContext(), ctx.types().T_size, isboxed, isunion, isghost, 0)); + ref = ctx.builder.CreateInsertValue(ref, data, 0); + ref = ctx.builder.CreateInsertValue(ref, mem, 1); + return mark_julia_type(ctx, ref, false, typ); +} + +static jl_cgval_t _emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &mem, bool isboxed, bool isunion, bool isghost, jl_value_t *typ) +{ + Value *data = (!isboxed && isunion) || isghost ? ConstantInt::get(ctx.types().T_size, 0) : emit_genericmemoryptr(ctx, mem, isboxed, isunion, isghost, 0); + return _emit_memoryref(ctx, boxed(ctx, mem), data, isboxed, isunion, isghost, typ); +} + +static Value *emit_memoryref_FCA(jl_codectx_t &ctx, const jl_cgval_t &ref, bool isboxed, bool isunion, bool isghost) +{ + if (ref.ispointer()) { + LLVMContext &C = ctx.builder.getContext(); + Type *type = get_memoryref_type(C, ctx.types().T_size, isboxed, isunion, isghost, 0); + LoadInst *load = ctx.builder.CreateLoad(type, emit_bitcast(ctx, data_pointer(ctx, ref), PointerType::get(type, 0))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ref.tbaa); + ai.decorateInst(load); + return load; + } + else { + return ref.V; + } +} + +static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cgval_t idx, jl_value_t *inbounds, bool isboxed, bool isunion, bool isghost) +{ + ++EmittedArrayNdIndex; + emit_typecheck(ctx, idx, (jl_value_t*)jl_long_type, "memoryref"); + idx = update_julia_type(ctx, idx, (jl_value_t*)jl_long_type); + if (idx.typ == jl_bottom_type) + return jl_cgval_t(); + Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, isghost); + Value *data = CreateSimplifiedExtractValue(ctx, V, 0); + Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); + Value *i = emit_unbox(ctx, ctx.types().T_size, idx, (jl_value_t*)jl_long_type); + Value *offset = ctx.builder.CreateSub(i, ConstantInt::get(ctx.types().T_size, 1)); + Value *elsz = emit_genericmemoryelsize(ctx, mem, ref.typ); + bool bc = bounds_check_enabled(ctx, inbounds); +#if 1 + Value *ovflw = nullptr; +#endif + Value *newdata; + if ((!isboxed && isunion) || isghost) { + newdata = ctx.builder.CreateAdd(data, offset); + if (bc) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *inbound = ctx.builder.CreateICmpULT(newdata, mlen); + ctx.builder.CreateCondBr(inbound, endBB, failBB); + ctx.f->getBasicBlockList().push_back(failBB); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), + { mark_callee_rooted(ctx, boxed(ctx, ref)), i }); + ctx.builder.CreateUnreachable(); + ctx.f->getBasicBlockList().push_back(endBB); + ctx.builder.SetInsertPoint(endBB); + } + } + else { + Value *boffset; +#if 0 + if (bc) { + auto *MulF = Intrinsic::getDeclaration(jl_Module, Intrinsic::smul_with_overflow, offset->getType()); + CallInst *Mul = ctx.builder.CreateCall(MulF, {offset, elsz}); + boffset = ctx.builder.CreateExtractValue(Mul, 0); + ovflw = ctx.builder.CreateExtractValue(Mul, 1); + } + else +#else + if (bc) { + // n.b. we could boundscheck that -len<=offset<=len instead of using smul.ovflw, + // since we know that len*elsz does not overflow, + // and we can further rearrange that as ovflw = !( offset+len < len+len ) as unsigned math + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + ovflw = ctx.builder.CreateICmpUGE(ctx.builder.CreateAdd(offset, mlen), ctx.builder.CreateNUWAdd(mlen, mlen)); + } +#endif + boffset = ctx.builder.CreateMul(offset, elsz); + newdata = emit_bitcast(ctx, data, getInt8PtrTy(ctx.builder.getContext())); + newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), newdata, boffset); + newdata = emit_bitcast(ctx, newdata, data->getType()); + if (bc) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "idxend"); + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *mptr = emit_genericmemoryptr(ctx, mem, isboxed, isunion, isghost, 0); + mptr = emit_bitcast(ctx, mptr, newdata->getType()); +#if 0 + Value *mend = emit_bitcast(ctx, mptr, getInt8PtrTy(ctx.builder.getContext())); + Value *blen = ctx.builder.CreateMul(mlen, elsz, "", true, true); + mend = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), mend, blen); + mend = emit_bitcast(ctx, mend, newdata->getType()); + Value *inbound = ctx.builder.CreateAnd( + ctx.builder.CreateICmpULE(mptr, newdata), + ctx.builder.CreateICmpULT(newdata, mend)); + inbound = ctx.builder.CreateAnd( + ctx.builder.CreateNot(ovflw), + inbound); +#elif 1 + Value *bidx0 = ctx.builder.CreateSub( + ctx.builder.CreatePtrToInt(newdata, ctx.types().T_size), + ctx.builder.CreatePtrToInt(mptr, ctx.types().T_size)); + Value *blen = ctx.builder.CreateMul(mlen, elsz, "", true, true); + Value *inbound = ctx.builder.CreateICmpULT(bidx0, blen); + inbound = ctx.builder.CreateAnd(ctx.builder.CreateNot(ovflw), inbound); +#else + Value *idx0; // (newdata - mptr) / elsz + idx0 = ctx.builder.CreateSub( + ctx.builder.CreatePtrToInt(newdata, ctx.types().T_size), + ctx.builder.CreatePtrToInt(mptr, ctx.types().T_size)); + idx0 = ctx.builder.CreateExactUDiv(idx0, elsz); + Value *inbound = ctx.builder.CreateICmpULT(idx0, mlen); +#endif + ctx.builder.CreateCondBr(inbound, endBB, failBB); + ctx.f->getBasicBlockList().push_back(failBB); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), + { mark_callee_rooted(ctx, boxed(ctx, ref)), i }); + ctx.builder.CreateUnreachable(); + ctx.f->getBasicBlockList().push_back(endBB); + ctx.builder.SetInsertPoint(endBB); + } + } + return _emit_memoryref(ctx, mem, newdata, isboxed, isunion, isghost, ref.typ); +} + +static Value *emit_memoryref_mem(jl_codectx_t &ctx, const jl_cgval_t &ref, bool isboxed, bool isunion, bool isghost) +{ + Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, isghost); + return CreateSimplifiedExtractValue(ctx, V, 1); +} + +static Value *emit_memoryref_ptr(jl_codectx_t &ctx, const jl_cgval_t &ref, bool isboxed, bool isunion, bool isghost) +{ + assert((isboxed || !isunion) && !isghost); + Value *newref = emit_memoryref_FCA(ctx, ref, isboxed, isunion, isghost); + Value *data = CreateSimplifiedExtractValue(ctx, newref, 0); + if (isboxed || !(isunion && isghost)) { + unsigned AS = AddressSpace::Loaded; + Value *mem = CreateSimplifiedExtractValue(ctx, newref, 1); + // rebuild GEP on data, so that we manually hoist this gc_loaded_func call over it, back to the original load + // we should add this to llvm-julia-licm too, so we can attempt hoisting over PhiNodes too (which aren't defined yet here) + { + IRBuilder<>::InsertPointGuard resetIP(ctx.builder); + std::vector GEPlist; + while (GetElementPtrInst *GEP = dyn_cast(data->stripPointerCastsSameRepresentation())) { // ignoring bitcast will not be required with opaque pointers + GEPlist.push_back(GEP); + data = GEP->getPointerOperand(); + } + // Under the common assumption that data and mem will come from the same basic block and both will dominate the first GEP, + // manually hoist these right away to improve the IR structure. But tolerate the case where we could not verify that. + bool moveip = false; + //Instruction *defmem = dyn_cast(mem); + //if (Instruction *def = dyn_cast(data)) { + // if (defmem == nullptr || def->getParent() == defmem->getParent()) { + // if (defmem != nullptr && !defmem->comesBefore(def)) + // def = defmem; + // ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator())); + // ctx.builder.SetCurrentDebugLocation(def->getDebugLoc()); + // moveip = true; + // } + // else if (LoadInst *defbase = dyn_cast(def)) { + // def = defbase; + // if (defmem != nullptr && !defmem->comesBefore(def)) + // def = defmem; + // ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator())); + // ctx.builder.SetCurrentDebugLocation(def->getDebugLoc()); + // moveip = true; + // } + //} + //// if we couldn't find the def Instruction, we might still be able to put it right before the GEP chain + //if (!moveip && !GEPlist.empty()) { + // Instruction *defgep = GEPlist[GEPlist.size() - 1]; + // if (defmem == nullptr || defmem->getParent() == defgep->getParent()) { + // if (defmem == nullptr || defmem->comesBefore(defgep)) { + // ctx.builder.SetInsertPoint(GEPlist[GEPlist.size() - 1]); + // moveip = true; + // } + // } + //} + data = ctx.builder.CreateBitCast(data, ctx.types().T_pprjlvalue); + data = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, data }); + if (!GEPlist.empty()) { + for (auto &GEP : make_range(GEPlist.rbegin(), GEPlist.rend())) { + if (moveip) + ctx.builder.SetInsertPoint(GEP); + data = ctx.builder.CreateBitCast(data, PointerType::get(GEP->getSourceElementType(), AS)); + Instruction *GEP2 = GEP->clone(); + GEP2->mutateType(PointerType::get(GEP->getResultElementType(), AS)); + GEP2->setOperand(GetElementPtrInst::getPointerOperandIndex(), data); + ctx.builder.Insert(GEP2); + data = GEP2; + } + } + } // resetIP + if (isboxed) + data = ctx.builder.CreateBitCast(data, PointerType::get(ctx.types().T_prjlvalue, AS)); + } + return data; +} + + + // Reset us back to codegen debug type #undef DEBUG_TYPE #define DEBUG_TYPE "julia_irgen_codegen" diff --git a/src/clangsa/GCChecker.cpp b/src/clangsa/GCChecker.cpp index 1dcb7f5a754d7..8aa0ef009f4eb 100644 --- a/src/clangsa/GCChecker.cpp +++ b/src/clangsa/GCChecker.cpp @@ -806,6 +806,8 @@ bool GCChecker::isGCTrackedType(QualType QT) { Name.endswith_lower("jl_expr_t") || Name.endswith_lower("jl_code_info_t") || Name.endswith_lower("jl_array_t") || + Name.endswith_lower("jl_genericmemory_t") || + //Name.endswith_lower("jl_genericmemoryref_t") || Name.endswith_lower("jl_method_t") || Name.endswith_lower("jl_method_instance_t") || Name.endswith_lower("jl_tupletype_t") || diff --git a/src/codegen.cpp b/src/codegen.cpp index 342d751c95a87..df2b9fd5e604a 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -40,6 +40,7 @@ #include #include #include +#include // support #include @@ -287,6 +288,7 @@ struct jl_typecache_t { Type *T_prjlvalue; Type *T_ppjlvalue; Type *T_pprjlvalue; + StructType *T_jlgenericmemory; StructType *T_jlarray; Type *T_pjlarray; FunctionType *T_jlfunc; @@ -302,9 +304,10 @@ struct jl_typecache_t { jl_typecache_t() : T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr), - T_ppjlvalue(nullptr), T_pprjlvalue(nullptr), T_jlarray(nullptr), - T_pjlarray(nullptr), T_jlfunc(nullptr), T_jlfuncparams(nullptr), - T_sigatomic(nullptr), T_ppint8(nullptr), initialized(false) {} + T_ppjlvalue(nullptr), T_pprjlvalue(nullptr), + T_jlgenericmemory(nullptr), T_jlarray(nullptr), T_pjlarray(nullptr), + T_jlfunc(nullptr), T_jlfuncparams(nullptr), T_sigatomic(nullptr), T_ppint8(nullptr), + initialized(false) {} void initialize(LLVMContext &context, const DataLayout &DL) { if (initialized) { @@ -328,14 +331,11 @@ struct jl_typecache_t { T_jlfuncparams = JuliaType::get_jlfuncparams_ty(context); assert(T_jlfuncparams != NULL); - Type *vaelts[] = {PointerType::get(getInt8Ty(context), AddressSpace::Loaded) - , T_size - , getInt16Ty(context) - , getInt16Ty(context) - , getInt32Ty(context) + T_jlgenericmemory = StructType::get(context, { T_size, T_pprjlvalue /* [, real-owner] */ }); + Type *vaelts[] = { PointerType::get(getInt8Ty(context), AddressSpace::Loaded), + PointerType::get(T_jlgenericmemory, AddressSpace::Tracked), + // dimsize[ndims] }; - static_assert(sizeof(jl_array_flags_t) == sizeof(int16_t), - "Size of jl_array_flags_t is not the same as int16_t"); T_jlarray = StructType::get(context, makeArrayRef(vaelts)); T_pjlarray = PointerType::get(T_jlarray, 0); } @@ -351,19 +351,19 @@ struct jl_tbaacache_t { MDNode *tbaa_unionselbyte; // a selector byte in isbits Union struct fields MDNode *tbaa_data; // Any user data that `pointerset/ref` are allowed to alias MDNode *tbaa_binding; // jl_binding_t::value - MDNode *tbaa_value; // jl_value_t, that is not jl_array_t + MDNode *tbaa_value; // jl_value_t, that is not jl_array_t or jl_genericmemory_t MDNode *tbaa_mutab; // mutable type MDNode *tbaa_datatype; // datatype MDNode *tbaa_immut; // immutable type MDNode *tbaa_ptrarraybuf; // Data in an array of boxed values MDNode *tbaa_arraybuf; // Data in an array of POD - MDNode *tbaa_array; // jl_array_t - MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t + MDNode *tbaa_array; // jl_array_t or jl_genericmemory_t + MDNode *tbaa_arrayptr; // The pointer inside a jl_array_t (to memoryref) MDNode *tbaa_arraysize; // A size in a jl_array_t - MDNode *tbaa_arraylen; // The len in a jl_array_t - MDNode *tbaa_arrayflags; // The flags in a jl_array_t - MDNode *tbaa_arrayoffset; // The offset in a jl_array_t - MDNode *tbaa_arrayselbyte; // a selector byte in a isbits Union jl_array_t + MDNode *tbaa_arrayselbyte; // a selector byte in a isbits Union jl_genericmemory_t + MDNode *tbaa_memoryptr; // The pointer inside a jl_genericmemory_t + MDNode *tbaa_memorylen; // The length in a jl_genericmemory_t + MDNode *tbaa_memoryown; // The owner in a foreign jl_genericmemory_t MDNode *tbaa_const; // Memory that is immutable by the time LLVM can see it bool initialized; @@ -372,8 +372,8 @@ struct jl_tbaacache_t { tbaa_value(nullptr), tbaa_mutab(nullptr), tbaa_datatype(nullptr), tbaa_immut(nullptr), tbaa_ptrarraybuf(nullptr), tbaa_arraybuf(nullptr), tbaa_array(nullptr), tbaa_arrayptr(nullptr), tbaa_arraysize(nullptr), - tbaa_arraylen(nullptr), tbaa_arrayflags(nullptr), tbaa_arrayoffset(nullptr), - tbaa_arrayselbyte(nullptr), tbaa_const(nullptr), initialized(false) {} + tbaa_arrayselbyte(nullptr), tbaa_memoryptr(nullptr), tbaa_memorylen(nullptr), tbaa_memoryown(nullptr), + tbaa_const(nullptr), initialized(false) {} auto tbaa_make_child(MDBuilder &mbuilder, const char *name, MDNode *parent = nullptr, bool isConstant = false) { MDNode *scalar = mbuilder.createTBAAScalarTypeNode(name, parent ? parent : tbaa_root); @@ -411,11 +411,11 @@ struct jl_tbaacache_t { std::tie(tbaa_array, tbaa_array_scalar) = tbaa_make_child(mbuilder, "jtbaa_array"); tbaa_arrayptr = tbaa_make_child(mbuilder, "jtbaa_arrayptr", tbaa_array_scalar).first; tbaa_arraysize = tbaa_make_child(mbuilder, "jtbaa_arraysize", tbaa_array_scalar).first; - tbaa_arraylen = tbaa_make_child(mbuilder, "jtbaa_arraylen", tbaa_array_scalar).first; - tbaa_arrayflags = tbaa_make_child(mbuilder, "jtbaa_arrayflags", tbaa_array_scalar).first; - tbaa_arrayoffset = tbaa_make_child(mbuilder, "jtbaa_arrayoffset", tbaa_array_scalar).first; - tbaa_const = tbaa_make_child(mbuilder, "jtbaa_const", nullptr, true).first; tbaa_arrayselbyte = tbaa_make_child(mbuilder, "jtbaa_arrayselbyte", tbaa_array_scalar).first; + tbaa_memoryptr = tbaa_make_child(mbuilder, "jtbaa_memoryptr", tbaa_array_scalar, true).first; + tbaa_memorylen = tbaa_make_child(mbuilder, "jtbaa_memorylen", tbaa_array_scalar, true).first; + tbaa_memoryown = tbaa_make_child(mbuilder, "jtbaa_memoryown", tbaa_array_scalar, true).first; + tbaa_const = tbaa_make_child(mbuilder, "jtbaa_const", nullptr, true).first; } }; @@ -428,7 +428,7 @@ struct jl_noaliascache_t { MDNode *gcframe; // GC frame MDNode *stack; // Stack slot MDNode *data; // Any user data that `pointerset/ref` are allowed to alias - MDNode *type_metadata; // Non-user-accessible type metadata incl. size, union selectors, etc. + MDNode *type_metadata; // Non-user-accessible type metadata incl. union selectors, etc. MDNode *constant; // Memory that is immutable by the time LLVM can see it jl_regions_t(): gcframe(nullptr), stack(nullptr), data(nullptr), type_metadata(nullptr), constant(nullptr) {} @@ -602,11 +602,13 @@ static inline void add_named_global(StringRef name, T *addr) add_named_global(name, (void*)(uintptr_t)addr); } -AttributeSet Attributes(LLVMContext &C, std::initializer_list attrkinds) +AttributeSet Attributes(LLVMContext &C, std::initializer_list attrkinds, std::initializer_list extra={}) { - SmallVector attrs(attrkinds.size()); + SmallVector attrs(attrkinds.size() + extra.size()); for (size_t i = 0; i < attrkinds.size(); i++) attrs[i] = Attribute::get(C, attrkinds.begin()[i]); + for (size_t i = 0; i < extra.size(); i++) + attrs[attrkinds.size() + i] = extra.begin()[i]; return AttributeSet::get(C, makeArrayRef(attrs)); } @@ -1193,18 +1195,6 @@ static const auto sync_gc_total_bytes_func = new JuliaFunction<>{ {getInt64Ty(C)}, false); }, nullptr, }; -static const auto jlarray_data_owner_func = new JuliaFunction<>{ - XSTR(jl_array_data_owner), - [](LLVMContext &C) { - auto T_prjlvalue = JuliaType::get_prjlvalue_ty(C); - return FunctionType::get(T_prjlvalue, - {T_prjlvalue}, false); - }, - [](LLVMContext &C) { return AttributeList::get(C, - Attributes(C, {Attribute::ReadOnly, Attribute::NoUnwind}), - Attributes(C, {Attribute::NonNull}), - None); }, -}; #define BOX_FUNC(ct,at,attrs) \ static const auto box_##ct##_func = new JuliaFunction<>{ \ XSTR(jl_box_##ct), \ @@ -1254,7 +1244,7 @@ static const auto except_enter_func = new JuliaFunction<>{ "julia.except_enter", [](LLVMContext &C) { return FunctionType::get(getInt32Ty(C), false); }, [](LLVMContext &C) { return AttributeList::get(C, - AttributeSet::get(C, makeArrayRef({Attribute::get(C, Attribute::ReturnsTwice)})), + Attributes(C, {Attribute::ReturnsTwice}), AttributeSet(), None); }, }; @@ -1263,10 +1253,27 @@ static const auto pointer_from_objref_func = new JuliaFunction<>{ [](LLVMContext &C) { return FunctionType::get(JuliaType::get_pjlvalue_ty(C), {PointerType::get(JuliaType::get_jlvalue_ty(C), AddressSpace::Derived)}, false); }, [](LLVMContext &C) { return AttributeList::get(C, - AttributeSet::get(C, makeArrayRef({Attribute::get(C, Attribute::ReadNone), Attribute::get(C, Attribute::NoUnwind)})), + Attributes(C, {Attribute::ReadNone, Attribute::NoUnwind}), Attributes(C, {Attribute::NonNull}), None); }, }; +static const auto gc_loaded_func = new JuliaFunction<>{ + "julia.gc_loaded", + // # memory(none) nosync nounwind speculatable willreturn norecurse + // declare nonnull noundef ptr(Loaded) @"julia.gc_loaded"(ptr(Tracked) nocapture nonnull noundef readnone, ptr nonnull noundef readnone) + // top: + // %metadata GC base pointer is ptr(Tracked) + // ret addrspacecast ptr to ptr(Loaded) + [](LLVMContext &C) { return FunctionType::get(PointerType::get(JuliaType::get_prjlvalue_ty(C), AddressSpace::Loaded), + {JuliaType::get_prjlvalue_ty(C), PointerType::get(JuliaType::get_prjlvalue_ty(C), 0)}, false); }, + [](LLVMContext &C) { + AttributeSet FnAttrs = Attributes(C, {Attribute::ReadNone, Attribute::NoSync, Attribute::NoUnwind, Attribute::Speculatable, Attribute::WillReturn, Attribute::NoRecurse}); + AttributeSet RetAttrs = Attributes(C, {Attribute::NonNull, Attribute::NoUndef}); + return AttributeList::get(C, FnAttrs, RetAttrs, + { Attributes(C, {Attribute::NonNull, Attribute::NoUndef, Attribute::ReadNone, Attribute::NoCapture}), + Attributes(C, {Attribute::NonNull, Attribute::NoUndef, Attribute::ReadNone}) }); }, +}; + // julia.call represents a call with julia calling convention, it is used as // @@ -1333,10 +1340,9 @@ static const auto &builtin_func_map() { { jl_f_nfields_addr, new JuliaFunction<>{XSTR(jl_f_nfields), get_func_sig, get_func_attrs} }, { jl_f__expr_addr, new JuliaFunction<>{XSTR(jl_f__expr), get_func_sig, get_func_attrs} }, { jl_f__typevar_addr, new JuliaFunction<>{XSTR(jl_f__typevar), get_func_sig, get_func_attrs} }, - { jl_f_arrayref_addr, new JuliaFunction<>{XSTR(jl_f_arrayref), get_func_sig, get_func_attrs} }, - { jl_f_const_arrayref_addr, new JuliaFunction<>{XSTR(jl_f_const_arrayref), get_func_sig, get_func_attrs} }, - { jl_f_arrayset_addr, new JuliaFunction<>{XSTR(jl_f_arrayset), get_func_sig, get_func_attrs} }, - { jl_f_arraysize_addr, new JuliaFunction<>{XSTR(jl_f_arraysize), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefget_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefget), get_func_sig, get_func_attrs} }, + { jl_f_memoryrefset_addr, new JuliaFunction<>{XSTR(jl_f_memoryrefset), get_func_sig, get_func_attrs} }, + { jl_f_memoryref_isassigned_addr,new JuliaFunction<>{XSTR(jl_f_memoryref_isassigned), get_func_sig, get_func_attrs} }, { jl_f_apply_type_addr, new JuliaFunction<>{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} }, { jl_f_donotdelete_addr, new JuliaFunction<>{XSTR(jl_f_donotdelete), get_donotdelete_sig, get_donotdelete_func_attrs} }, { jl_f_compilerbarrier_addr, new JuliaFunction<>{XSTR(jl_f_compilerbarrier), get_func_sig, get_func_attrs} }, @@ -1379,6 +1385,8 @@ static MDNode *best_tbaa(jl_tbaacache_t &tbaa_cache, jl_value_t *jt) { return tbaa_cache.tbaa_value; if (jl_is_abstracttype(jt)) return tbaa_cache.tbaa_value; + if (jl_is_genericmemory_type(jt) || jl_is_array_type(jt)) + return tbaa_cache.tbaa_array; // If we're here, we know all subtypes are (im)mutable, even if we // don't know what the exact type is return jl_is_mutable(jt) ? tbaa_cache.tbaa_mutab : tbaa_cache.tbaa_immut; @@ -1885,9 +1893,9 @@ static void undef_derived_strct(jl_codectx_t &ctx, Value *ptr, jl_datatype_t *st size_t first_offset = sty->layout->nfields ? jl_field_offset(sty, 0) : 0; if (first_offset != 0) ctx.builder.CreateMemSet(ptr, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), first_offset, MaybeAlign(0)); - size_t i, np = sty->layout->npointers; - if (np == 0) + if (sty->layout->first_ptr < 0) return; + size_t i, np = sty->layout->npointers; auto T_prjlvalue = JuliaType::get_prjlvalue_ty(ctx.builder.getContext()); ptr = ctx.builder.CreateBitCast(ptr, T_prjlvalue->getPointerTo(ptr->getType()->getPointerAddressSpace())); for (i = 0; i < np; i++) { @@ -2830,7 +2838,7 @@ static void general_use_analysis(jl_codectx_t &ctx, jl_value_t *expr, callback & } else if (jl_is_phicnode(expr)) { jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(expr, 0); - size_t i, elen = jl_array_len(values); + size_t i, elen = jl_array_nrows(values); for (i = 0; i < elen; i++) { jl_value_t *v = jl_array_ptr_ref(values, i); general_use_analysis(ctx, v, f); @@ -2838,7 +2846,7 @@ static void general_use_analysis(jl_codectx_t &ctx, jl_value_t *expr, callback & } else if (jl_is_phinode(expr)) { jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(expr, 1); - size_t i, elen = jl_array_len(values); + size_t i, elen = jl_array_nrows(values); for (i = 0; i < elen; i++) { jl_value_t *v = jl_array_ptr_ref(values, i); if (v) @@ -3473,272 +3481,274 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, return true; } - else if (f == jl_builtin_arraysize && nargs == 2) { - const jl_cgval_t &ary = argv[1]; - const jl_cgval_t &idx = argv[2]; - jl_value_t *aty = jl_unwrap_unionall(ary.typ); - if (jl_is_array_type(aty) && idx.typ == (jl_value_t*)jl_long_type) { - jl_value_t *ndp = jl_tparam1(aty); - if (jl_is_long(ndp)) { - size_t ndims = jl_unbox_long(ndp); - if (idx.constant) { - uint32_t idx_const = (uint32_t)jl_unbox_long(idx.constant); - if (idx_const > 0 && idx_const <= ndims) { - jl_value_t *ary_ex = jl_exprarg(ex, 1); - *ret = mark_julia_type(ctx, emit_arraysize(ctx, ary, ary_ex, idx_const), false, jl_long_type); - return true; - } - else if (idx_const > ndims) { - *ret = mark_julia_type(ctx, ConstantInt::get(ctx.types().T_size, 1), false, jl_long_type); - return true; - } + else if (f == jl_builtin_memoryref && nargs == 1) { + const jl_cgval_t &mem = argv[1]; + jl_value_t *mty_dt = jl_unwrap_unionall(mem.typ); + if (jl_is_genericmemory_type(mty_dt) && jl_is_concrete_type(mty_dt)) { + jl_value_t *isatomic = jl_tparam0(mty_dt); + jl_value_t *ety = jl_tparam1(mty_dt); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + jl_value_t *typ = jl_apply_type2((jl_value_t*)jl_genericmemoryref_type, isatomic, ety); + *ret = _emit_memoryref(ctx, mem, layout->arrayelem_isboxed, layout->arrayelem_isunion, layout->size == 0, typ); + return true; + } + } + + else if (f == jl_builtin_memoryref && (nargs == 2 || nargs == 3)) { + const jl_cgval_t &ref = argv[1]; + jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { + mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + jl_value_t *boundscheck = nargs == 3 ? argv[3].constant : nullptr; + if (nargs == 3) + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + *ret = emit_memoryref(ctx, ref, argv[2], boundscheck, layout->arrayelem_isboxed, layout->arrayelem_isunion, layout->size == 0); + return true; + } + } + + else if (f == jl_builtin_memoryrefget && nargs == 3) { + const jl_cgval_t &ref = argv[1]; + jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { + jl_value_t *isatomic = jl_tparam0(mty_dt); (void)isatomic; // TODO + jl_value_t *ety = jl_tparam1(mty_dt); + mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); + jl_value_t *order = argv[2].constant; + if (order != (jl_value_t*)jl_not_atomic_sym) + return false; + jl_value_t *boundscheck = argv[3].constant; + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memoryref"); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + size_t elsz = layout->size; + size_t al = layout->alignment; + Value *mem = emit_memoryref_mem(ctx, ref, isboxed, isunion, elsz == 0); + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + if (bounds_check_enabled(ctx, boundscheck)) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "load"); + ctx.builder.CreateCondBr(ctx.builder.CreateIsNull(mlen), failBB, endBB); + ctx.f->getBasicBlockList().push_back(failBB); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), { mark_callee_rooted(ctx, mem), ConstantInt::get(ctx.types().T_size, 1) }); + ctx.builder.CreateUnreachable(); + ctx.f->getBasicBlockList().push_back(endBB); + ctx.builder.SetInsertPoint(endBB); + } + if (!isboxed && !isunion && elsz == 0) { + assert(jl_is_datatype(ety) && jl_is_datatype_singleton((jl_datatype_t*)ety)); + *ret = ghostValue(ctx, ety); + } + else if (isunion) { + Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, elsz == 0); + Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); + Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); + Value *data = emit_genericmemoryptr(ctx, mem, isboxed, isunion, elsz == 0, AddressSpace::Loaded); + Value *ptindex; + if (elsz == 0) { + ptindex = data; } else { - Value *idx_dyn = emit_unbox(ctx, ctx.types().T_size, idx, (jl_value_t*)jl_long_type); - auto positive = ctx.builder.CreateICmpSGT(idx_dyn, Constant::getNullValue(ctx.types().T_size)); - setName(ctx.emission_context, positive, "ispositive"); - error_unless(ctx, positive, "arraysize: dimension out of range"); - BasicBlock *outBB = BasicBlock::Create(ctx.builder.getContext(), "outofrange", ctx.f); - BasicBlock *inBB = BasicBlock::Create(ctx.builder.getContext(), "inrange"); - BasicBlock *ansBB = BasicBlock::Create(ctx.builder.getContext(), "arraysize"); - auto oor = ctx.builder.CreateICmpSLE(idx_dyn, - ConstantInt::get(ctx.types().T_size, ndims)); - setName(ctx.emission_context, oor, "sizeddim"); - ctx.builder.CreateCondBr(oor, inBB, outBB); - ctx.builder.SetInsertPoint(outBB); - Value *v_one = ConstantInt::get(ctx.types().T_size, 1); - ctx.builder.CreateBr(ansBB); - ctx.f->getBasicBlockList().push_back(inBB); - ctx.builder.SetInsertPoint(inBB); - Value *v_sz = emit_arraysize(ctx, ary, idx_dyn); - ctx.builder.CreateBr(ansBB); - inBB = ctx.builder.GetInsertBlock(); // could have changed - ctx.f->getBasicBlockList().push_back(ansBB); - ctx.builder.SetInsertPoint(ansBB); - PHINode *result = ctx.builder.CreatePHI(ctx.types().T_size, 2); - result->addIncoming(v_one, outBB); - result->addIncoming(v_sz, inBB); - setName(ctx.emission_context, result, "arraysize"); - *ret = mark_julia_type(ctx, result, false, jl_long_type); - return true; + Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); + data = emit_bitcast(ctx, data, AT->getPointerTo()); + // isbits union selector bytes are stored after mem->length bytes + ptindex = ctx.builder.CreateInBoundsGEP(AT, data, mlen); + data = ctx.builder.CreateInBoundsGEP(AT, data, idx0); } + ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); + ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx0); + size_t elsz_c = 0, al_c = 0; + int union_max = jl_islayout_inline(ety, &elsz_c, &al_c); + assert(union_max && elsz_c == elsz && al_c == al); + *ret = emit_unionload(ctx, data, ptindex, ety, elsz, al, ctx.tbaa().tbaa_arraybuf, true, union_max, ctx.tbaa().tbaa_arrayselbyte); } + else { + MDNode *aliasscope = nullptr; + *ret = typed_load(ctx, + emit_memoryref_ptr(ctx, ref, isboxed, isunion, elsz == 0), + nullptr, ety, + isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, + aliasscope, + isboxed, + AtomicOrdering::NotAtomic); + } + return true; } } - else if ((f == jl_builtin_arrayref || f == jl_builtin_const_arrayref) && nargs >= 3) { - const jl_cgval_t &ary = argv[2]; - bool indices_ok = true; - for (size_t i = 3; i <= nargs; i++) { - if (argv[i].typ != (jl_value_t*)jl_long_type) { - indices_ok = false; - break; + else if (f == jl_builtin_memoryrefset && nargs == 4) { + const jl_cgval_t &ref = argv[1]; + jl_cgval_t val = argv[2]; + jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { + jl_value_t *isatomic = jl_tparam0(mty_dt); (void)isatomic; // TODO + jl_value_t *ety = jl_tparam1(mty_dt); + mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); + jl_value_t *order = argv[3].constant; + if (order != (jl_value_t*)jl_not_atomic_sym) + return false; + jl_value_t *boundscheck = argv[4].constant; + emit_typecheck(ctx, argv[4], (jl_value_t*)jl_bool_type, "memoryset"); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + size_t elsz = layout->size; + size_t al = layout->alignment; + Value *mem = emit_memoryref_mem(ctx, ref, isboxed, isunion, elsz == 0); + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + if (bounds_check_enabled(ctx, boundscheck)) { + BasicBlock *failBB, *endBB; + failBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "load"); + ctx.builder.CreateCondBr(ctx.builder.CreateIsNull(mlen), failBB, endBB); + ctx.f->getBasicBlockList().push_back(failBB); + ctx.builder.SetInsertPoint(failBB); + ctx.builder.CreateCall(prepare_call(jlboundserror_func), { mark_callee_rooted(ctx, mem), ConstantInt::get(ctx.types().T_size, 1) }); + ctx.builder.CreateUnreachable(); + ctx.f->getBasicBlockList().push_back(endBB); + ctx.builder.SetInsertPoint(endBB); + } + emit_typecheck(ctx, val, ety, "memoryset"); + val = update_julia_type(ctx, val, ety); + if (val.typ == jl_bottom_type) + return true; + if (isboxed) + ety = (jl_value_t*)jl_any_type; + if (!isboxed && !isunion && elsz == 0) { + assert(jl_is_datatype(ety) && jl_datatype_size(ety) == 0); + // no-op } - } - jl_value_t *aty_dt = jl_unwrap_unionall(ary.typ); - if (jl_is_array_type(aty_dt) && indices_ok) { - jl_value_t *ety = jl_tparam0(aty_dt); - jl_value_t *ndp = jl_tparam1(aty_dt); - if (!jl_has_free_typevars(ety) && (jl_is_long(ndp) || nargs == 3)) { - jl_value_t *ary_ex = jl_exprarg(ex, 2); - size_t elsz = 0, al = 0; - int union_max = jl_islayout_inline(ety, &elsz, &al); - bool isboxed = (union_max == 0); - if (isboxed) - ety = (jl_value_t*)jl_any_type; - ssize_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : -1; - jl_value_t *boundscheck = argv[1].constant; - emit_typecheck(ctx, argv[1], (jl_value_t*)jl_bool_type, "arrayref"); - Value *idx = emit_array_nd_index(ctx, ary, ary_ex, nd, &argv[3], nargs - 2, boundscheck); - auto get_arrayname = [&]() { - return ary.V ? ary.V->getName() : StringRef(""); - }; - if (!isboxed && jl_is_datatype(ety) && jl_datatype_size(ety) == 0) { - assert(((jl_datatype_t*)ety)->instance != NULL); - *ret = ghostValue(ctx, ety); + else { + Value *V = emit_memoryref_FCA(ctx, ref, isboxed, isunion, elsz == 0); + Value *data_owner = NULL; // owner object against which the write barrier must check + if (isboxed || layout->first_ptr >= 0) { // if elements are just bits, don't need a write barrier + data_owner = emit_genericmemoryowner(ctx, CreateSimplifiedExtractValue(ctx, V, 1)); } - else if (!isboxed && jl_is_uniontype(ety)) { - Value *data = emit_arrayptr(ctx, ary, ary_ex); - Value *offset = emit_arrayoffset(ctx, ary, nd); + if (isunion) { + Value *idx0 = CreateSimplifiedExtractValue(ctx, V, 0); + Value *mem = CreateSimplifiedExtractValue(ctx, V, 1); + Value *data = emit_genericmemoryptr(ctx, mem, isboxed, isunion, elsz == 0, AddressSpace::Loaded); + Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); + data = emit_bitcast(ctx, data, AT->getPointerTo()); + // compute tindex from val + jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety); + Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety); + tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 1)); Value *ptindex; if (elsz == 0) { ptindex = data; } else { - Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); data = emit_bitcast(ctx, data, AT->getPointerTo()); - // isbits union selector bytes are stored after a->maxsize - Value *ndims = (nd == -1 ? emit_arrayndims(ctx, ary) : ConstantInt::get(getInt16Ty(ctx.builder.getContext()), nd)); - Value *is_vector = ctx.builder.CreateICmpEQ(ndims, ConstantInt::get(getInt16Ty(ctx.builder.getContext()), 1)); - setName(ctx.emission_context, is_vector, get_arrayname() + ".isvec"); - Value *selidx_v = ctx.builder.CreateSub(emit_vectormaxsize(ctx, ary), ctx.builder.CreateZExt(offset, ctx.types().T_size)); - setName(ctx.emission_context, selidx_v, get_arrayname() + ".vec_selidx"); - Value *selidx_m = emit_arraylen(ctx, ary); - Value *selidx = ctx.builder.CreateSelect(is_vector, selidx_v, selidx_m); - setName(ctx.emission_context, selidx, get_arrayname() + ".selidx"); - ptindex = ctx.builder.CreateInBoundsGEP(AT, data, selidx); - data = ctx.builder.CreateInBoundsGEP(AT, data, idx); - setName(ctx.emission_context, data, get_arrayname() + ".data_ptr"); + // isbits union selector bytes are stored after mem->length + ptindex = ctx.builder.CreateInBoundsGEP(AT, data, mlen); + data = ctx.builder.CreateInBoundsGEP(AT, data, idx0); } ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); - ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, offset); - ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx); - setName(ctx.emission_context, ptindex, get_arrayname() + ".tindex_ptr"); - *ret = emit_unionload(ctx, data, ptindex, ety, elsz, al, ctx.tbaa().tbaa_arraybuf, true, union_max, ctx.tbaa().tbaa_arrayselbyte); - if (ret->V) - setName(ctx.emission_context, ret->V, get_arrayname() + ".ref"); + ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx0); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayselbyte); + ai.decorateInst(ctx.builder.CreateStore(tindex, ptindex)); + if (elsz > 0 && (!jl_is_datatype(val.typ) || jl_datatype_size(val.typ) > 0)) { + // copy data (if any) + emit_unionmove(ctx, data, ctx.tbaa().tbaa_arraybuf, val, nullptr); + } } else { - MDNode *aliasscope = (f == jl_builtin_const_arrayref) ? ctx.noalias().aliasscope.current : nullptr; - *ret = typed_load(ctx, - emit_arrayptr(ctx, ary, ary_ex), - idx, ety, - isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, - aliasscope, - isboxed, - AtomicOrdering::NotAtomic); - if (ret->V) - setName(ctx.emission_context, ret->V, get_arrayname() + ".ref"); + typed_store(ctx, + emit_memoryref_ptr(ctx, ref, isboxed, isunion, elsz == 0), + nullptr, val, jl_cgval_t(), ety, + isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, + ctx.noalias().aliasscope.current, + data_owner, + isboxed, + isboxed ? AtomicOrdering::Release : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 + /*FailOrder*/AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 + 0, + false, + true, + false, + false, + false, + false, + nullptr, + ""); } - return true; } + *ret = ref; + return true; } } - else if (f == jl_builtin_arrayset && nargs >= 4) { - const jl_cgval_t &ary = argv[2]; - jl_cgval_t val = argv[3]; - bool indices_ok = true; - for (size_t i = 4; i <= nargs; i++) { - if (argv[i].typ != (jl_value_t*)jl_long_type) { - indices_ok = false; - break; - } - } - jl_value_t *aty_dt = jl_unwrap_unionall(ary.typ); - if (jl_is_array_type(aty_dt) && indices_ok) { - jl_value_t *ety = jl_tparam0(aty_dt); - jl_value_t *ndp = jl_tparam1(aty_dt); - if (!jl_has_free_typevars(ety) && (jl_is_long(ndp) || nargs == 4)) { - emit_typecheck(ctx, val, ety, "arrayset"); - val = update_julia_type(ctx, val, ety); - if (val.typ == jl_bottom_type) - return true; - size_t elsz = 0, al = 0; - int union_max = jl_islayout_inline(ety, &elsz, &al); - bool isboxed = (union_max == 0); - if (isboxed) - ety = (jl_value_t*)jl_any_type; - jl_value_t *ary_ex = jl_exprarg(ex, 2); - ssize_t nd = jl_is_long(ndp) ? jl_unbox_long(ndp) : -1; - jl_value_t *boundscheck = argv[1].constant; - emit_typecheck(ctx, argv[1], (jl_value_t*)jl_bool_type, "arrayset"); - Value *idx = emit_array_nd_index(ctx, ary, ary_ex, nd, &argv[4], nargs - 3, boundscheck); - if (!isboxed && jl_is_datatype(ety) && jl_datatype_size(ety) == 0) { - // no-op + else if (f == jl_builtin_memoryref_isassigned && nargs == 3) { + const jl_cgval_t &ref = argv[1]; + jl_value_t *mty_dt = jl_unwrap_unionall(ref.typ); + if (jl_is_genericmemoryref_type(mty_dt) && jl_is_concrete_type(mty_dt)) { + jl_value_t *isatomic = jl_tparam0(mty_dt); (void)isatomic; // TODO + mty_dt = jl_field_type_concrete((jl_datatype_t*)mty_dt, 1); + jl_value_t *order = argv[2].constant; + if (order != (jl_value_t*)jl_not_atomic_sym) + return false; + jl_value_t *boundscheck = argv[3].constant; + emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "memory_isassigned"); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty_dt)->layout; + bool isboxed = layout->arrayelem_isboxed; + bool isunion = layout->arrayelem_isunion; + size_t elsz = layout->size; + Value *mem = emit_memoryref_mem(ctx, ref, isboxed, isunion, elsz == 0); + Value *mlen = emit_genericmemorylen(ctx, mem, ref.typ); + Value *oob = bounds_check_enabled(ctx, boundscheck) ? ctx.builder.CreateIsNull(mlen) : nullptr; + if (isboxed || layout->first_ptr >= 0) { + PHINode *result = nullptr; + if (oob) { + BasicBlock *passBB, *endBB, *fromBB; + passBB = BasicBlock::Create(ctx.builder.getContext(), "load"); + endBB = BasicBlock::Create(ctx.builder.getContext(), "oob"); + ctx.f->getBasicBlockList().push_back(passBB); + ctx.f->getBasicBlockList().push_back(endBB); + fromBB = ctx.builder.CreateCondBr(oob, endBB, passBB)->getParent(); + ctx.builder.SetInsertPoint(endBB); + result = ctx.builder.CreatePHI(getInt1Ty(ctx.builder.getContext()), 2); + result->addIncoming(ConstantInt::get(result->getType(), 0), fromBB); + setName(ctx.emission_context, result, "arraysize"); + ctx.builder.SetInsertPoint(passBB); } - else { - PHINode *data_owner = NULL; // owner object against which the write barrier must check - if (isboxed || (jl_is_datatype(ety) && ((jl_datatype_t*)ety)->layout->npointers > 0)) { // if elements are just bits, don't need a write barrier - Value *aryv = boxed(ctx, ary); - Value *flags = emit_arrayflags(ctx, ary); - // the owner of the data is ary itself except if ary->how == 3 - flags = ctx.builder.CreateAnd(flags, 3); - Value *is_owned = ctx.builder.CreateICmpEQ(flags, ConstantInt::get(getInt16Ty(ctx.builder.getContext()), 3)); - setName(ctx.emission_context, is_owned, "has_owner"); - BasicBlock *curBB = ctx.builder.GetInsertBlock(); - BasicBlock *ownedBB = BasicBlock::Create(ctx.builder.getContext(), "array_owned", ctx.f); - BasicBlock *mergeBB = BasicBlock::Create(ctx.builder.getContext(), "merge_own", ctx.f); - ctx.builder.CreateCondBr(is_owned, ownedBB, mergeBB); - ctx.builder.SetInsertPoint(ownedBB); - // load owner pointer - Instruction *own_ptr; - if (jl_is_long(ndp)) { - own_ptr = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, - ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, - emit_bitcast(ctx, decay_derived(ctx, aryv), ctx.types().T_pprjlvalue), - jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)), - Align(sizeof(void*))); - setName(ctx.emission_context, own_ptr, "external_owner"); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); - ai.decorateInst(maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type)); - } - else { - own_ptr = ctx.builder.CreateCall( - prepare_call(jlarray_data_owner_func), - {aryv}); - } - ctx.builder.CreateBr(mergeBB); - ctx.builder.SetInsertPoint(mergeBB); - data_owner = ctx.builder.CreatePHI(ctx.types().T_prjlvalue, 2); - data_owner->addIncoming(aryv, curBB); - data_owner->addIncoming(own_ptr, ownedBB); - setName(ctx.emission_context, data_owner, "data_owner"); - } - if (!isboxed && jl_is_uniontype(ety)) { - Type *AT = ArrayType::get(IntegerType::get(ctx.builder.getContext(), 8 * al), (elsz + al - 1) / al); - Value *data = emit_bitcast(ctx, emit_arrayptr(ctx, ary, ary_ex), AT->getPointerTo()); - Value *offset = emit_arrayoffset(ctx, ary, nd); - // compute tindex from val - jl_cgval_t rhs_union = convert_julia_type(ctx, val, ety); - Value *tindex = compute_tindex_unboxed(ctx, rhs_union, ety); - tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 1)); - Value *ptindex; - if (elsz == 0) { - ptindex = data; - } - else { - Value *ndims = (nd == -1 ? emit_arrayndims(ctx, ary) : ConstantInt::get(getInt16Ty(ctx.builder.getContext()), nd)); - Value *is_vector = ctx.builder.CreateICmpEQ(ndims, ConstantInt::get(getInt16Ty(ctx.builder.getContext()), 1)); - setName(ctx.emission_context, is_vector, "is_vector"); - Value *selidx_v = ctx.builder.CreateSub(emit_vectormaxsize(ctx, ary), ctx.builder.CreateZExt(offset, ctx.types().T_size)); - setName(ctx.emission_context, selidx_v, "selidx_v"); - Value *selidx_m = emit_arraylen(ctx, ary); - Value *selidx = ctx.builder.CreateSelect(is_vector, selidx_v, selidx_m); - setName(ctx.emission_context, selidx, "selidx"); - ptindex = ctx.builder.CreateInBoundsGEP(AT, data, selidx); - setName(ctx.emission_context, ptindex, "ptindex"); - data = ctx.builder.CreateInBoundsGEP(AT, data, idx); - setName(ctx.emission_context, data, "data"); - } - ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); - ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, offset); - ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx); - setName(ctx.emission_context, ptindex, "ptindex"); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayselbyte); - ai.decorateInst(ctx.builder.CreateStore(tindex, ptindex)); - if (elsz > 0 && (!jl_is_datatype(val.typ) || jl_datatype_size(val.typ) > 0)) { - // copy data (if any) - emit_unionmove(ctx, data, ctx.tbaa().tbaa_arraybuf, val, nullptr); - } - } - else { - typed_store(ctx, - emit_arrayptr(ctx, ary, ary_ex, isboxed), - idx, val, jl_cgval_t(), ety, - isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, - ctx.noalias().aliasscope.current, - data_owner, - isboxed, - isboxed ? AtomicOrdering::Release : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - /*FailOrder*/AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 - 0, - false, - true, - false, - false, - false, - false, - nullptr, - ""); - } + Value *elem = emit_memoryref_ptr(ctx, ref, isboxed, isunion, elsz == 0); + elem = emit_bitcast(ctx, elem, ctx.types().T_pprjlvalue); + if (!isboxed) + elem = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, elem, layout->first_ptr); + // emit this using the same type as jl_builtin_memoryrefget + // so that LLVM may be able to load-load forward them and fold the result + auto tbaa = isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf; + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + LoadInst *fldv = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, elem, ctx.types().alignof_ptr); + //fldv->setOrdering(AtomicOrdering::Unordered); + ai.decorateInst(fldv); + Value *isdef = ctx.builder.CreateIsNotNull(fldv); + setName(ctx.emission_context, isdef, "memoryref_isassigned"); + if (oob) { + assert(result); + result->addIncoming(isdef, ctx.builder.CreateBr(result->getParent())->getParent()); + ctx.builder.SetInsertPoint(result->getParent()); + isdef = result; } - *ret = ary; - return true; + *ret = mark_julia_type(ctx, isdef, false, jl_bool_type); } + else if (oob) { + Value *isdef = ctx.builder.CreateNot(oob); + *ret = mark_julia_type(ctx, isdef, false, jl_bool_type); + } + else { + *ret = mark_julia_const(ctx, jl_true); + } + return true; } } + else if (f == jl_builtin_getfield && (nargs == 2 || nargs == 3 || nargs == 4)) { const jl_cgval_t &obj = argv[1]; const jl_cgval_t &fld = argv[2]; @@ -4036,19 +4046,11 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, *ret = mark_julia_type(ctx, len, false, jl_long_type); return true; } - else if (jl_is_array_type(sty)) { - auto len = emit_arraylen(ctx, obj); - Value *elsize; - size_t elsz; - if (arraytype_constelsize(sty, &elsz)) { - elsize = ConstantInt::get(ctx.types().T_size, elsz); - } - else { - elsize = ctx.builder.CreateZExt(emit_arrayelsize(ctx, obj), ctx.types().T_size); - } + else if (jl_is_genericmemory_type(sty)) { + Value *v = boxed(ctx, obj); + auto len = emit_genericmemorylen(ctx, v, (jl_value_t*)sty); + auto elsize = emit_genericmemoryelsize(ctx, v, obj.typ); *ret = mark_julia_type(ctx, ctx.builder.CreateMul(len, elsize), false, jl_long_type); - if (ret->V) - setName(ctx.emission_context, ret->V, "sizeof"); return true; } } @@ -4143,10 +4145,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, else if (jl_field_isptr(stt, fieldidx) || jl_type_hasptr(jl_field_type(stt, fieldidx))) { Value *fldv; size_t offs = jl_field_offset(stt, fieldidx) / sizeof(jl_value_t*); - auto tbaa = obj.tbaa; - if (tbaa == ctx.tbaa().tbaa_datatype && offs != offsetof(jl_datatype_t, types)) - tbaa = ctx.tbaa().tbaa_const; if (obj.ispointer()) { + auto tbaa = best_field_tbaa(ctx, obj, stt, fieldidx, offs); if (!jl_field_isptr(stt, fieldidx)) offs += ((jl_datatype_t*)jl_field_type(stt, fieldidx))->layout->first_ptr; Value *ptr = emit_bitcast(ctx, data_pointer(ctx, obj), ctx.types().T_pprjlvalue); @@ -4432,7 +4432,7 @@ static jl_cgval_t emit_call_specfun_boxed(jl_codectx_t &ctx, jl_value_t *jlretty static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) { - jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + jl_value_t **args = jl_array_data(ex->args, jl_value_t*); size_t arglen = jl_array_dim0(ex->args); size_t nargs = arglen - 1; assert(arglen >= 2); @@ -4554,7 +4554,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) { ++EmittedInvokes; - jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + jl_value_t **args = jl_array_data(ex->args, jl_value_t*); size_t arglen = jl_array_dim0(ex->args); size_t nargs = arglen - 1; assert(arglen >= 2); @@ -4618,7 +4618,7 @@ static jl_cgval_t emit_specsig_oc_call(jl_codectx_t &ctx, jl_value_t *oc_type, j static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bool is_promotable) { ++EmittedCalls; - jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + jl_value_t **args = jl_array_data(ex->args, jl_value_t*); size_t nargs = jl_array_dim0(ex->args); assert(nargs >= 1); jl_cgval_t f = emit_expr(ctx, args[0]); @@ -5080,9 +5080,9 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) if (dest) { Instruction *phi = dest->clone(); phi->insertAfter(dest); - PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_len(edges), "tindex_phi"); + PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_nrows(edges), "tindex_phi"); BB->getInstList().insert(InsertPt, Tindex_phi); - PHINode *ptr_phi = PHINode::Create(ctx.types().T_prjlvalue, jl_array_len(edges), "ptr_phi"); + PHINode *ptr_phi = PHINode::Create(ctx.types().T_prjlvalue, jl_array_nrows(edges), "ptr_phi"); BB->getInstList().insert(InsertPt, ptr_phi); Value *isboxed = ctx.builder.CreateICmpNE( ctx.builder.CreateAnd(Tindex_phi, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0x80)), @@ -5100,7 +5100,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) return; } else if (allunbox) { - PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_len(edges), "tindex_phi"); + PHINode *Tindex_phi = PHINode::Create(getInt8Ty(ctx.builder.getContext()), jl_array_nrows(edges), "tindex_phi"); BB->getInstList().insert(InsertPt, Tindex_phi); jl_cgval_t val = mark_julia_slot(NULL, phiType, Tindex_phi, ctx.tbaa().tbaa_stack); ctx.PhiNodes.push_back(std::make_tuple(val, BB, dest, (PHINode*)NULL, r)); @@ -5114,7 +5114,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) // The frontend should really not emit this, but we allow it // for convenience. if (type_is_ghost(vtype)) { - assert(jl_is_datatype(phiType) && ((jl_datatype_t*)phiType)->instance); + assert(jl_is_datatype(phiType) && jl_is_datatype_singleton((jl_datatype_t*)phiType)); // Skip adding it to the PhiNodes list, since we didn't create one. ctx.SAvalues.at(idx) = mark_julia_const(ctx, ((jl_datatype_t*)phiType)->instance); ctx.ssavalue_assigned.at(idx) = true; @@ -5134,7 +5134,7 @@ static void emit_phinode_assign(jl_codectx_t &ctx, ssize_t idx, jl_value_t *r) slot = mark_julia_slot(phi, phiType, NULL, ctx.tbaa().tbaa_stack); } else { - value_phi = PHINode::Create(vtype, jl_array_len(edges), "value_phi"); + value_phi = PHINode::Create(vtype, jl_array_nrows(edges), "value_phi"); BB->getInstList().insert(InsertPt, value_phi); slot = mark_julia_type(ctx, value_phi, isboxed, phiType); } @@ -5373,7 +5373,7 @@ static void emit_stmtpos(jl_codectx_t &ctx, jl_value_t *expr, int ssaval_result) return; } jl_expr_t *ex = (jl_expr_t*)expr; - jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); + jl_value_t **args = jl_array_data(ex->args, jl_value_t*); jl_sym_t *head = ex->head; if (head == jl_meta_sym || head == jl_inbounds_sym || head == jl_coverageeffect_sym || head == jl_aliasscope_sym || head == jl_popaliasscope_sym || head == jl_inline_sym || head == jl_noinline_sym) { @@ -5524,8 +5524,8 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ } jl_expr_t *ex = (jl_expr_t*)expr; - jl_value_t **args = (jl_value_t**)jl_array_data(ex->args); - size_t nargs = jl_array_len(ex->args); + jl_value_t **args = jl_array_data(ex->args, jl_value_t*); + size_t nargs = jl_array_nrows(ex->args); jl_sym_t *head = ex->head; // this is object-disoriented. // however, this is a good way to do it because it should *not* be easy @@ -6017,7 +6017,7 @@ static void emit_cfunc_invalidate( myargs[i] = mark_julia_const(ctx, jl_tparam0(jt)); } else if (type_is_ghost(et)) { - assert(jl_is_datatype(jt) && ((jl_datatype_t*)jt)->instance); + assert(jl_is_datatype(jt) && jl_is_datatype_singleton((jl_datatype_t*)jt)); myargs[i] = mark_julia_const(ctx, ((jl_datatype_t*)jt)->instance); } else { @@ -6328,7 +6328,7 @@ static Function* gen_cfun_wrapper( *closure_types = jl_alloc_vec_any(0); jl_array_ptr_1d_push(*closure_types, jargty); Value *runtime_dt = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, - ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, nestPtr, jl_array_len(*closure_types)), + ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, nestPtr, jl_array_nrows(*closure_types)), Align(sizeof(void*))); BasicBlock *boxedBB = BasicBlock::Create(ctx.builder.getContext(), "isboxed", cw); BasicBlock *loadBB = BasicBlock::Create(ctx.builder.getContext(), "need-load", cw); @@ -6395,7 +6395,7 @@ static Function* gen_cfun_wrapper( *closure_types = jl_alloc_vec_any(0); jl_array_ptr_1d_push(*closure_types, jargty); Value *runtime_dt = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, - ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, nestPtr, jl_array_len(*closure_types)), + ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, nestPtr, jl_array_nrows(*closure_types)), Align(sizeof(void*))); Value *strct = box_ccall_result(ctx, val, runtime_dt, jargty); inputarg = mark_julia_type(ctx, strct, true, jargty_proper); @@ -6769,7 +6769,7 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con jl_svec_t *fill = jl_emptysvec; if (closure_types) { assert(ctx.spvals_ptr); - size_t n = jl_array_len(closure_types); + size_t n = jl_array_nrows(closure_types); jl_svec_t *fill = jl_alloc_svec_uninit(n); for (size_t i = 0; i < n; i++) { jl_svecset(fill, i, jl_array_ptr_ref(closure_types, i)); @@ -7056,7 +7056,7 @@ static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, Value rt = julia_type_to_llvm(ctx, jlrettype, &retboxed); assert(!retboxed); if (rt != getVoidTy(ctx.builder.getContext()) && deserves_sret(jlrettype, rt)) { - auto tracked = CountTrackedPointers(rt); + auto tracked = CountTrackedPointers(rt, true); assert(!tracked.derived); if (tracked.count && !tracked.all) props.return_roots = tracked.count; @@ -7330,7 +7330,7 @@ static jl_llvm_functions_t toplineno = lam->def.method->line; ctx.file = jl_symbol_name(lam->def.method->file); } - else if (jl_array_len(src->linetable) > 0) { + else if (jl_array_nrows(src->linetable) > 0) { jl_value_t *locinfo = jl_array_ptr_ref(src->linetable, 0); ctx.file = jl_symbol_name((jl_sym_t*)jl_fieldref_noalloc(locinfo, 2)); toplineno = jl_unbox_int32(jl_fieldref(locinfo, 3)); @@ -7345,7 +7345,7 @@ static jl_llvm_functions_t debug_enabled = false; // step 2. process var-info lists to see what vars need boxing - int n_ssavalues = jl_is_long(src->ssavaluetypes) ? jl_unbox_long(src->ssavaluetypes) : jl_array_len(src->ssavaluetypes); + int n_ssavalues = jl_is_long(src->ssavaluetypes) ? jl_unbox_long(src->ssavaluetypes) : jl_array_nrows(src->ssavaluetypes); size_t vinfoslen = jl_array_dim0(src->slotflags); ctx.slots.resize(vinfoslen, jl_varinfo_t(ctx.builder.getContext())); assert(lam->specTypes); // the specTypes field should always be assigned @@ -7375,7 +7375,7 @@ static jl_llvm_functions_t // OpaqueClosure implicitly loads the env if (i == 0 && ctx.is_opaque_closure) { if (jl_is_array(src->slottypes)) { - ty = jl_arrayref((jl_array_t*)src->slottypes, i); + ty = jl_array_ptr_ref((jl_array_t*)src->slottypes, i); } else { ty = (jl_value_t*)jl_any_type; @@ -7472,7 +7472,7 @@ static jl_llvm_functions_t // case the apply-generic call can re-use the original box for the return int retarg = [stmts, nreq]() { int retarg = -1; - for (size_t i = 0; i < jl_array_len(stmts); ++i) { + for (size_t i = 0; i < jl_array_nrows(stmts); ++i) { jl_value_t *stmt = jl_array_ptr_ref(stmts, i); if (jl_is_returnnode(stmt)) { stmt = jl_returnnode_value(stmt); @@ -7775,7 +7775,7 @@ static jl_llvm_functions_t // yield to them. // Also count ssavalue uses. { - for (size_t i = 0; i < jl_array_len(stmts); ++i) { + for (size_t i = 0; i < jl_array_nrows(stmts); ++i) { jl_value_t *stmt = jl_array_ptr_ref(stmts, i); auto scan_ssavalue = [&](jl_value_t *val) { @@ -7789,7 +7789,7 @@ static jl_llvm_functions_t if (jl_is_phicnode(stmt)) { jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(stmt, 0); - for (size_t j = 0; j < jl_array_len(values); ++j) { + for (size_t j = 0; j < jl_array_nrows(values); ++j) { jl_value_t *val = jl_array_ptr_ref(values, j); assert(jl_is_ssavalue(val)); upsilon_to_phic[((jl_ssavalue_t*)val)->id] = i; @@ -8056,7 +8056,7 @@ static jl_llvm_functions_t std::vector linetable; { // populate the linetable data format assert(jl_is_array(src->linetable)); - size_t nlocs = jl_array_len(src->linetable); + size_t nlocs = jl_array_nrows(src->linetable); std::map, DISubprogram*> subprograms; linetable.resize(nlocs + 1); DebugLineTable &topinfo = linetable[0]; @@ -8128,7 +8128,7 @@ static jl_llvm_functions_t std::vector scope_stack; std::vector scope_list_stack; { - size_t nstmts = jl_array_len(stmts); + size_t nstmts = jl_array_nrows(stmts); aliasscopes.resize(nstmts + 1, nullptr); MDBuilder mbuilder(ctx.builder.getContext()); MDNode *alias_domain = mbuilder.createAliasScopeDomain(ctx.name); @@ -8298,8 +8298,8 @@ static jl_llvm_functions_t branch_targets.insert(i + 2); } else if (jl_is_phinode(stmt)) { jl_array_t *edges = (jl_array_t*)jl_fieldref_noalloc(stmt, 0); - for (size_t j = 0; j < jl_array_len(edges); ++j) { - size_t edge = ((int32_t*)jl_array_data(edges))[j]; + for (size_t j = 0; j < jl_array_nrows(edges); ++j) { + size_t edge = jl_array_data(edges, int32_t)[j]; if (edge == i) branch_targets.insert(i + 1); } @@ -8328,7 +8328,7 @@ static jl_llvm_functions_t find_next_stmt(0); while (cursor != -1) { - int32_t debuginfoloc = ((int32_t*)jl_array_data(src->codelocs))[cursor]; + int32_t debuginfoloc = jl_array_data(src->codelocs, int32_t)[cursor]; if (debuginfoloc > 0) { if (debug_enabled) ctx.builder.SetCurrentDebugLocation(linetable.at(debuginfoloc).loc); @@ -8491,7 +8491,7 @@ static jl_llvm_functions_t continue; } else if (expr && expr->head == jl_enter_sym) { - jl_value_t **args = (jl_value_t**)jl_array_data(expr->args); + jl_value_t **args = jl_array_data(expr->args, jl_value_t*); assert(jl_is_long(args[0])); int lname = jl_unbox_long(args[0]); @@ -8544,8 +8544,8 @@ static jl_llvm_functions_t jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(r, 1); PHINode *TindexN = cast_or_null(phi_result.TIndex); DenseSet preds; - for (size_t i = 0; i < jl_array_len(edges); ++i) { - size_t edge = ((int32_t*)jl_array_data(edges))[i]; + for (size_t i = 0; i < jl_array_nrows(edges); ++i) { + size_t edge = jl_array_data(edges, int32_t)[i]; jl_value_t *value = jl_array_ptr_ref(values, i); // This edge value is undef, handle it the same as if the edge wasn't listed at all if (!value) @@ -8564,7 +8564,7 @@ static jl_llvm_functions_t // Only codegen this branch once for each PHI (the expression must be the same on all branches) #ifndef NDEBUG for (size_t j = 0; j < i; ++j) { - size_t j_edge = ((int32_t*)jl_array_data(edges))[j]; + size_t j_edge = jl_array_data(edges, int32_t)[j]; if (j_edge == edge) { assert(jl_egal(value, jl_array_ptr_ref(values, j))); } @@ -9248,7 +9248,6 @@ static void init_jit_functions(void) add_named_global(jlfieldindex_func, &jl_field_index); add_named_global(diff_gc_total_bytes_func, &jl_gc_diff_total_bytes); add_named_global(sync_gc_total_bytes_func, &jl_gc_sync_total_bytes); - add_named_global(jlarray_data_owner_func, &jl_array_data_owner); add_named_global(gcroot_flush_func, (void*)NULL); add_named_global(gc_preserve_begin_func, (void*)NULL); add_named_global(gc_preserve_end_func, (void*)NULL); diff --git a/src/common_symbols1.inc b/src/common_symbols1.inc index 547d5d0eabede..f54be52729a4f 100644 --- a/src/common_symbols1.inc +++ b/src/common_symbols1.inc @@ -1,98 +1,92 @@ jl_symbol("="), jl_symbol("getproperty"), -jl_symbol("apply_type"), jl_symbol("getfield"), +jl_symbol("apply_type"), +jl_symbol("==="), jl_symbol("getindex"), jl_symbol("convert"), -jl_symbol("==="), -jl_symbol("iterate"), jl_symbol("=="), jl_symbol("new"), -jl_symbol("foreigncall"), jl_symbol("int.jl"), -jl_symbol("throw"), -jl_symbol("nothing"), -jl_symbol("essentials.jl"), jl_symbol("+"), -jl_symbol("unsafe_convert"), +jl_symbol("boot.jl"), +jl_symbol("essentials.jl"), +jl_symbol("ccall"), +jl_symbol("foreigncall"), +jl_symbol("iterate"), jl_symbol("not_int"), +jl_symbol("Base.jl"), jl_symbol("-"), -jl_symbol("boot.jl"), -jl_symbol("number.jl"), +jl_symbol("throw"), +jl_symbol("promotion.jl"), jl_symbol("length"), jl_symbol("<"), -jl_symbol("cconvert"), -jl_symbol("Base.jl"), -jl_symbol("promotion.jl"), -jl_symbol("tuple.jl"), -jl_symbol("static_parameter"), -jl_symbol("isempty"), -jl_symbol("<="), -jl_symbol("array.jl"), +jl_symbol("isa"), jl_symbol("operators.jl"), -jl_symbol("NamedTuple"), +jl_symbol("number.jl"), +jl_symbol("unsafe_convert"), +jl_symbol("tuple.jl"), +jl_symbol("nothing"), jl_symbol("bitcast"), -jl_symbol("!"), +jl_symbol("NamedTuple"), jl_symbol("indexed_iterate"), -jl_symbol("sle_int"), jl_symbol("bool.jl"), -jl_symbol("Ptr"), -jl_symbol("size"), +jl_symbol("!"), +jl_symbol("isempty"), +jl_symbol("<="), +jl_symbol("cconvert"), jl_symbol("add_int"), +jl_symbol("static_parameter"), +jl_symbol("array.jl"), jl_symbol("slt_int"), -jl_symbol("*"), -jl_symbol("range.jl"), -jl_symbol("abstractarray.jl"), jl_symbol("!="), -jl_symbol("isa"), -jl_symbol("setindex!"), -jl_symbol("string"), -jl_symbol("ifelse"), -jl_symbol(":"), -jl_symbol(">"), -jl_symbol("_apply_iterate"), jl_symbol("UInt64"), +jl_symbol("range.jl"), +jl_symbol("sle_int"), +jl_symbol("size"), jl_symbol("&"), -jl_symbol("max"), +jl_symbol("abstractarray.jl"), jl_symbol("rem"), -jl_symbol("sub_int"), -jl_symbol(">="), -jl_symbol("UInt8"), -jl_symbol("iterators.jl"), +jl_symbol(">"), jl_symbol("Int64"), -jl_symbol("pairs"), +jl_symbol("sub_int"), +jl_symbol("*"), jl_symbol("and_int"), +jl_symbol("string"), +jl_symbol(">="), +jl_symbol("Ptr"), +jl_symbol("toInt64"), jl_symbol("last"), -jl_symbol("typeof"), -jl_symbol("arrayref"), jl_symbol("pointer.jl"), -jl_symbol("toInt64"), -jl_symbol("arraylen"), +jl_symbol("reinterpret"), +jl_symbol("first"), +jl_symbol("pairs"), +jl_symbol("_apply_iterate"), jl_symbol("typeassert"), -jl_symbol("map"), +jl_symbol(":"), +jl_symbol("UInt8"), +jl_symbol("setindex!"), +jl_symbol("isdefined"), +jl_symbol("typeof"), +jl_symbol("promote"), jl_symbol("kwcall"), -jl_symbol("ArgumentError"), +jl_symbol("unsigned"), +jl_symbol("_promote"), +jl_symbol("toUInt64"), +jl_symbol("map"), jl_symbol("lshr_int"), +jl_symbol("gc_preserve_begin"), +jl_symbol("gc_preserve_end"), +jl_symbol("trunc_int"), +jl_symbol("ArgumentError"), jl_symbol("axes"), -jl_symbol("reinterpret"), +jl_symbol("ult_int"), +jl_symbol("UInt"), +jl_symbol("zext_int"), +jl_symbol("strings/string.jl"), +jl_symbol("ifelse"), jl_symbol("Array"), -jl_symbol("first"), -jl_symbol("trunc_int"), -jl_symbol("OneTo"), -jl_symbol("haskey"), -jl_symbol("Int"), -jl_symbol("oneto"), jl_symbol("eq_int"), jl_symbol("throw_inexacterror"), -jl_symbol("toUInt64"), -jl_symbol("arraysize"), -jl_symbol("UInt"), +jl_symbol("|"), jl_symbol("setproperty!"), -jl_symbol("check_top_bit"), -jl_symbol("promote"), -jl_symbol("unsigned"), -jl_symbol("is_top_bit_set"), -jl_symbol("structdiff"), -jl_symbol("undef"), -jl_symbol("sizeof"), -jl_symbol("String"), diff --git a/src/common_symbols2.inc b/src/common_symbols2.inc index b5a334172dd76..6850fcd3033d1 100644 --- a/src/common_symbols2.inc +++ b/src/common_symbols2.inc @@ -1,254 +1,249 @@ -jl_symbol("namedtuple.jl"), -jl_symbol("pop"), -jl_symbol("inbounds"), -jl_symbol("strings/string.jl"), -jl_symbol("Ref"), -jl_symbol("Vector"), -jl_symbol("kwerr"), -jl_symbol("_promote"), jl_symbol("sext_int"), -jl_symbol("pointer"), -jl_symbol("similar"), -jl_symbol("arrayset"), +jl_symbol("String"), +jl_symbol("Int"), +jl_symbol("iterators.jl"), +jl_symbol("Colon"), +jl_symbol("unchecked_oneto"), +jl_symbol("structdiff"), +jl_symbol("UnitRange"), +jl_symbol("unitrange_last"), +jl_symbol("sizeof"), +jl_symbol("check_top_bit"), +jl_symbol("is_top_bit_set"), +jl_symbol("data"), +jl_symbol("kwerr"), jl_symbol("axes1"), jl_symbol("eachindex"), -jl_symbol("|"), -jl_symbol("ult_int"), -jl_symbol("lastindex"), -jl_symbol("setfield!"), -jl_symbol("UnitRange"), -jl_symbol("push!"), +jl_symbol("or_int"), jl_symbol("Bool"), -jl_symbol("Colon"), +jl_symbol("setfield!"), jl_symbol("fieldtype"), -jl_symbol("unitrange_last"), -jl_symbol("bitarray.jl"), -jl_symbol("<<"), -jl_symbol("zext_int"), -jl_symbol("Tuple"), +jl_symbol("Ref"), +jl_symbol("pointer"), +jl_symbol("max"), +jl_symbol("push!"), +jl_symbol("lastindex"), jl_symbol("reflection.jl"), -jl_symbol("TypeError"), -jl_symbol("print"), -jl_symbol("eltype"), +jl_symbol("<<"), +jl_symbol("similar"), +jl_symbol("Vector"), +jl_symbol("UInt32"), jl_symbol(">>"), -jl_symbol("strings/basic.jl"), -jl_symbol("gc_preserve_begin"), -jl_symbol("require_one_based_indexing"), -jl_symbol("gc_preserve_end"), -jl_symbol("DimensionMismatch"), -jl_symbol("indices.jl"), -jl_symbol("Cvoid"), -jl_symbol("oftype"), -jl_symbol("zero"), -jl_symbol("float.jl"), -jl_symbol("Any"), -jl_symbol("checkbounds"), -jl_symbol("or_int"), -jl_symbol("isdefined"), jl_symbol("dict.jl"), +jl_symbol("checkbounds"), +jl_symbol("undef"), +jl_symbol("jl_string_ptr"), +jl_symbol("error"), jl_symbol("strings/io.jl"), -jl_symbol("shl_int"), -jl_symbol("copy"), -jl_symbol("macro expansion"), -jl_symbol("abstractdict.jl"), +jl_symbol("strings/substring.jl"), +jl_symbol("bitarray.jl"), +jl_symbol("strings/basic.jl"), +jl_symbol("merge"), +jl_symbol("TypeError"), +jl_symbol("keyword argument"), jl_symbol("in"), -jl_symbol("io.jl"), -jl_symbol("BlasInt"), -jl_symbol("Float64"), +jl_symbol("print"), +jl_symbol("macro expansion"), jl_symbol("mul_int"), -jl_symbol("UInt32"), +jl_symbol("shl_int"), jl_symbol("C_NULL"), +jl_symbol("oftype"), +jl_symbol("_growend!"), +jl_symbol("Any"), +jl_symbol("__inbounds_setindex!"), +jl_symbol("Tuple"), +jl_symbol("float.jl"), +jl_symbol("ncodeunits"), jl_symbol("Integer"), +jl_symbol("io.jl"), +jl_symbol("eltype"), +jl_symbol("name"), +jl_symbol("parent"), jl_symbol("!=="), -jl_symbol("merge"), -jl_symbol("BoundsError"), -jl_symbol("broadcasted"), -jl_symbol("Cint"), -jl_symbol("min"), -jl_symbol("libblastrampoline"), jl_symbol("iszero"), +jl_symbol("min"), +jl_symbol("DimensionMismatch"), jl_symbol("refvalue.jl"), -jl_symbol("stride"), -jl_symbol("error"), -jl_symbol("ncodeunits"), -jl_symbol("LinearIndices"), -jl_symbol("Clong"), -jl_symbol("pair.jl"), -jl_symbol("_growend!"), -jl_symbol("char.jl"), -jl_symbol("copyto!"), -jl_symbol("get"), -jl_symbol("tail"), -jl_symbol("real"), jl_symbol("Union"), -jl_symbol("multidimensional.jl"), -jl_symbol("enter"), -jl_symbol("leave"), +jl_symbol("BlasInt"), +jl_symbol("unsafe_load"), +jl_symbol("indices.jl"), +jl_symbol("x"), +jl_symbol("require_one_based_indexing"), +jl_symbol("namedtuple.jl"), +jl_symbol("tail"), +jl_symbol("Float64"), +jl_symbol("head"), +jl_symbol("Cvoid"), +jl_symbol("copy"), +jl_symbol("libblastrampoline"), +jl_symbol("get"), +jl_symbol("neg_int"), +jl_symbol("stop"), +jl_symbol("zero"), jl_symbol("add_ptr"), -jl_symbol("chkstride1"), +jl_symbol("toUInt32"), +jl_symbol("ptr"), +jl_symbol("char.jl"), +jl_symbol("trunc"), +jl_symbol("not_atomic"), +jl_symbol("enter"), +jl_symbol("Pair"), +jl_symbol("jl_value_ptr"), jl_symbol("Expr"), -jl_symbol("write"), -jl_symbol("broadcast.jl"), +jl_symbol("broadcasted"), +jl_symbol("pointerref"), +jl_symbol("multidimensional.jl"), +jl_symbol("Generator"), +jl_symbol("leave"), +jl_symbol("memoryref"), jl_symbol("show.jl"), +jl_symbol("pointer_from_objref"), +jl_symbol("memoryrefget"), +jl_symbol("reduce.jl"), +jl_symbol("stride"), +jl_symbol("pair.jl"), +jl_symbol("_string"), +jl_symbol("cmem.jl"), +jl_symbol("generator.jl"), +jl_symbol("broadcast.jl"), jl_symbol("none"), -jl_symbol("Generator"), +jl_symbol("copyto!"), +jl_symbol("chkstride1"), +jl_symbol("value"), +jl_symbol("write"), +jl_symbol("identity"), +jl_symbol("real"), +jl_symbol("start"), +jl_symbol("Cint"), +jl_symbol("fill!"), +jl_symbol("checkindex"), +jl_symbol("keys"), +jl_symbol("BoundsError"), +jl_symbol("vals"), +jl_symbol("Symbol"), +jl_symbol("strings/util.jl"), jl_symbol("Int32"), -jl_symbol("materialize"), +jl_symbol("ht_keyindex"), +jl_symbol("io"), +jl_symbol("~"), +jl_symbol("AssertionError"), +jl_symbol("abstractdict.jl"), jl_symbol("show"), -jl_symbol("lock"), -jl_symbol("unsafe_load"), -jl_symbol("gmp.jl"), jl_symbol("mpfr.jl"), -jl_symbol("Symbol"), -jl_symbol("Pair"), -jl_symbol("resize!"), -jl_symbol("neg_int"), -jl_symbol("strings/substring.jl"), -jl_symbol("AssertionError"), -jl_symbol("identity"), -jl_symbol("one"), -jl_symbol("reduce.jl"), -jl_symbol("libcholmod"), jl_symbol("isless"), +jl_symbol("args"), +jl_symbol("lock"), jl_symbol("reducedim.jl"), +jl_symbol("gmp.jl"), +jl_symbol("offset"), +jl_symbol("resize!"), +jl_symbol("throw_boundserror"), +jl_symbol("Clong"), +jl_symbol("_call_latest"), +jl_symbol("argtail"), +jl_symbol("compiler/ssair/ir.jl"), +jl_symbol("sub_ptr"), +jl_symbol("materialize"), jl_symbol("checksquare"), -jl_symbol("sort.jl"), -jl_symbol("generator.jl"), -jl_symbol("pointer_from_objref"), -jl_symbol("Float32"), -jl_symbol("chklapackerror"), -jl_symbol("parent"), -jl_symbol("task.jl"), +jl_symbol("LinearIndices"), +jl_symbol("ule_int"), +jl_symbol("dict"), jl_symbol("div"), -jl_symbol("cholmod_common"), -jl_symbol("ht_keyindex"), -jl_symbol("pop_exception"), -jl_symbol("c.jl"), -jl_symbol("firstindex"), -jl_symbol("some.jl"), -jl_symbol("iobuffer.jl"), -jl_symbol("sub_ptr"), -jl_symbol("vect"), -jl_symbol("unsafe_string"), -jl_symbol("llvmcall"), -jl_symbol("checkindex"), -jl_symbol("_call_latest"), +jl_symbol("chklapackerror"), +jl_symbol("count"), +jl_symbol("Float32"), +jl_symbol("genericmemory.jl"), +jl_symbol("print_to_string"), jl_symbol("rethrow"), -jl_symbol("pointerref"), +jl_symbol("sort.jl"), +jl_symbol("boundscheck"), jl_symbol("println"), -jl_symbol("keys"), -jl_symbol("RefValue"), +jl_symbol("loading.jl"), +jl_symbol("collect"), +jl_symbol("ashr_int"), jl_symbol("_expr"), -jl_symbol("toUInt32"), -jl_symbol("ismissing"), -jl_symbol("throw_boundserror"), -jl_symbol("IteratorSize"), -jl_symbol("iddict.jl"), +jl_symbol("iobuffer.jl"), +jl_symbol("DataType"), +jl_symbol("Dict"), +jl_symbol("unsafe_string"), +jl_symbol("RefValue"), +jl_symbol("step"), jl_symbol("to_shape"), -jl_symbol("Csize_t"), -jl_symbol("~"), -jl_symbol("argtail"), -jl_symbol("include"), -jl_symbol("set.jl"), -jl_symbol("isequal"), +jl_symbol("pop_exception"), +jl_symbol("Memory"), +jl_symbol("KeyError"), +jl_symbol("chunks"), jl_symbol("refpointer.jl"), -jl_symbol("=>"), -jl_symbol("Val"), -jl_symbol("Base"), +jl_symbol("llvmcall"), +jl_symbol("c.jl"), +jl_symbol("set.jl"), +jl_symbol("abs"), +jl_symbol("checked_trunc_uint"), +jl_symbol("Type"), jl_symbol("%"), -jl_symbol("collect"), -jl_symbol("Type##kw"), -jl_symbol("typemax"), -jl_symbol("fill!"), -jl_symbol("ule_int"), -jl_symbol("atomics.jl"), -jl_symbol("libgit2"), +jl_symbol("len"), jl_symbol("BigFloat"), -jl_symbol("ashr_int"), -jl_symbol("boundscheck"), -jl_symbol("abs"), -jl_symbol("^"), -jl_symbol("ensure_initialized"), -jl_symbol("_array_for"), -jl_symbol("strings/util.jl"), -jl_symbol("Dict"), +jl_symbol("isequal"), +jl_symbol("vect"), +jl_symbol("sprint"), +jl_symbol("mode"), +jl_symbol("expr.jl"), jl_symbol("Nothing"), -jl_symbol("compiler/ssair/ir.jl"), +jl_symbol("Val"), +jl_symbol("IteratorSize"), +jl_symbol("=>"), +jl_symbol("haskey"), +jl_symbol("iddict.jl"), jl_symbol("unsafe_write"), -jl_symbol("util.jl"), +jl_symbol("val"), +jl_symbol("flags"), +jl_symbol("task.jl"), +jl_symbol("UnionAll"), +jl_symbol("memset"), +jl_symbol("xor"), +jl_symbol("jl_alloc_genericmemory"), +jl_symbol("uplo"), jl_symbol("toInt32"), -jl_symbol("loading.jl"), -jl_symbol("value"), -jl_symbol("expr.jl"), -jl_symbol("print_to_string"), +jl_symbol("Base"), +jl_symbol("atomics.jl"), +jl_symbol("uuid"), +jl_symbol("one"), +jl_symbol("math.jl"), +jl_symbol("position"), +jl_symbol("typemax"), +jl_symbol("all"), +jl_symbol("error.jl"), +jl_symbol("path.jl"), +jl_symbol("^"), +jl_symbol("nextind"), +jl_symbol("include"), jl_symbol("the_exception"), -jl_symbol("nonzeros"), -jl_symbol("<:"), -jl_symbol("KeyError"), -jl_symbol("xor"), -jl_symbol("logging.jl"), +jl_symbol("ensure_initialized"), +jl_symbol("Const"), +jl_symbol("UInt128"), +jl_symbol("codeunit"), jl_symbol("stat.jl"), -jl_symbol("close"), -jl_symbol("adjoint"), -jl_symbol("meta"), -jl_symbol("path.jl"), -jl_symbol("round"), -jl_symbol("Cstring"), -jl_symbol("SizeUnknown"), -jl_symbol("esc"), -jl_symbol("missing.jl"), +jl_symbol("gcutils.jl"), +jl_symbol("UndefRefError"), +jl_symbol("diag"), jl_symbol("throw_undef_if_not"), -jl_symbol("error.jl"), -jl_symbol("Type"), -jl_symbol("mul!"), -jl_symbol("math.jl"), -jl_symbol("unsafe_trunc"), jl_symbol("missing"), -jl_symbol("subarray.jl"), -jl_symbol("noinline"), jl_symbol("isnan"), -jl_symbol("ldiv!"), -jl_symbol("DataType"), -jl_symbol("codeunit"), -jl_symbol("condition.jl"), -jl_symbol("step"), -jl_symbol("copyast"), -jl_symbol("bitset.jl"), -jl_symbol("float"), +jl_symbol("Enums.jl"), +jl_symbol("logging.jl"), +jl_symbol("_deleteend!"), +jl_symbol("indices"), +jl_symbol("compiler/utilities.jl"), +jl_symbol("Pairs"), +jl_symbol("<:"), +jl_symbol("compiler/tfuncs.jl"), +jl_symbol("close"), +jl_symbol("subarray.jl"), jl_symbol("fastmath.jl"), +jl_symbol("invokelatest"), +jl_symbol("jl_array_del_end"), jl_symbol("_mod64"), -jl_symbol("_div64"), -jl_symbol("all"), -jl_symbol("parse"), -jl_symbol("joinpath"), -jl_symbol("nextind"), +jl_symbol("parameters"), +jl_symbol("monotonic"), jl_symbol("regex.jl"), -jl_symbol("Enums.jl"), -jl_symbol("promote_type"), -jl_symbol("Cdouble"), -jl_symbol("ComplexF32"), -jl_symbol("read"), -jl_symbol("intfuncs.jl"), -jl_symbol("Complex"), -jl_symbol("_deleteend!"), -jl_symbol("stat"), -jl_symbol("UnionAll"), -jl_symbol("special/trig.jl"), -jl_symbol("UInt128"), -jl_symbol("_copyto_impl!"), -jl_symbol("stream.jl"), -jl_symbol("lmul!"), -jl_symbol("repr"), -jl_symbol("promote_rule"), -jl_symbol("xor_int"), -jl_symbol("complex.jl"), -jl_symbol("transpose"), -jl_symbol(">>>"), -jl_symbol("cholmod_sparse"), -jl_symbol("filemode"), -jl_symbol("ComplexF64"), -jl_symbol("SparseMatrixCSC"), -jl_symbol("view"), -jl_symbol("GitError"), -jl_symbol("zeros"), -jl_symbol("InexactError"), diff --git a/src/datatype.c b/src/datatype.c index 905959fb80e0a..87f54006cd37f 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -49,7 +49,7 @@ JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *mo mt->name = jl_demangle_typename(name); mt->module = module; jl_atomic_store_relaxed(&mt->defs, jl_nothing); - jl_atomic_store_relaxed(&mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_relaxed(&mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_atomic_store_relaxed(&mt->cache, jl_nothing); jl_atomic_store_relaxed(&mt->max_args, 0); mt->backedges = NULL; @@ -137,7 +137,7 @@ static uint32_t _hash_layout_djb2(uintptr_t _layout, void *unused) JL_NOTSAFEPOI size_t fields_size = layout->nfields * jl_fielddesc_size(layout->fielddesc_type); const char *pointers = jl_dt_layout_ptrs(layout); assert(pointers); - size_t pointers_size = (layout->npointers << layout->fielddesc_type); + size_t pointers_size = layout->first_ptr < 0 ? 0 : (layout->npointers << layout->fielddesc_type); uint_t hash = 5381; hash = _hash_djb2(hash, (char *)layout, own_size); @@ -160,7 +160,7 @@ static int layout_eq(void *_l1, void *_l2, void *unused) JL_NOTSAFEPOINT return 0; const char *p1 = jl_dt_layout_ptrs(l1); const char *p2 = jl_dt_layout_ptrs(l2); - size_t pointers_size = (l1->npointers << l1->fielddesc_type); + size_t pointers_size = l1->first_ptr < 0 ? 0 : (l1->npointers << l1->fielddesc_type); if (memcmp(p1, p2, pointers_size)) return 0; return 1; @@ -179,6 +179,7 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, uint32_t npointers, uint32_t alignment, int haspadding, + int arrayelem, jl_fielddesc32_t desc[], uint32_t pointers[]) JL_NOTSAFEPOINT { @@ -186,32 +187,34 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, // compute the smallest fielddesc type that can hold the layout description int fielddesc_type = 0; + uint32_t max_size = 0; + uint32_t max_offset = 0; if (nfields > 0) { - uint32_t max_size = 0; - uint32_t max_offset = desc[nfields - 1].offset; - if (npointers > 0 && pointers[npointers - 1] > max_offset) - max_offset = pointers[npointers - 1]; + max_offset = desc[nfields - 1].offset; for (size_t i = 0; i < nfields; i++) { if (desc[i].size > max_size) max_size = desc[i].size; } - jl_fielddesc8_t maxdesc8 = { 0, max_size, max_offset }; - jl_fielddesc16_t maxdesc16 = { 0, max_size, max_offset }; - jl_fielddesc32_t maxdesc32 = { 0, max_size, max_offset }; - if (maxdesc8.size != max_size || maxdesc8.offset != max_offset) { - fielddesc_type = 1; - if (maxdesc16.size != max_size || maxdesc16.offset != max_offset) { - fielddesc_type = 2; - if (maxdesc32.size != max_size || maxdesc32.offset != max_offset) { - assert(0); // should have been verified by caller - } + } + if (npointers > 0 && pointers[npointers - 1] > max_offset) + max_offset = pointers[npointers - 1]; + jl_fielddesc8_t maxdesc8 = { 0, max_size, max_offset }; + jl_fielddesc16_t maxdesc16 = { 0, max_size, max_offset }; + jl_fielddesc32_t maxdesc32 = { 0, max_size, max_offset }; + if (maxdesc8.size != max_size || maxdesc8.offset != max_offset) { + fielddesc_type = 1; + if (maxdesc16.size != max_size || maxdesc16.offset != max_offset) { + fielddesc_type = 2; + if (maxdesc32.size != max_size || maxdesc32.offset != max_offset) { + assert(0); // should have been verified by caller } } } + int32_t first_ptr = (npointers > 0 ? (int32_t)pointers[0] : -1); // allocate a new descriptor, on the stack if possible. size_t fields_size = nfields * jl_fielddesc_size(fielddesc_type); - size_t pointers_size = (npointers << fielddesc_type); + size_t pointers_size = first_ptr < 0 ? 0 : (npointers << fielddesc_type); size_t flddesc_sz = sizeof(jl_datatype_layout_t) + fields_size + pointers_size; int should_malloc = flddesc_sz >= jl_page_size; jl_datatype_layout_t *mallocmem = (jl_datatype_layout_t *)(should_malloc ? malloc(flddesc_sz) : NULL); @@ -223,9 +226,11 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, flddesc->alignment = alignment; flddesc->haspadding = haspadding; flddesc->fielddesc_type = fielddesc_type; + flddesc->arrayelem_isboxed = arrayelem == 1; + flddesc->arrayelem_isunion = arrayelem == 2; flddesc->padding = 0; flddesc->npointers = npointers; - flddesc->first_ptr = (npointers > 0 ? pointers[0] : -1); + flddesc->first_ptr = first_ptr; // fill out the fields of the new descriptor jl_fielddesc8_t *desc8 = (jl_fielddesc8_t *)jl_dt_layout_fields(flddesc); @@ -248,18 +253,20 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, desc32[i].isptr = desc[i].isptr; } } - uint8_t *ptrs8 = (uint8_t *)jl_dt_layout_ptrs(flddesc); - uint16_t *ptrs16 = (uint16_t *)jl_dt_layout_ptrs(flddesc); - uint32_t *ptrs32 = (uint32_t *)jl_dt_layout_ptrs(flddesc); - for (size_t i = 0; i < npointers; i++) { - if (fielddesc_type == 0) { - ptrs8[i] = pointers[i]; - } - else if (fielddesc_type == 1) { - ptrs16[i] = pointers[i]; - } - else { - ptrs32[i] = pointers[i]; + if (first_ptr >= 0) { + uint8_t *ptrs8 = (uint8_t *)jl_dt_layout_ptrs(flddesc); + uint16_t *ptrs16 = (uint16_t *)jl_dt_layout_ptrs(flddesc); + uint32_t *ptrs32 = (uint32_t *)jl_dt_layout_ptrs(flddesc); + for (size_t i = 0; i < npointers; i++) { + if (fielddesc_type == 0) { + ptrs8[i] = pointers[i]; + } + else if (fielddesc_type == 1) { + ptrs16[i] = pointers[i]; + } + else { + ptrs32[i] = pointers[i]; + } } } @@ -318,23 +325,23 @@ unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) STATIC_INLINE int jl_is_datatype_make_singleton(jl_datatype_t *d) JL_NOTSAFEPOINT { - return (!d->name->abstract && jl_datatype_size(d) == 0 && d != jl_symbol_type && d->name != jl_array_typename && - d->isconcretetype && !d->name->mutabl); + return d->isconcretetype && jl_datatype_size(d) == 0 && d->layout->npointers == 0 && !d->name->mutabl; // implies jl_is_layout_opaque } STATIC_INLINE void jl_maybe_allocate_singleton_instance(jl_datatype_t *st) JL_NOTSAFEPOINT { + // It's possible for st to already have an ->instance if it was redefined + if (st->instance) + return; if (jl_is_datatype_make_singleton(st)) { - // It's possible for st to already have an ->instance if it was redefined - if (!st->instance) - st->instance = jl_gc_permobj(0, st); + st->instance = jl_gc_permobj(0, st); } } // return whether all concrete subtypes of this type have the same layout int jl_struct_try_layout(jl_datatype_t *dt) { - if (dt->layout) + if (dt->layout || jl_is_genericmemory_type(dt)) return 1; else if (!jl_has_fixed_layout(dt)) return 0; @@ -478,6 +485,82 @@ static int is_type_identityfree(jl_value_t *t) return 0; } +// make a copy of the layout of st, but with nfields=0 +void jl_get_genericmemory_layout(jl_datatype_t *st) +{ + jl_value_t *isatomic = jl_tparam0(st); (void)isatomic; // TODO + jl_value_t *eltype = jl_tparam1(st); + if (!jl_is_typevar(eltype) && !jl_is_type(eltype)) { + // this is expected to have a layout, but since it is not constructable, we don't care too much what it is + static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), 0, 0}; + st->layout = &opaque_ptr_layout; + st->has_concrete_subtype = 0; + return; + } + + size_t elsz = 0, al = 0; + int isunboxed = jl_islayout_inline(eltype, &elsz, &al); + int isunion = isunboxed && jl_is_uniontype(eltype); + int haspadding = 1; // we may want to eventually actually compute this + int nfields = 0; // aka jl_is_layout_opaque + int npointers = 1; + int zi; + uint32_t first_ptr = -1; + uint32_t *pointers = &first_ptr; + + if (isunboxed) { + elsz = LLT_ALIGN(elsz, al); + if (isunion) { + zi = 1; + } + else { + assert(jl_is_datatype(eltype)); + zi = ((jl_datatype_t*)eltype)->zeroinit; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)eltype)->layout; + if (layout->first_ptr >= 0) { + first_ptr = layout->first_ptr; + npointers = layout->npointers; + if (layout->fielddesc_type == 2) { + pointers = (uint32_t*)jl_dt_layout_ptrs(layout); + } + else { + pointers = (uint32_t*)alloca(npointers * sizeof(uint32_t)); + for (int j = 0; j < npointers; j++) { + pointers[j] = jl_ptr_offset((jl_datatype_t*)eltype, j); + } + } + } + } + } + else { + elsz = sizeof(void*); + al = elsz; + zi = 1; + } + + int arrayelem; + if (!isunboxed) + arrayelem = 1; + else if (isunion) + arrayelem = 2; + else + arrayelem = 0; + assert(!st->layout); + st->layout = jl_get_layout(elsz, nfields, npointers, al, haspadding, arrayelem, NULL, pointers); + st->zeroinit = zi; + //st->has_concrete_subtype = 1; + //st->isbitstype = 0; + //st->ismutationfree = 0; + //st->isidentityfree = 0; + + jl_genericmemory_t *zeroinst = (jl_genericmemory_t*)jl_gc_permobj(sizeof(jl_genericmemory_t) + (elsz ? elsz : isunion), st); + zeroinst->length = 0; + zeroinst->data = (char*)(zeroinst + 1); + memset(zeroinst->data, 0, elsz ? elsz : isunion); + assert(!st->instance); + st->instance = (jl_value_t*)zeroinst; +} + void jl_compute_field_offsets(jl_datatype_t *st) { const uint64_t max_offset = (((uint64_t)1) << 32) - 1; @@ -492,6 +575,10 @@ void jl_compute_field_offsets(jl_datatype_t *st) st->zeroinit = 0; st->has_concrete_subtype = 1; } + if (st->name == jl_genericmemory_typename) { + jl_get_genericmemory_layout(st); + return; + } int isbitstype = st->isconcretetype && st->name->mayinlinealloc; int ismutationfree = !w->layout || !jl_is_layout_opaque(w->layout); int isidentityfree = !st->name->mutabl; @@ -501,7 +588,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) st->layout = w->layout; st->zeroinit = w->zeroinit; st->has_concrete_subtype = w->has_concrete_subtype; - if (!jl_is_layout_opaque(st->layout)) { // e.g. jl_array_typename + if (!jl_is_layout_opaque(st->layout)) { // e.g. jl_simplevector_type st->isbitstype = isbitstype && st->layout->npointers == 0; jl_maybe_allocate_singleton_instance(st); } @@ -518,7 +605,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) st->layout = &opaque_byte_layout; return; } - else if (st == jl_simplevector_type || st == jl_module_type || st->name == jl_array_typename) { + else if (st == jl_simplevector_type || st == jl_module_type) { static const jl_datatype_layout_t opaque_ptr_layout = {0, 0, 1, -1, sizeof(void*), 0, 0}; st->layout = &opaque_ptr_layout; return; @@ -670,7 +757,7 @@ void jl_compute_field_offsets(jl_datatype_t *st) } } assert(ptr_i == npointers); - st->layout = jl_get_layout(sz, nfields, npointers, alignm, haspadding, desc, pointers); + st->layout = jl_get_layout(sz, nfields, npointers, alignm, haspadding, 0, desc, pointers); if (should_malloc) { free(desc); if (npointers) @@ -710,7 +797,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_datatype( jl_typename_t *tn = NULL; JL_GC_PUSH2(&t, &tn); - assert(parameters); + assert(parameters && fnames); // init enough before possibly calling jl_new_typename_in t = jl_new_uninitialized_datatype(); @@ -832,7 +919,7 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t * bt->ismutationfree = 1; bt->isidentityfree = 1; bt->isbitstype = (parameters == jl_emptysvec); - bt->layout = jl_get_layout(nbytes, 0, 0, alignm, 0, NULL, NULL); + bt->layout = jl_get_layout(nbytes, 0, 0, alignm, 0, 0, NULL, NULL); bt->instance = NULL; return bt; } @@ -1227,18 +1314,6 @@ JL_DLLEXPORT jl_value_t *jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_datatype_t return y; } -// used by boot.jl -JL_DLLEXPORT jl_value_t *jl_typemax_uint(jl_datatype_t *bt) -{ - uint64_t data = 0xffffffffffffffffULL; - jl_task_t *ct = jl_current_task; - jl_value_t *v = jl_gc_alloc(ct->ptls, sizeof(size_t), bt); - if (bt->smalltag) - jl_set_typetagof(v, bt->smalltag, 0); - memcpy(v, &data, sizeof(size_t)); - return v; -} - #define PERMBOXN_FUNC(nb) \ jl_value_t *jl_permbox##nb(jl_datatype_t *t, uintptr_t tag, uint##nb##_t x) \ { /* n.b. t must be a concrete isbits datatype of the right size */ \ @@ -1359,6 +1434,9 @@ void jl_init_int32_int64_cache(void) for(i=0; i < NBOX_C; i++) { boxed_int32_cache[i] = jl_permbox32(jl_int32_type, jl_int32_tag, i-NBOX_C/2); boxed_int64_cache[i] = jl_permbox64(jl_int64_type, jl_int64_tag, i-NBOX_C/2); + boxed_uint16_cache[i] = jl_permbox16(jl_uint16_type, jl_uint16_tag, i); + boxed_uint64_cache[i] = jl_permbox64(jl_uint64_type, jl_uint64_tag, i); + boxed_uint32_cache[i] = jl_permbox32(jl_uint32_type, jl_uint32_tag, i); #ifdef _P64 boxed_ssavalue_cache[i] = jl_permbox64(jl_ssavalue_type, 0, i); boxed_slotnumber_cache[i] = jl_permbox64(jl_slotnumber_type, 0, i); @@ -1383,9 +1461,6 @@ void jl_init_box_caches(void) } for (i = 0; i < NBOX_C; i++) { boxed_int16_cache[i] = jl_permbox16(jl_int16_type, jl_int16_tag, i-NBOX_C/2); - boxed_uint16_cache[i] = jl_permbox16(jl_uint16_type, jl_uint16_tag, i); - boxed_uint32_cache[i] = jl_permbox32(jl_uint32_type, jl_uint32_tag, i); - boxed_uint64_cache[i] = jl_permbox64(jl_uint64_type, jl_uint64_tag, i); } } @@ -1401,10 +1476,11 @@ JL_DLLEXPORT jl_value_t *jl_box_bool(int8_t x) JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) { jl_task_t *ct = jl_current_task; - if (type->instance != NULL) return type->instance; - if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL) { + if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL || jl_is_layout_opaque(type->layout)) { jl_type_error("new", (jl_value_t*)jl_datatype_type, (jl_value_t*)type); } + if (type->instance != NULL) + return type->instance; va_list args; size_t i, nf = jl_datatype_nfields(type); va_start(args, type); @@ -1424,7 +1500,7 @@ JL_DLLEXPORT jl_value_t *jl_new_struct(jl_datatype_t *type, ...) JL_DLLEXPORT jl_value_t *jl_new_structv(jl_datatype_t *type, jl_value_t **args, uint32_t na) { jl_task_t *ct = jl_current_task; - if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL) { + if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL || jl_is_layout_opaque(type->layout)) { jl_type_error("new", (jl_value_t*)jl_datatype_type, (jl_value_t*)type); } size_t nf = jl_datatype_nfields(type); @@ -1463,7 +1539,7 @@ JL_DLLEXPORT jl_value_t *jl_new_structt(jl_datatype_t *type, jl_value_t *tup) jl_task_t *ct = jl_current_task; if (!jl_is_tuple(tup)) jl_type_error("new", (jl_value_t*)jl_tuple_type, tup); - if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL) + if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL || jl_is_layout_opaque(type->layout)) jl_type_error("new", (jl_value_t *)jl_datatype_type, (jl_value_t *)type); size_t nargs = jl_nfields(tup); size_t nf = jl_datatype_nfields(type); @@ -1510,10 +1586,11 @@ JL_DLLEXPORT jl_value_t *jl_new_structt(jl_datatype_t *type, jl_value_t *tup) JL_DLLEXPORT jl_value_t *jl_new_struct_uninit(jl_datatype_t *type) { jl_task_t *ct = jl_current_task; - if (type->instance != NULL) return type->instance; - if (!jl_is_datatype(type) || type->layout == NULL) { + if (!jl_is_datatype(type) || !type->isconcretetype || type->layout == NULL || jl_is_layout_opaque(type->layout)) { jl_type_error("new", (jl_value_t*)jl_datatype_type, (jl_value_t*)type); } + if (type->instance != NULL) + return type->instance; size_t size = jl_datatype_size(type); jl_value_t *jv = jl_gc_alloc(ct->ptls, size, type); if (type->smalltag) // TODO: do we need this? @@ -1722,6 +1799,7 @@ jl_value_t *swap_nth_field(jl_datatype_t *st, jl_value_t *v, size_t i, jl_value_ } else { hasptr = ((jl_datatype_t*)ty)->layout->npointers > 0; + r = NULL; } size_t fsz = jl_datatype_size((jl_datatype_t*)rty); // need to shrink-wrap the final copy int needlock = (isatomic && fsz > MAX_ATOMIC_SIZE); diff --git a/src/dlload.c b/src/dlload.c index 8124605880b5e..f2ba02e14c5d1 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -312,7 +312,7 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, jl_array_t *DL_LOAD_PATH = (jl_array_t*)(b ? jl_atomic_load_relaxed(&b->value) : NULL); if (DL_LOAD_PATH != NULL) { size_t j; - for (j = 0; j < jl_array_len(DL_LOAD_PATH); j++) { + for (j = 0; j < jl_array_nrows(DL_LOAD_PATH); j++) { char *dl_path = jl_string_data(jl_array_ptr_data(DL_LOAD_PATH)[j]); size_t len = strlen(dl_path); if (len == 0) diff --git a/src/gc-debug.c b/src/gc-debug.c index cb5ba66b9cce6..c20df84a5e54f 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -616,8 +616,7 @@ void objprofile_count(void *ty, int old, int sz) ty = (void*)jl_buff_tag; } else if (ty != (void*)jl_buff_tag && ty != jl_malloc_tag && - jl_typeof(ty) == (jl_value_t*)jl_datatype_type && - ((jl_datatype_t*)ty)->instance) { + jl_is_datatype(ty) && jl_is_datatype_singleton((jl_datatype_t*)ty)) { ty = jl_singleton_tag; } void **bp = ptrhash_bp(&obj_counts[old], ty); @@ -1203,12 +1202,6 @@ int gc_slot_to_arrayidx(void *obj, void *_slot) JL_NOTSAFEPOINT start = (char*)jl_svec_data(obj); len = jl_svec_len(obj); } - else if (vt->name == jl_array_typename) { - jl_array_t *a = (jl_array_t*)obj; - start = (char*)a->data; - len = jl_array_len(a); - elsize = a->elsize; - } if (slot < start || slot >= start + elsize * len) return -1; return (slot - start) / elsize; diff --git a/src/gc-stacks.c b/src/gc-stacks.c index b35c1722c82ff..cf2f61365532b 100644 --- a/src/gc-stacks.c +++ b/src/gc-stacks.c @@ -273,12 +273,12 @@ JL_DLLEXPORT jl_array_t *jl_live_tasks(void) l = live_tasks->len; void **lst = live_tasks->items; j = 0; - ((void**)jl_array_data(a))[j++] = ptls->root_task; + jl_array_data(a, void*)[j++] = ptls->root_task; for (i = 0; i < l; i++) { if (((jl_task_t*)lst[i])->stkbuf != NULL) - ((void**)jl_array_data(a))[j++] = lst[i]; + jl_array_data(a, void*)[j++] = lst[i]; } - l = jl_array_len(a); + l = jl_array_nrows(a); if (j < l) { JL_GC_PUSH1(&a); jl_array_del_end(a, l - j); diff --git a/src/gc.c b/src/gc.c index cf04641d1fb69..48eefe99a5fd5 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1124,10 +1124,9 @@ static void sweep_big(jl_ptls_t ptls, int sweep_full) JL_NOTSAFEPOINT gc_time_big_end(); } -// tracking Arrays with malloc'd storage +// tracking Memorys with malloc'd storage -void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT -{ +void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, int isaligned){ // This is **NOT** a GC safe point. mallocarray_t *ma; if (ptls->heap.mafreelist == NULL) { @@ -1137,11 +1136,12 @@ void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT ma = ptls->heap.mafreelist; ptls->heap.mafreelist = ma->next; } - ma->a = a; + ma->a = (jl_value_t*)((uintptr_t)m | !!isaligned); ma->next = ptls->heap.mallocarrays; ptls->heap.mallocarrays = ma; } + void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT { jl_ptls_t ptls = jl_current_task->ptls; @@ -1210,33 +1210,32 @@ void jl_gc_reset_alloc_count(void) JL_NOTSAFEPOINT reset_thread_gc_counts(); } -size_t jl_array_nbytes(jl_array_t *a) JL_NOTSAFEPOINT +size_t jl_genericmemory_nbytes(jl_genericmemory_t *m) JL_NOTSAFEPOINT { - size_t sz = 0; - int isbitsunion = jl_array_isbitsunion(a); - if (jl_array_ndims(a) == 1) - sz = a->elsize * a->maxsize + ((a->elsize == 1 && !isbitsunion) ? 1 : 0); - else - sz = a->elsize * jl_array_len(a); - if (isbitsunion) + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + size_t sz = layout->size * m->length; + if (layout->arrayelem_isunion) // account for isbits Union array selector bytes - sz += jl_array_len(a); + sz += m->length; return sz; } -static void jl_gc_free_array(jl_array_t *a) JL_NOTSAFEPOINT + +static void jl_gc_free_array(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT { - if (a->flags.how == 2) { - char *d = (char*)a->data - a->offset*a->elsize; - if (a->flags.isaligned) - jl_free_aligned(d); - else - free(d); - jl_atomic_store_relaxed(&gc_heap_stats.heap_size, - jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_array_nbytes(a)); - gc_num.freed += jl_array_nbytes(a); - gc_num.freecall++; - } + assert(jl_is_genericmemory(v)); + jl_genericmemory_t *m = (jl_genericmemory_t*)v; + assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2); + char *d = (char*)m->data; + if (isaligned) + jl_free_aligned(d); + else + free(d); + jl_atomic_store_relaxed(&gc_heap_stats.heap_size, + jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_genericmemory_nbytes(m)); + gc_num.freed += jl_genericmemory_nbytes(m); + gc_num.freecall++; + gc_num.freecall++; } static void sweep_malloced_arrays(void) JL_NOTSAFEPOINT @@ -1250,14 +1249,15 @@ static void sweep_malloced_arrays(void) JL_NOTSAFEPOINT mallocarray_t **pma = &ptls2->heap.mallocarrays; while (ma != NULL) { mallocarray_t *nxt = ma->next; - int bits = jl_astaggedvalue(ma->a)->bits.gc; + jl_value_t *a = (jl_value_t*)((uintptr_t)ma->a & ~1); + int bits = jl_astaggedvalue(a)->bits.gc; if (gc_marked(bits)) { pma = &ma->next; } else { *pma = nxt; - assert(ma->a->flags.how == 2); - jl_gc_free_array(ma->a); + int isaligned = (uintptr_t)ma->a & 1; + jl_gc_free_array(a, isaligned); ma->next = ptls2->heap.mafreelist; ptls2->heap.mafreelist = ma; } @@ -1986,6 +1986,7 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_value_t *new_obj; // Decide whether need to chunk objary + assert(step > 0); (void)jl_assume(step > 0); if ((nptr & 0x2) == 0x2) { // pre-scan this object: most of this object should be old, so look for @@ -2045,13 +2046,13 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v // Mark array with 8bit field descriptors STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_value_t **ary8_begin, - jl_value_t **ary8_end, uint8_t *elem_begin, uint8_t *elem_end, + jl_value_t **ary8_end, uint8_t *elem_begin, uint8_t *elem_end, uintptr_t elsize, uintptr_t nptr) JL_NOTSAFEPOINT { jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_value_t *new_obj; - size_t elsize = ((jl_array_t *)ary8_parent)->elsize / sizeof(jl_value_t *); assert(elsize > 0); + (void)jl_assume(elsize > 0); // Decide whether need to chunk objary if ((nptr & 0x2) == 0x2) { // pre-scan this object: most of this object should be old, so look for @@ -2090,7 +2091,7 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va // case 2: lowest two bits of `nptr` are already set to 0x3, so won't change after // scanning the array elements if ((nptr & 0x2) != 0x2 || (nptr & 0x3) == 0x3) { - jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, scan_end, ary8_end, elem_begin, elem_end, 0, nptr}; + jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, scan_end, ary8_end, elem_begin, elem_end, elsize, nptr}; gc_chunkqueue_push(mq, &c); pushed_chunk = 1; } @@ -2109,7 +2110,7 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va } if (too_big) { if (!pushed_chunk) { - jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, scan_end, ary8_end, elem_begin, elem_end, 0, nptr}; + jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, scan_end, ary8_end, elem_begin, elem_end, elsize, nptr}; gc_chunkqueue_push(mq, &c); } } @@ -2120,13 +2121,13 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va // Mark array with 16bit field descriptors STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_value_t **ary16_begin, - jl_value_t **ary16_end, uint16_t *elem_begin, uint16_t *elem_end, + jl_value_t **ary16_end, uint16_t *elem_begin, uint16_t *elem_end, size_t elsize, uintptr_t nptr) JL_NOTSAFEPOINT { jl_gc_markqueue_t *mq = &ptls->mark_queue; jl_value_t *new_obj; - size_t elsize = ((jl_array_t *)ary16_parent)->elsize / sizeof(jl_value_t *); assert(elsize > 0); + (void)jl_assume(elsize > 0); // Decide whether need to chunk objary if ((nptr & 0x2) == 0x2) { // pre-scan this object: most of this object should be old, so look for @@ -2203,8 +2204,8 @@ STATIC_INLINE void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_ch jl_value_t **obj_end = c->end; uint32_t step = c->step; uintptr_t nptr = c->nptr; - gc_mark_objarray(ptls, obj_parent, obj_begin, obj_end, step, - nptr); + gc_mark_objarray(ptls, obj_parent, obj_begin, obj_end, + step, nptr); break; } case GC_ary8_chunk: { @@ -2213,9 +2214,10 @@ STATIC_INLINE void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_ch jl_value_t **ary8_end = c->end; uint8_t *elem_begin = (uint8_t *)c->elem_begin; uint8_t *elem_end = (uint8_t *)c->elem_end; + size_t elsize = c->step; uintptr_t nptr = c->nptr; gc_mark_array8(ptls, ary8_parent, ary8_begin, ary8_end, elem_begin, elem_end, - nptr); + elsize, nptr); break; } case GC_ary16_chunk: { @@ -2224,9 +2226,10 @@ STATIC_INLINE void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_ch jl_value_t **ary16_end = c->end; uint16_t *elem_begin = (uint16_t *)c->elem_begin; uint16_t *elem_end = (uint16_t *)c->elem_end; + size_t elsize = c->step; uintptr_t nptr = c->nptr; gc_mark_array16(ptls, ary16_parent, ary16_begin, ary16_end, elem_begin, elem_end, - nptr); + elsize, nptr); break; } case GC_finlist_chunk: { @@ -2551,70 +2554,76 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ jl_datatype_t *vt = (jl_datatype_t *)vtag; if (vt->name == jl_array_typename) { jl_array_t *a = (jl_array_t *)new_obj; - jl_array_flags_t flags = a->flags; + size_t dtsz = sizeof(jl_array_t); + if (update_meta) + gc_setmark(ptls, o, bits, dtsz); + else if (foreign_alloc) + objprofile_count(vt, bits == GC_OLD_MARKED, dtsz); + jl_genericmemory_t *owner = a->ref.mem; + uintptr_t nptr = (1 << 2) | (bits & GC_OLD); + gc_try_claim_and_push(mq, owner, &nptr); + gc_heap_snapshot_record_internal_array_edge(new_obj, (jl_value_t*)owner); + gc_mark_push_remset(ptls, new_obj, nptr); + return; + } + if (vt->name == jl_genericmemory_typename) { + jl_genericmemory_t *m = (jl_genericmemory_t*)new_obj; + int pooled = 1; // The jl_genericmemory_t itself is always pooled-size, even with data attached to it if (update_meta) { - if (flags.pooled) + if (pooled) gc_setmark_pool(ptls, o, bits); else gc_setmark_big(ptls, o, bits); } else if (foreign_alloc) { - objprofile_count(vt, bits == GC_OLD_MARKED, sizeof(jl_array_t)); - } - if (flags.how == 0) { - void *data_ptr = (char*)a + sizeof(jl_array_t) +jl_array_ndimwords(a->flags.ndims) * sizeof(size_t); - gc_heap_snapshot_record_hidden_edge(new_obj, data_ptr, jl_array_nbytes(a), 2); + objprofile_count(vt, bits == GC_OLD_MARKED, sizeof(jl_genericmemory_t)); } - if (flags.how == 1) { - void *val_buf = jl_astaggedvalue((char*)a->data - a->offset * a->elsize); - verify_parent1("array", new_obj, &val_buf, "buffer ('loc' addr is meaningless)"); - gc_heap_snapshot_record_hidden_edge(new_obj, jl_valueof(val_buf), jl_array_nbytes(a), flags.pooled); - (void)val_buf; - gc_setmark_buf_(ptls, (char*)a->data - a->offset * a->elsize, - bits, jl_array_nbytes(a)); + int how = jl_genericmemory_how(m); + if (how == 0 || how == 2) { + gc_heap_snapshot_record_hidden_edge(new_obj, m->data, jl_genericmemory_nbytes(m), how == 0 ? 2 : 0); } - else if (flags.how == 2) { + else if (how == 1) { if (update_meta || foreign_alloc) { objprofile_count(jl_malloc_tag, bits == GC_OLD_MARKED, - jl_array_nbytes(a)); - gc_heap_snapshot_record_hidden_edge(new_obj, a->data, jl_array_nbytes(a), flags.pooled); + jl_genericmemory_nbytes(m)); + size_t nb = jl_genericmemory_nbytes(m); + gc_heap_snapshot_record_hidden_edge(new_obj, m->data, nb, 0); if (bits == GC_OLD_MARKED) { - ptls->gc_cache.perm_scanned_bytes += jl_array_nbytes(a); + ptls->gc_cache.perm_scanned_bytes += nb; } else { - ptls->gc_cache.scanned_bytes += jl_array_nbytes(a); + ptls->gc_cache.scanned_bytes += nb; } } } - else if (flags.how == 3) { - jl_value_t *owner = jl_array_data_owner(a); + else if (how == 3) { + jl_value_t *owner = jl_genericmemory_data_owner_field(m); uintptr_t nptr = (1 << 2) | (bits & GC_OLD); gc_try_claim_and_push(mq, owner, &nptr); gc_heap_snapshot_record_internal_array_edge(new_obj, owner); gc_mark_push_remset(ptls, new_obj, nptr); return; } - if (!a->data || jl_array_len(a) == 0) + if (m->length == 0) return; - if (flags.ptrarray) { - if ((jl_datatype_t *)jl_tparam0(vt) == jl_symbol_type) + const jl_datatype_layout_t *layout = vt->layout; + if (layout->arrayelem_isboxed) { + if ((jl_datatype_t*)jl_tparam1(vt) == jl_symbol_type) return; - size_t l = jl_array_len(a); jl_value_t *objary_parent = new_obj; - jl_value_t **objary_begin = (jl_value_t **)a->data; - jl_value_t **objary_end = objary_begin + l; + jl_value_t **objary_begin = (jl_value_t **)m->data; + jl_value_t **objary_end = objary_begin + m->length; uint32_t step = 1; - uintptr_t nptr = (l << 2) | (bits & GC_OLD); + uintptr_t nptr = (m->length << 2) | (bits & GC_OLD); gc_mark_objarray(ptls, objary_parent, objary_begin, objary_end, step, nptr); } - else if (flags.hasptr) { - jl_datatype_t *et = (jl_datatype_t *)jl_tparam0(vt); - const jl_datatype_layout_t *layout = et->layout; + else if (layout->first_ptr >= 0) { + const jl_datatype_layout_t *layout = vt->layout; unsigned npointers = layout->npointers; - unsigned elsize = a->elsize / sizeof(jl_value_t *); - size_t l = jl_array_len(a); + unsigned elsize = layout->size / sizeof(jl_value_t*); + size_t l = m->length; jl_value_t *objary_parent = new_obj; - jl_value_t **objary_begin = (jl_value_t **)a->data; + jl_value_t **objary_begin = (jl_value_t**)m->data; jl_value_t **objary_end = objary_begin + l * elsize; uint32_t step = elsize; uintptr_t nptr = ((l * npointers) << 2) | (bits & GC_OLD); @@ -2623,16 +2632,16 @@ FORCE_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_ gc_mark_objarray(ptls, objary_parent, objary_begin, objary_end, step, nptr); } else if (layout->fielddesc_type == 0) { - uint8_t *obj8_begin = (uint8_t *)jl_dt_layout_ptrs(layout); + uint8_t *obj8_begin = (uint8_t*)jl_dt_layout_ptrs(layout); uint8_t *obj8_end = obj8_begin + npointers; - gc_mark_array8(ptls, objary_parent, objary_begin, objary_end, obj8_begin, - obj8_end, nptr); + gc_mark_array8(ptls, objary_parent, objary_begin, objary_end, obj8_begin, obj8_end, + elsize, nptr); } else if (layout->fielddesc_type == 1) { - uint16_t *obj16_begin = (uint16_t *)jl_dt_layout_ptrs(layout); + uint16_t *obj16_begin = (uint16_t*)jl_dt_layout_ptrs(layout); uint16_t *obj16_end = obj16_begin + npointers; - gc_mark_array16(ptls, objary_parent, objary_begin, objary_end, obj16_begin, - obj16_end, nptr); + gc_mark_array16(ptls, objary_parent, objary_begin, objary_end, obj16_begin, obj16_end, + elsize, nptr); } else { assert(0 && "unimplemented"); diff --git a/src/gc.h b/src/gc.h index 90943cba84d4b..9bb5edf75d6a0 100644 --- a/src/gc.h +++ b/src/gc.h @@ -137,10 +137,10 @@ JL_EXTENSION typedef struct _bigval_t { // must be 64-byte aligned here, in 32 & 64 bit modes } bigval_t; -// data structure for tracking malloc'd arrays. +// data structure for tracking malloc'd arrays and genericmemory. typedef struct _mallocarray_t { - jl_array_t *a; + jl_value_t *a; struct _mallocarray_t *next; } mallocarray_t; diff --git a/src/gen_sysimg_symtab.jl b/src/gen_sysimg_symtab.jl index 8f03cc1560767..a91f2f994194c 100644 --- a/src/gen_sysimg_symtab.jl +++ b/src/gen_sysimg_symtab.jl @@ -15,12 +15,6 @@ function _eachmethod(f, m::Module, visited, vmt) x = getfield(m, nm) if isa(x, Module) && !in(x, visited) _eachmethod(f, x, visited, vmt) - elseif isa(x, Function) - mt = typeof(x).name.mt - if !in(mt, vmt) - push!(vmt, mt) - Base.visit(f, mt) - end elseif isa(x, Type) x = Base.unwrap_unionall(x) if isa(x, DataType) && isdefined(x.name, :mt) @@ -69,5 +63,5 @@ function outputline(io, name) println(io, "jl_symbol(\"", name, "\"),") end -open(f->foreach(l->outputline(f,l), take(syms, 100)), "common_symbols1.inc", "w") -open(f->foreach(l->outputline(f,l), take(drop(syms, 100), 254)), "common_symbols2.inc", "w") +open(f->foreach(l->outputline(f,l), take(syms, 94)), "common_symbols1.inc", "w") +open(f->foreach(l->outputline(f,l), take(drop(syms, 94), 254)), "common_symbols2.inc", "w") diff --git a/src/genericmemory.c b/src/genericmemory.c new file mode 100644 index 0000000000000..7a21e82fe9352 --- /dev/null +++ b/src/genericmemory.c @@ -0,0 +1,511 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +/* + GenericMemory{kind, T} constructors and primitives +*/ +#include +#include +#ifdef _OS_WINDOWS_ +#include +#endif +#include "julia.h" +#include "julia_internal.h" +#include "julia_assert.h" + +#ifdef __cplusplus +extern "C" { +#endif + +static inline void genericmemoryassign_safe(int hasptr, jl_value_t *parent, char *dst, const jl_value_t *src) JL_NOTSAFEPOINT +{ + size_t nb = jl_datatype_size(jl_typeof(src)); // make sure to shrink-wrap this copy + if (hasptr) { + size_t nptr = nb / sizeof(void*); + memmove_refs((void**)dst, (void* const*)src, nptr); + jl_gc_multi_wb(parent, src); + } + else { + // genericmemory can assume more alignment than a field would normally have + switch (nb) { + case 0: break; + case 1: *(uint8_t*)dst = *(uint8_t*)src; break; + case 2: *(uint16_t*)dst = *(uint16_t*)src; break; + case 4: *(uint32_t*)dst = *(uint32_t*)src; break; + case 8: *(uint64_t*)dst = *(uint64_t*)src; break; + case 16: + memcpy(jl_assume_aligned(dst, 16), jl_assume_aligned(src, 16), 16); + break; + default: memcpy(dst, src, nb); + } + } +} + +// genericmemory constructors --------------------------------------------------------- +JL_DLLEXPORT char *jl_genericmemory_typetagdata(jl_genericmemory_t *m) JL_NOTSAFEPOINT +{ + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + assert(layout->arrayelem_isunion); + return (char*)m->data + m->length * layout->size; +} + +#if defined(_P64) && defined(UINT128MAX) +typedef __uint128_t wideint_t; +#else +typedef uint64_t wideint_t; +#endif + +#define MAXINTVAL (((size_t)-1)>>1) + +jl_genericmemory_t *_new_genericmemory_(jl_value_t *mtype, size_t nel, int8_t isunion, int8_t zeroinit, size_t elsz) +{ + jl_task_t *ct = jl_current_task; + char *data; + jl_genericmemory_t *m; + if (nel == 0) // zero-sized allocation optimization + return (jl_genericmemory_t*)((jl_datatype_t*)mtype)->instance; + size_t tot = nel == 0 ? 1 : nel; // always reserve space for at least one element for extra safety (relevant for JL_ARRAY_IMPL_NUL) + wideint_t prod = (wideint_t)tot * elsz; + if (isunion) { + // an extra byte for each isbits union memory element, stored at m->data + m->length + prod += tot; + } + if (tot >= MAXINTVAL || prod >= (wideint_t) MAXINTVAL) + jl_exceptionf(jl_argumenterror_type, "invalid GenericMemory size"); + tot = (size_t)prod + sizeof(jl_genericmemory_t); + + int pooled = tot <= GC_MAX_SZCLASS; + if (!pooled) { + data = (char*)jl_gc_managed_malloc(prod); + tot = sizeof(jl_genericmemory_t) + sizeof(void*); + } + m = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, tot, mtype); + if (pooled) { + data = (char*)(m + 1); + } + else { + int isaligned = 1; // jl_gc_managed_malloc is always aligned + jl_gc_track_malloced_genericmemory(ct->ptls, m, isaligned); + jl_genericmemory_data_owner_field(m) = (jl_value_t*)m; + } + m->length = nel; + m->data = data; + + if (zeroinit) + memset(data, 0, (size_t)prod); + return m; +} + +JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory(jl_value_t *mtype, size_t nel) +{ + assert(jl_is_datatype(mtype)); + jl_genericmemory_t *m = (jl_genericmemory_t*)((jl_datatype_t*)mtype)->instance; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + if (m == NULL) { + if (jl_tparam0((jl_datatype_t*)mtype) != (jl_value_t*)jl_not_atomic_sym) + jl_error("GenericMemory kind must be :not_atomic"); + if (!((jl_datatype_t*)mtype)->has_concrete_subtype || layout == NULL) + jl_type_error_rt("GenericMemory", "element type", (jl_value_t*)jl_type_type, jl_tparam1(mtype)); + abort(); + } + assert(jl_tparam0((jl_datatype_t*)mtype) == (jl_value_t*)jl_not_atomic_sym); + assert(((jl_datatype_t*)mtype)->has_concrete_subtype && layout != NULL); + if (nel == 0) // zero-sized allocation optimization fast path + return m; + + size_t elsz = layout->size; + int isboxed = layout->arrayelem_isboxed; + int isunion = layout->arrayelem_isunion; + int zi = ((jl_datatype_t*)mtype)->zeroinit; + if (isboxed) + elsz = sizeof(void*); + return _new_genericmemory_(mtype, nel, isunion, zi, elsz); +} + +JL_DLLEXPORT jl_genericmemory_t *jl_string_to_genericmemory(jl_value_t *str) +{ + jl_task_t *ct = jl_current_task; + int tsz = sizeof(jl_genericmemory_t) + sizeof(void*); + jl_genericmemory_t *m = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, tsz, jl_memory_uint8_type); + m->length = jl_string_len(str); + m->data = jl_string_data(str); + jl_genericmemory_data_owner_field(m) = str; + return m; +} + +// own_buffer != 0 iff GC should call free() on this pointer eventually +JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void *data, + size_t nel, int own_buffer) +{ + jl_task_t *ct = jl_current_task; + assert(jl_is_datatype(mtype)); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + if (jl_tparam0((jl_datatype_t*)mtype) != (jl_value_t*)jl_not_atomic_sym) + jl_error("GenericMemory kind must be :not_atomic"); + if (!((jl_datatype_t*)mtype)->has_concrete_subtype || layout == NULL) + jl_type_error_rt("GenericMemory", "element type", (jl_value_t*)jl_type_type, jl_tparam1(mtype)); + + size_t elsz = layout->size; + size_t align = layout->alignment; + int isboxed = layout->arrayelem_isboxed; + int isunion = layout->arrayelem_isunion; + int zi = ((jl_datatype_t*)mtype)->zeroinit; + if (isboxed) + elsz = sizeof(void*); + jl_genericmemory_t *m = _new_genericmemory_(mtype, nel, isunion, zi, elsz); + + if (isunion) + jl_exceptionf(jl_argumenterror_type, + "unsafe_wrap: unspecified layout for union element type"); + if (((uintptr_t)data) & ((align > JL_HEAP_ALIGNMENT ? JL_HEAP_ALIGNMENT : align) - 1)) + jl_exceptionf(jl_argumenterror_type, + "unsafe_wrap: pointer %p is not properly aligned to %u bytes", data, align); + + int tsz = sizeof(jl_genericmemory_t) + sizeof(void*); + m = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, tsz, mtype); + m->data = data; + m->length = nel; + jl_genericmemory_data_owner_field(m) = NULL; + int isaligned = 0; // TODO: allow passing memalign'd buffers + if (own_buffer) { + jl_gc_track_malloced_genericmemory(ct->ptls, m, isaligned); + jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0)); + } + return m; +} + +JL_DLLEXPORT jl_genericmemory_t *jl_new_genericmemory(jl_value_t *mtype, jl_value_t *nel) +{ + return jl_alloc_genericmemory(mtype, jl_unbox_long(nel)); +} + +JL_DLLEXPORT jl_genericmemory_t *jl_pchar_to_genericmemory(const char *str, size_t len) +{ + jl_genericmemory_t *m = jl_alloc_genericmemory(jl_memory_uint8_type, len); + memcpy(m->data, str, len); + return m; +} + +JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_t len) +{ + assert(len <= m->length); + if (len == 0) { + // this may seem like purely an optimization (which it also is), but it + // also ensures that calling `String(m)` doesn't corrupt a previous + // string also created the same way, where `m = StringVector(_)`. + return jl_an_empty_string; + } + int how = jl_genericmemory_how(m); + size_t mlength = m->length; + m->length = 0; + if (how != 0) { + jl_value_t *o = jl_genericmemory_data_owner_field(m); + jl_genericmemory_data_owner_field(m) = NULL; + if (how == 3 && + ((mlength + sizeof(void*) + 1 <= GC_MAX_SZCLASS) == (len + sizeof(void*) + 1 <= GC_MAX_SZCLASS))) { + if (jl_string_data(o)[len] != '\0') + jl_string_data(o)[len] = '\0'; + if (*(size_t*)o != len) + *(size_t*)o = len; + return o; + } + JL_GC_PUSH1(&o); + jl_value_t *str = jl_pchar_to_string((const char*)m->data, len); + JL_GC_POP(); + return str; + } + return jl_pchar_to_string((const char*)m->data, len); +} + +JL_DLLEXPORT jl_genericmemory_t *jl_alloc_memory_any(size_t n) +{ + return jl_alloc_genericmemory(jl_memory_any_type, n); +} + +JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_slice(jl_genericmemory_t *mem, size_t offset, size_t len) +{ + jl_datatype_t *dt = (jl_datatype_t*)jl_typetagof(mem); + const jl_datatype_layout_t *layout = dt->layout; + if (len + offset > mem->length) + jl_exceptionf(jl_argumenterror_type, "invalid GenericMemory slice"); // TODO: make a BoundsError + if (layout->arrayelem_isunion && !(offset == 0 && mem->length == len)) + jl_exceptionf(jl_argumenterror_type, "invalid GenericMemory slice"); + jl_task_t *ct = jl_current_task; + jl_genericmemory_t *newmem = (jl_genericmemory_t*)jl_gc_alloc(ct->ptls, sizeof(jl_genericmemory_t) + sizeof(void*), dt); + newmem->length = len; + newmem->data = (void*)((char*)mem->data + offset * layout->size); + jl_genericmemory_data_owner_field(newmem) = jl_genericmemory_owner(mem); + return newmem; +} + +// genericmemory primitives ----------------------------------------------------------- + +JL_DLLEXPORT jl_value_t *jl_ptrmemref(jl_genericmemory_t *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT +{ + assert(i < m->length); + assert(((jl_datatype_t*)jl_typetagof(m))->layout->arrayelem_isboxed); + jl_value_t *elt = jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)m->data) + i); + if (elt == NULL) + jl_throw(jl_undefref_exception); + return elt; +} + +JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *m, size_t i) +{ + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->arrayelem_isboxed) + return jl_ptrmemref(m, i); + assert(i < m->length); + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m)); (void)isatomic; // TODO + jl_value_t *eltype = jl_tparam1(jl_typetagof(m)); + if (jl_is_uniontype(eltype)) { + // isbits union selector bytes are always stored directly after the last memory element + uint8_t sel = jl_genericmemory_typetagdata(m)[i]; + eltype = jl_nth_union_component(eltype, sel); + if (jl_is_datatype_singleton((jl_datatype_t*)eltype)) + return ((jl_datatype_t*)eltype)->instance; + } + jl_value_t *r = undefref_check((jl_datatype_t*)eltype, jl_new_bits(eltype, &((char*)m->data)[i * layout->size])); + if (__unlikely(r == NULL)) + jl_throw(jl_undefref_exception); + return r; +} + +JL_DLLEXPORT int jl_genericmemory_isassigned(jl_genericmemory_t *m, size_t i) +{ + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->arrayelem_isboxed) { + return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)m->data) + i) != NULL; + } + else if (layout->first_ptr >= 0) { + jl_value_t **elem = (jl_value_t**)((char*)m->data + i * layout->size); + return elem[layout->first_ptr] != NULL; + } + return 1; +} + +JL_DLLEXPORT void jl_genericmemoryset(jl_genericmemory_t *m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i) +{ + assert(i < m->length); + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m)); (void)isatomic; // TODO + jl_value_t *eltype = jl_tparam1(jl_typetagof(m)); + if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { + JL_GC_PUSH1(&rhs); + if (!jl_isa(rhs, eltype)) + jl_type_error("genericmemoryset", eltype, rhs); + JL_GC_POP(); + } + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->arrayelem_isboxed) { + jl_atomic_store_release(((_Atomic(jl_value_t*)*)m->data) + i, rhs); + jl_gc_wb(jl_genericmemory_owner(m), rhs); + } + else { + int hasptr; + if (jl_is_uniontype(eltype)) { + uint8_t *psel = &((uint8_t*)jl_genericmemory_typetagdata(m))[i]; + unsigned nth = 0; + if (!jl_find_union_component(eltype, jl_typeof(rhs), &nth)) + assert(0 && "invalid genericmemoryset to isbits union"); + *psel = nth; + if (jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(rhs))) + return; + hasptr = 0; + } + else { + hasptr = layout->first_ptr >= 0; + } + genericmemoryassign_safe(hasptr, jl_genericmemory_owner(m), &((char*)m->data)[i * layout->size], rhs); + } +} + +JL_DLLEXPORT void jl_genericmemoryunset(jl_genericmemory_t *m, size_t i) +{ + if (i >= m->length) + jl_bounds_error_int((jl_value_t*)m, i + 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m))->layout; + if (layout->arrayelem_isboxed) + jl_atomic_store_relaxed(((_Atomic(jl_value_t*)*)m->data) + i, NULL); + else if (layout->first_ptr >= 0) { + size_t elsize = layout->size; + jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0); + memset((char*)m->data + elsize * i, 0, elsize); + } +} + +JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_copy(jl_genericmemory_t *m) +{ + jl_value_t *mtype = (jl_value_t*)jl_typetagof(m); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mtype)->layout; + size_t elsz = layout->size; + size_t len = m->length; + int isunion = layout->arrayelem_isunion; + jl_genericmemory_t *new_mem = _new_genericmemory_(mtype, len, isunion, 0, elsz); + memcpy(new_mem->data, m->data, len * elsz); + // ensure isbits union memorys copy their selector bytes correctly + if (isunion) + memcpy(jl_genericmemory_typetagdata(new_mem), jl_genericmemory_typetagdata(m), len); + return new_mem; +} + +JL_DLLEXPORT jl_value_t *(jl_genericmemory_data_owner)(jl_genericmemory_t *m) JL_NOTSAFEPOINT +{ + return jl_genericmemory_data_owner_field(m); +} + +// Create a memory block with the same content and a trailing \0 +//JL_DLLEXPORT jl_genericmemory_t *jl_memory_cconvert_cstring(jl_genericmemory_t *m) +//{ +// static_assert(JL_MEMORY_IMPL_NUL, ""); +// assert(jl_typetagis(m, jl_memory_uint8_type)); +// if (!jl_has_implicit_byte(m)) +// m = jl_genericmemory_copy(m); +// if (((char*)m->data)[m->length] != 0) // avoid issues with immutable memory +// ((char*)m->data)[m->length] = 0; +// return m; +//} + +jl_genericmemoryref_t *jl_new_memoryref(jl_value_t *typ, jl_genericmemory_t *mem, void *data) +{ + jl_task_t *ct = jl_current_task; + jl_genericmemoryref_t *m = (jl_genericmemoryref_t*)jl_gc_alloc(ct->ptls, sizeof(jl_genericmemoryref_t), typ); + m->mem = mem; + m->data = data; + return m; +} + +// memoryref primitives +JL_DLLEXPORT jl_genericmemoryref_t jl_memoryrefindex(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, size_t idx) +{ + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + if ((layout->arrayelem_isboxed || !layout->arrayelem_isunion) && layout->size != 0) { + m.data = (void*)((char*)m.data + idx * layout->size); + assert((char*)m.data - (char*)m.mem->data < layout->size * m.mem->length); + } + else { + m.data = (void*)((size_t)m.data + idx); + assert((size_t)m.data < m.mem->length); + } + return m; +} + +JL_DLLEXPORT jl_value_t *jl_ptrmemrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + assert((char*)m.data - (char*)m.mem->data < sizeof(jl_value_t*) * m.mem->length); + assert(((jl_datatype_t*)jl_typetagof(m.mem))->layout->arrayelem_isboxed); + jl_value_t *elt = jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)m.data); + if (elt == NULL) + jl_throw(jl_undefref_exception); + return elt; +} + +JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m) +{ + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + if (layout->arrayelem_isboxed) + return jl_ptrmemrefget(m); + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); (void)isatomic; // TODO + jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); + char *data = (char*)m.data; + if (layout->arrayelem_isunion) { + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + // isbits union selector bytes are always stored directly after the last memory element + uint8_t sel = jl_genericmemory_typetagdata(m.mem)[i]; + eltype = jl_nth_union_component(eltype, sel); + if (jl_is_datatype_singleton((jl_datatype_t*)eltype)) + return ((jl_datatype_t*)eltype)->instance; + data = (char*)m.mem->data + i * layout->size; + } + else if (layout->size == 0) { + assert(jl_is_datatype_singleton((jl_datatype_t*)eltype)); + return ((jl_datatype_t*)eltype)->instance; + } + assert(data - (char*)m.mem->data < layout->size * m.mem->length); + jl_value_t *r = undefref_check((jl_datatype_t*)eltype, jl_new_bits(eltype, data)); + if (__unlikely(r == NULL)) + jl_throw(jl_undefref_exception); + return r; +} + +static int _jl_memoryref_isassigned(jl_genericmemoryref_t m) +{ + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + if (layout->arrayelem_isboxed) { + return jl_atomic_load_relaxed((_Atomic(jl_value_t*)*)m.data) != NULL; + } + else if (layout->first_ptr >= 0) { + jl_value_t **elem = (jl_value_t**)m.data; + return elem[layout->first_ptr] != NULL; + } + return 1; +} + +JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m) +{ + return _jl_memoryref_isassigned(m) ? jl_true : jl_false; +} + +JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *rhs JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED) +{ + jl_value_t *isatomic = jl_tparam0(jl_typetagof(m.mem)); (void)isatomic; // TODO + jl_value_t *eltype = jl_tparam1(jl_typetagof(m.mem)); + if (eltype != (jl_value_t*)jl_any_type && !jl_typeis(rhs, eltype)) { + JL_GC_PUSH1(&rhs); + if (!jl_isa(rhs, eltype)) + jl_type_error("memoryrefset", eltype, rhs); + JL_GC_POP(); + } + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + if (layout->arrayelem_isboxed) { + assert((char*)m.data - (char*)m.mem->data < sizeof(jl_value_t*) * m.mem->length); + jl_atomic_store_release((_Atomic(jl_value_t*)*)m.data, rhs); + jl_gc_wb(jl_genericmemory_owner(m.mem), rhs); + } + else { + int hasptr; + char *data = (char*)m.data; + if (layout->arrayelem_isunion) { + assert(jl_is_uniontype(eltype)); + size_t i = (size_t)data; + assert(i < m.mem->length); + uint8_t *psel = (uint8_t*)jl_genericmemory_typetagdata(m.mem) + i; + unsigned nth = 0; + if (!jl_find_union_component(eltype, jl_typeof(rhs), &nth)) + assert(0 && "invalid genericmemoryset to isbits union"); + *psel = nth; + if (jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(rhs))) + return; + hasptr = 0; + data = (char*)m.mem->data + i * layout->size; + } + else { + hasptr = layout->first_ptr >= 0; + } + assert(data - (char*)m.mem->data < layout->size * m.mem->length); + genericmemoryassign_safe(hasptr, jl_genericmemory_owner(m.mem), data, rhs); + } +} + +JL_DLLEXPORT void jl_memoryrefunset(jl_genericmemoryref_t m) +{ + if (m.mem->length == 0) + jl_bounds_error_int((jl_value_t*)m.mem, 1); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(m.mem))->layout; + if (layout->arrayelem_isboxed) { + jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)m.data, NULL); + } + else if (layout->first_ptr >= 0) { + size_t elsize = layout->size; + jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0); + memset(m.data, 0, elsize); + } +} + +JL_DLLEXPORT jl_value_t *jl_mem_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + return jl_genericmemory_owner(m); +} +#ifdef __cplusplus +} +#endif diff --git a/src/gf.c b/src/gf.c index e8cdb493026b0..1694340948d13 100644 --- a/src/gf.c +++ b/src/gf.c @@ -615,7 +615,7 @@ int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), voi if (mod_array) { JL_GC_PUSH1(&mod_array); int i; - for (i = 0; i < jl_array_len(mod_array); i++) { + for (i = 0; i < jl_array_nrows(mod_array); i++) { jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(mod_array, i); assert(jl_is_module(m)); if (m->parent == m) // some toplevel modules (really just Base) aren't actually @@ -640,7 +640,7 @@ static int reset_mt_caches(jl_methtable_t *mt, void *env) // removes all method caches // this might not be entirely safe (GC or MT), thus we only do it very early in bootstrapping if (!mt->frozen) { // make sure not to reset builtin functions - jl_atomic_store_release(&mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_release(&mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_atomic_store_release(&mt->cache, jl_nothing); } jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), get_method_unspec_list, env); @@ -664,7 +664,7 @@ JL_DLLEXPORT void jl_set_typeinf_func(jl_value_t *f) JL_GC_PUSH1(&unspec); jl_foreach_reachable_mtable(reset_mt_caches, (void*)unspec); size_t i, l; - for (i = 0, l = jl_array_len(unspec); i < l; i++) { + for (i = 0, l = jl_array_nrows(unspec); i < l; i++) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(unspec, i); if (jl_rettype_inferred(mi, world, world) == jl_nothing) jl_type_infer(mi, world, 1); @@ -1213,7 +1213,7 @@ static int concretesig_equal(jl_value_t *tt, jl_value_t *simplesig) JL_NOTSAFEPO return 1; } -static inline jl_typemap_entry_t *lookup_leafcache(jl_array_t *leafcache JL_PROPAGATES_ROOT, jl_value_t *tt, size_t world) JL_NOTSAFEPOINT +static inline jl_typemap_entry_t *lookup_leafcache(jl_genericmemory_t *leafcache JL_PROPAGATES_ROOT, jl_value_t *tt, size_t world) JL_NOTSAFEPOINT { jl_typemap_entry_t *entry = (jl_typemap_entry_t*)jl_eqtable_get(leafcache, (jl_value_t*)tt, NULL); if (entry) { @@ -1240,7 +1240,7 @@ static jl_method_instance_t *cache_method( int8_t offs = mt ? jl_cachearg_offset(mt) : 1; { // scope block if (mt) { - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); if (entry) return entry->func.linfo; @@ -1306,7 +1306,7 @@ static jl_method_instance_t *cache_method( } else { int unmatched_tvars = 0; - size_t i, l = jl_array_len(temp); + size_t i, l = jl_array_nrows(temp); for (i = 0; i < l; i++) { jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(temp, i); if (matc->method == definition) @@ -1339,7 +1339,7 @@ static jl_method_instance_t *cache_method( guardsigs = jl_alloc_svec(guards); temp3 = (jl_value_t*)guardsigs; guards = 0; - for (i = 0, l = jl_array_len(temp); i < l; i++) { + for (i = 0, l = jl_array_nrows(temp); i < l; i++) { jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(temp, i); jl_method_t *other = matc->method; if (other != definition) { @@ -1420,11 +1420,11 @@ static jl_method_instance_t *cache_method( jl_cache_type_(tt); JL_UNLOCK(&typecache_lock); // Might GC } - jl_array_t *oldcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *oldcache = jl_atomic_load_relaxed(&mt->leafcache); jl_typemap_entry_t *old = (jl_typemap_entry_t*)jl_eqtable_get(oldcache, (jl_value_t*)tt, jl_nothing); jl_atomic_store_relaxed(&newentry->next, old); jl_gc_wb(newentry, old); - jl_array_t *newcache = (jl_array_t*)jl_eqtable_put(jl_atomic_load_relaxed(&mt->leafcache), (jl_value_t*)tt, (jl_value_t*)newentry, NULL); + jl_genericmemory_t *newcache = jl_eqtable_put(jl_atomic_load_relaxed(&mt->leafcache), (jl_value_t*)tt, (jl_value_t*)newentry, NULL); if (newcache != oldcache) { jl_atomic_store_release(&mt->leafcache, newcache); jl_gc_wb(mt, newcache); @@ -1445,7 +1445,7 @@ static jl_method_instance_t *jl_mt_assoc_by_type(jl_methtable_t *mt JL_PROPAGATE // caller must hold the mt->writelock assert(tt->isdispatchtuple || tt->hasfreetypevars); if (tt->isdispatchtuple) { - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); if (entry) return entry->func.linfo; @@ -1609,7 +1609,7 @@ static void invalidate_external(jl_method_instance_t *mi, size_t max_world) { // AbstractInterpreter allows for MethodInstances to be present in non-local caches // inform those caches about the invalidation. JL_TRY { - size_t i, l = jl_array_len(callbacks); + size_t i, l = jl_array_nrows(callbacks); jl_value_t **args; JL_GC_PUSHARGS(args, 3); // these arguments are constant per call @@ -1672,7 +1672,7 @@ static void invalidate_method_instance(void (*f)(jl_code_instance_t*), jl_method if (backedges) { JL_GC_PUSH1(&backedges); replaced->backedges = NULL; - size_t i = 0, l = jl_array_len(backedges); + size_t i = 0, l = jl_array_nrows(backedges); jl_method_instance_t *replaced; while (i < l) { i = get_next_edge(backedges, i, NULL, &replaced); @@ -1693,7 +1693,7 @@ static void invalidate_backedges(void (*f)(jl_code_instance_t*), jl_method_insta // invalidate callers (if any) replaced_mi->backedges = NULL; JL_GC_PUSH1(&backedges); - size_t i = 0, l = jl_array_len(backedges); + size_t i = 0, l = jl_array_nrows(backedges); jl_method_instance_t *replaced; while (i < l) { i = get_next_edge(backedges, i, NULL, &replaced); @@ -1725,7 +1725,7 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, jl_gc_wb(callee, callee->backedges); } else { - size_t i = 0, l = jl_array_len(callee->backedges); + size_t i = 0, l = jl_array_nrows(callee->backedges); for (i = 0; i < l; i++) { // optimized version of while (i < l) i = get_next_edge(callee->backedges, i, &invokeTypes, &mi); jl_value_t *mi = jl_array_ptr_ref(callee->backedges, i); @@ -1759,7 +1759,7 @@ JL_DLLEXPORT void jl_method_table_add_backedge(jl_methtable_t *mt, jl_value_t *t } else { // TODO: use jl_cache_type_(tt) like cache_method does, instead of a linear scan - size_t i, l = jl_array_len(mt->backedges); + size_t i, l = jl_array_nrows(mt->backedges); for (i = 1; i < l; i += 2) { if (jl_types_equal(jl_array_ptr_ref(mt->backedges, i - 1), typ)) { if (jl_array_ptr_ref(mt->backedges, i) == caller) { @@ -1790,7 +1790,7 @@ static int invalidate_mt_cache(jl_typemap_entry_t *oldentry, void *closure0) jl_method_instance_t *mi = oldentry->func.linfo; int intersects = 0; jl_method_instance_t **d = (jl_method_instance_t**)jl_array_ptr_data(env->shadowed); - size_t i, n = jl_array_len(env->shadowed); + size_t i, n = jl_array_nrows(env->shadowed); for (i = 0; i < n; i++) { if (mi == d[i]) { intersects = 1; @@ -1872,10 +1872,10 @@ static void jl_method_table_invalidate(jl_methtable_t *mt, jl_typemap_entry_t *m mt_cache_env.shadowed = NULL; mt_cache_env.invalidated = 0; jl_typemap_visitor(jl_atomic_load_relaxed(&mt->cache), disable_mt_cache, (void*)&mt_cache_env); - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - size_t i, l = jl_array_len(leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + size_t i, l = leafcache->length; for (i = 1; i < l; i += 2) { - jl_typemap_entry_t *oldentry = (jl_typemap_entry_t*)jl_array_ptr_ref(leafcache, i); + jl_typemap_entry_t *oldentry = (jl_typemap_entry_t*)jl_genericmemory_ptr_ref(leafcache, i); if (oldentry) { while ((jl_value_t*)oldentry != jl_nothing) { if (oldentry->max_world == ~(size_t)0) @@ -2023,11 +2023,11 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method else { assert(jl_is_array(oldvalue)); d = (jl_method_t**)jl_array_ptr_data(oldvalue); - n = jl_array_len(oldvalue); + n = jl_array_nrows(oldvalue); } if (mt->backedges) { jl_value_t **backedges = jl_array_ptr_data(mt->backedges); - size_t i, na = jl_array_len(mt->backedges); + size_t i, na = jl_array_nrows(mt->backedges); size_t ins = 0; for (i = 1; i < na; i += 2) { jl_value_t *backedgetyp = backedges[i - 1]; @@ -2122,7 +2122,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method // but ignore invoke-type edges jl_array_t *backedges = mi->backedges; if (backedges) { - size_t ib = 0, insb = 0, nb = jl_array_len(backedges); + size_t ib = 0, insb = 0, nb = jl_array_nrows(backedges); jl_value_t *invokeTypes; jl_method_instance_t *caller; while (ib < nb) { @@ -2158,7 +2158,7 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method } } } - if (jl_array_len(oldmi)) { + if (jl_array_nrows(oldmi)) { // search mt->cache and leafcache and drop anything that might overlap with the new method // this is very cheap, so we don't mind being fairly conservative at over-approximating this struct invalidate_mt_env mt_cache_env; @@ -2168,10 +2168,10 @@ JL_DLLEXPORT void jl_method_table_insert(jl_methtable_t *mt, jl_method_t *method mt_cache_env.invalidated = 0; jl_typemap_visitor(jl_atomic_load_relaxed(&mt->cache), invalidate_mt_cache, (void*)&mt_cache_env); - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); - size_t i, l = jl_array_len(leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + size_t i, l = leafcache->length; for (i = 1; i < l; i += 2) { - jl_value_t *entry = jl_array_ptr_ref(leafcache, i); + jl_value_t *entry = jl_genericmemory_ptr_ref(leafcache, i); if (entry) { while (entry != jl_nothing) { invalidate_mt_cache((jl_typemap_entry_t*)entry, (void*)&mt_cache_env); @@ -2244,7 +2244,7 @@ jl_method_instance_t *jl_method_lookup(jl_value_t **args, size_t nargs, size_t w if (entry) return entry->func.linfo; jl_tupletype_t *tt = arg_type_tuple(args[0], &args[1], nargs); - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); entry = lookup_leafcache(leafcache, (jl_value_t*)tt, world); if (entry) return entry->func.linfo; @@ -2670,7 +2670,7 @@ jl_method_instance_t *jl_get_specialization1(jl_tupletype_t *types JL_PROPAGATES *min_valid = min_valid2; if (*max_valid > max_valid2) *max_valid = max_valid2; - if (matches == jl_nothing || jl_array_len(matches) != 1 || ambig) + if (matches == jl_nothing || jl_array_nrows(matches) != 1 || ambig) return NULL; JL_GC_PUSH1(&matches); jl_method_match_t *match = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); @@ -2696,7 +2696,7 @@ static jl_method_instance_t *jl_get_compile_hint_specialization(jl_tupletype_t * *min_valid = min_valid2; if (*max_valid > max_valid2) *max_valid = max_valid2; - size_t i, n = jl_array_len(matches); + size_t i, n = jl_array_nrows(matches); if (n == 0) return NULL; JL_GC_PUSH1(&matches); @@ -3000,9 +3000,9 @@ STATIC_INLINE jl_method_instance_t *jl_lookup_generic_(jl_value_t *F, jl_value_t // if no method was found in the associative cache, check the full cache JL_TIMING(METHOD_LOOKUP_FAST, METHOD_LOOKUP_FAST); mt = jl_gf_mtable(F); - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); entry = NULL; - if (leafcache != (jl_array_t*)jl_an_empty_vec_any && + if (leafcache != (jl_genericmemory_t*)jl_an_empty_memory_any && jl_typetagis(jl_atomic_load_relaxed(&mt->cache), jl_typemap_level_type)) { // hashing args is expensive, but looking at mt->cache is probably even more expensive tt = lookup_arg_type_tuple(F, args, nargs); @@ -3086,7 +3086,7 @@ static jl_method_match_t *_gf_invoke_lookup(jl_value_t *types JL_PROPAGATES_ROOT if (mt == jl_nothing) mt = NULL; jl_value_t *matches = ml_matches((jl_methtable_t*)mt, (jl_tupletype_t*)types, 1, 0, 0, world, 1, min_valid, max_valid, NULL); - if (matches == jl_nothing || jl_array_len(matches) != 1) + if (matches == jl_nothing || jl_array_nrows(matches) != 1) return NULL; jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(matches, 0); return matc; @@ -3201,7 +3201,8 @@ jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_ jl_gc_wb(ftype->name->mt, name); jl_set_const(module, tname, (jl_value_t*)ftype); jl_value_t *f = jl_new_struct(ftype); - ftype->instance = f; jl_gc_wb(ftype, f); + ftype->instance = f; + jl_gc_wb(ftype, f); JL_GC_POP(); return (jl_function_t*)f; } @@ -3276,7 +3277,7 @@ static int ml_matches_visitor(jl_typemap_entry_t *ml, struct typemap_intersectio closure->matc = make_method_match((jl_tupletype_t*)closure->match.ti, closure->match.env, meth, closure->match.issubty ? FULLY_COVERS : NOT_FULLY_COVERS); - size_t len = jl_array_len(closure->t); + size_t len = jl_array_nrows(closure->t); if (len == 0) { closure->t = (jl_value_t*)jl_alloc_vec_any(1); jl_array_ptr_set(closure->t, 0, (jl_value_t*)closure->matc); @@ -3363,7 +3364,7 @@ static int sort_mlmatches(jl_array_t *t, size_t idx, arraylist_t *visited, array // First visit all "strong" edges where the child is definitely better. // This likely won't hit any cycles, but might (because morespecific is not transitive). // Along the way, record if we hit any ambiguities-we may need to track those later. - for (size_t childidx = 0; childidx < jl_array_len(t); childidx++) { + for (size_t childidx = 0; childidx < jl_array_nrows(t); childidx++) { if (childidx == idx) continue; int child_cycle = (size_t)visited->items[childidx]; @@ -3608,7 +3609,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, if (mt) { // check the leaf cache if this type can be in there if (((jl_datatype_t*)unw)->isdispatchtuple) { - jl_array_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); + jl_genericmemory_t *leafcache = jl_atomic_load_relaxed(&mt->leafcache); jl_typemap_entry_t *entry = lookup_leafcache(leafcache, (jl_value_t*)type, world); if (entry) { jl_method_instance_t *mi = entry->func.linfo; @@ -3679,7 +3680,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, *max_valid = env.match.max_valid; // done with many of these values now env.match.ti = NULL; env.matc = NULL; env.match.env = NULL; search.env = NULL; - size_t i, j, len = jl_array_len(env.t); + size_t i, j, len = jl_array_nrows(env.t); jl_method_match_t *minmax = NULL; int minmax_ambig = 0; int all_subtypes = 1; @@ -3825,7 +3826,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, jl_method_match_t *matc = (jl_method_match_t*)jl_array_ptr_ref(env.t, idx); jl_method_t *m = matc->method; int subt = matc->fully_covers == FULLY_COVERS; // jl_subtype((jl_value_t*)type, (jl_value_t*)m->sig) - for (size_t idx2 = 0; idx2 < jl_array_len(env.t); idx2++) { + for (size_t idx2 = 0; idx2 < jl_array_nrows(env.t); idx2++) { if (idx2 == idx) continue; // laborious test, checking for existence and coverage of another method (m3) @@ -3907,7 +3908,7 @@ static jl_value_t *ml_matches(jl_methtable_t *mt, arraylist_push(&result, minmax); j++; } - memcpy(jl_array_data(env.t), result.items, j * sizeof(jl_method_match_t*)); + memcpy(jl_array_data(env.t, jl_method_match_t*), result.items, j * sizeof(jl_method_match_t*)); arraylist_free(&result); if (j != len) jl_array_del_end((jl_array_t*)env.t, len - j); diff --git a/src/iddict.c b/src/iddict.c index 1fa8a67d1ae96..9ebc54a1f4bfe 100644 --- a/src/iddict.c +++ b/src/iddict.c @@ -1,6 +1,6 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license -#define hash_size(h) (jl_array_len(h) / 2) +#define hash_size(h) (h->length / 2) // compute empirical max-probe for a given size #define max_probe(size) ((size) <= 1024 ? 16 : (size) >> 6) @@ -8,15 +8,15 @@ #define keyhash(k) jl_object_id_(jl_typeof(k), k) #define h2index(hv, sz) (size_t)(((hv) & ((sz)-1)) * 2) -static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_t *val); +static inline int jl_table_assign_bp(jl_genericmemory_t **pa, jl_value_t *key, jl_value_t *val); -JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz) +JL_DLLEXPORT jl_genericmemory_t *jl_idtable_rehash(jl_genericmemory_t *a, size_t newsz) { - size_t sz = jl_array_len(a); + size_t sz = a->length; size_t i; - jl_value_t **ol = (jl_value_t **)a->data; - jl_array_t *newa = jl_alloc_vec_any(newsz); - // keep the original array in the original slot since we need `ol` + jl_value_t **ol = (jl_value_t **) a->data; + jl_genericmemory_t *newa = jl_alloc_memory_any(newsz); + // keep the original memory in the original slot since we need `ol` // to be valid in the loop below. JL_GC_PUSH2(&newa, &a); for (i = 0; i < sz; i += 2) { @@ -30,20 +30,20 @@ JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz) return newa; } -static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_t *val) +static inline int jl_table_assign_bp(jl_genericmemory_t **pa, jl_value_t *key, jl_value_t *val) { // pa points to a **un**rooted address uint_t hv; - jl_array_t *a = *pa; + jl_genericmemory_t *a = *pa; size_t orig, index, iter, empty_slot; size_t newsz, sz = hash_size(a); if (sz == 0) { - a = jl_alloc_vec_any(HT_N_INLINE); + a = jl_alloc_memory_any(HT_N_INLINE); sz = hash_size(a); *pa = a; } size_t maxprobe = max_probe(sz); - _Atomic(jl_value_t*) *tab = (_Atomic(jl_value_t*)*)a->data; + _Atomic(jl_value_t*) *tab = (_Atomic(jl_value_t*)*) a->data; hv = keyhash(key); while (1) { @@ -92,7 +92,7 @@ static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_ /* quadruple size, rehash, retry the insert */ /* it's important to grow the table really fast; otherwise we waste */ /* lots of time rehashing all the keys over and over. */ - sz = jl_array_len(a); + sz = a -> length; if (sz < HT_N_INLINE) newsz = HT_N_INLINE; else if (sz >= (1 << 19) || (sz <= (1 << 8))) @@ -102,20 +102,20 @@ static inline int jl_table_assign_bp(jl_array_t **pa, jl_value_t *key, jl_value_ *pa = jl_idtable_rehash(*pa, newsz); a = *pa; - tab = (_Atomic(jl_value_t*)*)a->data; + tab = (_Atomic(jl_value_t*)*) a->data; sz = hash_size(a); maxprobe = max_probe(sz); } } /* returns bp if key is in hash, otherwise NULL */ -inline _Atomic(jl_value_t*) *jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT +inline _Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) JL_NOTSAFEPOINT { size_t sz = hash_size(a); if (sz == 0) return NULL; size_t maxprobe = max_probe(sz); - _Atomic(jl_value_t*) *tab = (_Atomic(jl_value_t*)*)a->data; + _Atomic(jl_value_t*) *tab = (_Atomic(jl_value_t*)*) a->data; uint_t hv = keyhash(key); size_t index = h2index(hv, sz); sz *= 2; @@ -142,7 +142,7 @@ inline _Atomic(jl_value_t*) *jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL } JL_DLLEXPORT -jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int *p_inserted) +jl_genericmemory_t *jl_eqtable_put(jl_genericmemory_t *h, jl_value_t *key, jl_value_t *val, int *p_inserted) { int inserted = jl_table_assign_bp(&h, key, val); if (p_inserted) @@ -153,20 +153,20 @@ jl_array_t *jl_eqtable_put(jl_array_t *h, jl_value_t *key, jl_value_t *val, int // Note: lookup in the IdDict is permitted concurrently, if you avoid deletions, // and assuming you do use an external lock around all insertions JL_DLLEXPORT -jl_value_t *jl_eqtable_get(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT +jl_value_t *jl_eqtable_get(jl_genericmemory_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT { _Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key); return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp); } -jl_value_t *jl_eqtable_getkey(jl_array_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT +jl_value_t *jl_eqtable_getkey(jl_genericmemory_t *h, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT { _Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key); return (bp == NULL) ? deflt : jl_atomic_load_relaxed(bp - 1); } JL_DLLEXPORT -jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, int *found) +jl_value_t *jl_eqtable_pop(jl_genericmemory_t *h, jl_value_t *key, jl_value_t *deflt, int *found) { _Atomic(jl_value_t*) *bp = jl_table_peek_bp(h, key); if (found) @@ -180,12 +180,12 @@ jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, in } JL_DLLEXPORT -size_t jl_eqtable_nextind(jl_array_t *t, size_t i) +size_t jl_eqtable_nextind(jl_genericmemory_t *t, size_t i) { if (i & 1) i++; - size_t alen = jl_array_dim0(t); - while (i < alen && ((void **)t->data)[i + 1] == NULL) + size_t alen = t->length; + while (i < alen && ((void**) t->data)[i + 1] == NULL) i += 2; if (i >= alen) return (size_t)-1; diff --git a/src/init.c b/src/init.c index a046b4e6dcb21..cde6eb51a2e39 100644 --- a/src/init.c +++ b/src/init.c @@ -850,14 +850,14 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_options.cpu_target = "native"; jl_init_codegen(); + jl_init_common_symbols(); if (jl_options.image_file) { jl_restore_system_image(jl_options.image_file); } else { jl_init_types(); - jl_global_roots_table = jl_alloc_vec_any(0); + jl_global_roots_table = jl_alloc_memory_any(0); } - jl_init_common_symbols(); jl_init_flisp(); jl_init_serializer(); @@ -892,7 +892,7 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_array_t *init_order = jl_module_init_order; JL_GC_PUSH1(&init_order); jl_module_init_order = NULL; - int i, l = jl_array_len(init_order); + int i, l = jl_array_nrows(init_order); for (i = 0; i < l; i++) { jl_value_t *mod = jl_array_ptr_ref(init_order, i); jl_module_run_initializer((jl_module_t*)mod); diff --git a/src/interpreter.c b/src/interpreter.c index 2ad56e76b2549..e4814d659d45b 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -160,12 +160,12 @@ jl_value_t *jl_eval_globalref(jl_globalref_t *g) static int jl_source_nslots(jl_code_info_t *src) JL_NOTSAFEPOINT { - return jl_array_len(src->slotflags); + return jl_array_nrows(src->slotflags); } static int jl_source_nssavalues(jl_code_info_t *src) JL_NOTSAFEPOINT { - return jl_is_long(src->ssavaluetypes) ? jl_unbox_long(src->ssavaluetypes) : jl_array_len(src->ssavaluetypes); + return jl_is_long(src->ssavaluetypes) ? jl_unbox_long(src->ssavaluetypes) : jl_array_nrows(src->ssavaluetypes); } static void eval_stmt_value(jl_value_t *stmt, interpreter_state *s) @@ -216,7 +216,7 @@ static jl_value_t *eval_value(jl_value_t *e, interpreter_state *s) return e; jl_expr_t *ex = (jl_expr_t*)e; jl_value_t **args = jl_array_ptr_data(ex->args); - size_t nargs = jl_array_len(ex->args); + size_t nargs = jl_array_nrows(ex->args); jl_sym_t *head = ex->head; if (head == jl_call_sym) { return do_call(args, nargs, s); @@ -385,8 +385,8 @@ static size_t eval_phi(jl_array_t *stmts, interpreter_state *s, size_t ns, size_ // %2 = phi ... // %3 = phi (1)[1 => %a], (2)[2 => %b] // from = 1, to = closest = 2, i = 1 --> edge = 2, edge_from = 2, from = 2 - for (unsigned j = 0; j < jl_array_len(edges); ++j) { - size_t edge_from = ((int32_t*)jl_array_data(edges))[j]; // 1-indexed + for (unsigned j = 0; j < jl_array_nrows(edges); ++j) { + size_t edge_from = jl_array_data(edges, int32_t)[j]; // 1-indexed if (edge_from == from + 1) { if (edge == -1) edge = j; @@ -443,7 +443,7 @@ static size_t eval_phi(jl_array_t *stmts, interpreter_state *s, size_t ns, size_ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, int toplevel) { jl_handler_t __eh; - size_t ns = jl_array_len(stmts); + size_t ns = jl_array_nrows(stmts); jl_task_t *ct = jl_current_task; while (1) { @@ -527,7 +527,7 @@ static jl_value_t *eval_body(jl_array_t *stmts, interpreter_state *s, size_t ip, if (!jl_is_phicnode(phicnode)) break; jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(phicnode, 0); - for (size_t i = 0; i < jl_array_len(values); ++i) { + for (size_t i = 0; i < jl_array_nrows(values); ++i) { jl_value_t *val = jl_array_ptr_ref(values, i); assert(jl_is_ssavalue(val)); size_t upsilon = ((jl_ssavalue_t*)val)->id - 1; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index c7f1263af030a..f5137cd2c1f67 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -15,7 +15,6 @@ STATISTIC(EmittedCoercedUnboxes, "Number of unbox coercions emitted"); STATISTIC(EmittedUnboxes, "Number of unboxes emitted"); STATISTIC(EmittedRuntimeCalls, "Number of runtime intrinsic calls emitted"); STATISTIC(EmittedIntrinsics, "Number of intrinsic calls emitted"); -STATISTIC(Emitted_arraylen, "Number of arraylen calls emitted"); STATISTIC(Emitted_pointerref, "Number of pointerref calls emitted"); STATISTIC(Emitted_pointerset, "Number of pointerset calls emitted"); STATISTIC(Emitted_atomic_fence, "Number of atomic_fence calls emitted"); @@ -1207,15 +1206,6 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar // return emit_runtime_call(ctx, f, argv, nargs); switch (f) { - case arraylen: { - ++Emitted_arraylen; - assert(nargs == 1); - const jl_cgval_t &x = argv[0]; - jl_value_t *typ = jl_unwrap_unionall(x.typ); - if (!jl_is_datatype(typ) || ((jl_datatype_t*)typ)->name != jl_array_typename) - return emit_runtime_call(ctx, f, argv.data(), nargs); - return mark_julia_type(ctx, emit_arraylen(ctx, x), false, jl_long_type); - } case pointerref: ++Emitted_pointerref; assert(nargs == 3); diff --git a/src/intrinsics.h b/src/intrinsics.h index 93747faa74160..805aa914b5366 100644 --- a/src/intrinsics.h +++ b/src/intrinsics.h @@ -99,8 +99,6 @@ /* c interface */ \ ADD_I(cglobal, 2) \ ALIAS(llvmcall, llvmcall) \ - /* object access */ \ - ADD_I(arraylen, 1) \ /* cpu feature tests */ \ ADD_I(have_fma, 1) \ /* hidden intrinsics */ \ diff --git a/src/ircode.c b/src/ircode.c index 508326c96314a..0ad4e91986632 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -59,6 +59,7 @@ jl_value_t *jl_deser_symbol(uint8_t tag) // --- encoding --- +static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) JL_GC_DISABLED; #define jl_encode_value(s, v) jl_encode_value_((s), (jl_value_t*)(v), 0) static void tagged_root(rle_reference *rr, jl_ircode_state *s, int i) @@ -70,7 +71,7 @@ static void tagged_root(rle_reference *rr, jl_ircode_state *s, int i) static void literal_val_id(rle_reference *rr, jl_ircode_state *s, jl_value_t *v) JL_GC_DISABLED { jl_array_t *rs = s->method->roots; - int i, l = jl_array_len(rs); + int i, l = jl_array_nrows(rs); if (jl_is_symbol(v) || jl_is_concrete_type(v)) { for (i = 0; i < l; i++) { if (jl_array_ptr_ref(rs, i) == v) @@ -84,7 +85,7 @@ static void literal_val_id(rle_reference *rr, jl_ircode_state *s, jl_value_t *v) } } jl_add_method_root(s->method, jl_precompile_toplevel_module, v); - return tagged_root(rr, s, jl_array_len(rs) - 1); + return tagged_root(rr, s, jl_array_nrows(rs) - 1); } static void jl_encode_int32(jl_ircode_state *s, int32_t x) @@ -121,6 +122,44 @@ static void jl_encode_as_indexed_root(jl_ircode_state *s, jl_value_t *v) } } +static void jl_encode_memory_slice(jl_ircode_state *s, jl_genericmemory_t *mem, size_t offset, size_t len) JL_GC_DISABLED +{ + jl_datatype_t *t = (jl_datatype_t*)jl_typetagof(mem); + size_t i; + const jl_datatype_layout_t *layout = t->layout; + if (layout->arrayelem_isboxed) { + for (i = 0; i < len; i++) { + jl_value_t *e = jl_genericmemory_ptr_ref(mem, offset + i); + jl_encode_value(s, e); + } + } + else if (layout->first_ptr >= 0) { + uint16_t elsz = layout->size; + size_t j, np = layout->npointers; + const char *data = (const char*)mem->data + offset * elsz; + for (i = 0; i < len; i++) { + const char *start = data; + for (j = 0; j < np; j++) { + uint32_t ptr = jl_ptr_offset(t, j); + const jl_value_t *const *fld = &((const jl_value_t *const *)data)[ptr]; + if ((const char*)fld != start) + ios_write(s->s, start, (const char*)fld - start); + JL_GC_PROMISE_ROOTED(*fld); + jl_encode_value(s, *fld); + start = (const char*)&fld[1]; + } + data += elsz; + if (data != start) + ios_write(s->s, start, data - start); + } + } + else { + ios_write(s->s, (char*)mem->data + offset * layout->size, len * layout->size); + if (jl_genericmemory_isbitsunion(mem)) + ios_write(s->s, jl_genericmemory_typetagdata(mem) + offset, len); + } +} + static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) JL_GC_DISABLED { size_t i; @@ -203,7 +242,7 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) } else if (jl_is_expr(v)) { jl_expr_t *e = (jl_expr_t*)v; - size_t l = jl_array_len(e->args); + size_t l = jl_array_nrows(e->args); if (e->head == jl_call_sym) { if (l == 2) { write_uint8(s->s, TAG_CALL1); @@ -235,31 +274,31 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) else if (jl_is_phinode(v)) { jl_array_t *edges = (jl_array_t*)jl_fieldref_noalloc(v, 0); jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(v, 1); - size_t l = jl_array_len(edges); - if (l <= 255 && jl_array_len(values) == l) { + size_t l = jl_array_nrows(edges); + if (l <= 255 && jl_array_nrows(values) == l) { write_uint8(s->s, TAG_PHINODE); write_uint8(s->s, (uint8_t)l); } else { write_uint8(s->s, TAG_LONG_PHINODE); write_int32(s->s, l); - write_int32(s->s, jl_array_len(values)); + write_int32(s->s, jl_array_nrows(values)); } for (i = 0; i < l; i++) { - int32_t e = ((int32_t*)jl_array_data(edges))[i]; + int32_t e = jl_array_data(edges, int32_t)[i]; if (e <= 20) jl_encode_value(s, jl_box_int32(e)); else jl_encode_int32(s, e); } - l = jl_array_len(values); + l = jl_array_nrows(values); for (i = 0; i < l; i++) { jl_encode_value(s, jl_array_ptr_ref(values, i)); } } else if (jl_is_phicnode(v)) { jl_array_t *values = (jl_array_t*)jl_fieldref_noalloc(v, 0); - size_t l = jl_array_len(values); + size_t l = jl_array_nrows(values); if (l <= 255) { write_uint8(s->s, TAG_PHICNODE); write_uint8(s->s, (uint8_t)l); @@ -337,52 +376,34 @@ static void jl_encode_value_(jl_ircode_state *s, jl_value_t *v, int as_literal) } else if (as_literal && jl_is_array(v)) { jl_array_t *ar = (jl_array_t*)v; - jl_value_t *et = jl_tparam0(jl_typeof(ar)); - int isunion = jl_is_uniontype(et); - if (ar->flags.ndims == 1 && ar->elsize <= 0x1f) { + if (jl_array_ndims(ar) == 1) { write_uint8(s->s, TAG_ARRAY1D); - write_uint8(s->s, (ar->flags.ptrarray << 7) | (ar->flags.hasptr << 6) | (isunion << 5) | (ar->elsize & 0x1f)); } else { write_uint8(s->s, TAG_ARRAY); - write_uint16(s->s, ar->flags.ndims); - write_uint16(s->s, (ar->flags.ptrarray << 15) | (ar->flags.hasptr << 14) | (isunion << 13) | (ar->elsize & 0x1fff)); + write_uint16(s->s, jl_array_ndims(ar)); } - for (i = 0; i < ar->flags.ndims; i++) - jl_encode_value(s, jl_box_long(jl_array_dim(ar,i))); + for (i = 0; i < jl_array_ndims(ar); i++) + jl_encode_value(s, jl_box_long(jl_array_dim(ar, i))); jl_encode_value(s, jl_typeof(ar)); size_t l = jl_array_len(ar); - if (ar->flags.ptrarray) { - for (i = 0; i < l; i++) { - jl_value_t *e = jl_array_ptr_ref(v, i); - jl_encode_value(s, e); - } - } - else if (ar->flags.hasptr) { - const char *data = (const char*)jl_array_data(ar); - uint16_t elsz = ar->elsize; - size_t j, np = ((jl_datatype_t*)et)->layout->npointers; - for (i = 0; i < l; i++) { - const char *start = data; - for (j = 0; j < np; j++) { - uint32_t ptr = jl_ptr_offset((jl_datatype_t*)et, j); - const jl_value_t *const *fld = &((const jl_value_t *const *)data)[ptr]; - if ((const char*)fld != start) - ios_write(s->s, start, (const char*)fld - start); - JL_GC_PROMISE_ROOTED(*fld); - jl_encode_value(s, *fld); - start = (const char*)&fld[1]; - } - data += elsz; - if (data != start) - ios_write(s->s, start, data - start); - } - } - else { - ios_write(s->s, (char*)jl_array_data(ar), l * ar->elsize); - if (jl_array_isbitsunion(ar)) - ios_write(s->s, jl_array_typetagdata(ar), l); - } + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(ar->ref.mem))->layout; + size_t offset; + if (layout->arrayelem_isunion || layout->size == 0) + offset = (uintptr_t)ar->ref.data; + else + offset = (char*)ar->ref.data - (char*)ar->ref.mem->data; + jl_encode_memory_slice(s, ar->ref.mem, offset, l); + } + else if (as_literal && jl_is_genericmemory(v)) { + jl_genericmemory_t* m = (jl_genericmemory_t*)v; + write_uint8(s->s, TAG_MEMORYT); + jl_encode_value(s, (jl_datatype_t*)jl_typetagof(v)); + jl_encode_value(s, jl_box_long(m->length)); + jl_encode_memory_slice(s, m, 0, m->length); + } + else if (as_literal && jl_is_layout_opaque(((jl_datatype_t*)jl_typeof(v))->layout)) { + assert(0 && "not legal to store this as literal"); } else if (as_literal || jl_is_uniontype(v) || jl_is_newvarnode(v) || jl_is_linenode(v) || jl_is_upsilonnode(v) || jl_is_pinode(v) || jl_is_slotnumber(v) || jl_is_ssavalue(v) || @@ -466,52 +487,27 @@ static jl_value_t *jl_decode_value_svec(jl_ircode_state *s, uint8_t tag) JL_GC_D return (jl_value_t*)sv; } -static jl_value_t *jl_decode_value_array(jl_ircode_state *s, uint8_t tag) JL_GC_DISABLED +static jl_value_t *jl_decode_value_memory(jl_ircode_state *s, jl_value_t *mty, size_t nel) JL_GC_DISABLED { - int16_t i, ndims; - int isptr, isunion, hasptr, elsize; - if (tag == TAG_ARRAY1D) { - ndims = 1; - elsize = read_uint8(s->s); - isptr = (elsize >> 7) & 1; - hasptr = (elsize >> 6) & 1; - isunion = (elsize >> 5) & 1; - elsize = elsize & 0x1f; - } - else { - ndims = read_uint16(s->s); - elsize = read_uint16(s->s); - isptr = (elsize >> 15) & 1; - hasptr = (elsize >> 14) & 1; - isunion = (elsize >> 13) & 1; - elsize = elsize & 0x1fff; - } - size_t *dims = (size_t*)alloca(ndims * sizeof(size_t)); - for (i = 0; i < ndims; i++) { - dims[i] = jl_unbox_long(jl_decode_value(s)); - } - jl_array_t *a = jl_new_array_for_deserialization( - (jl_value_t*)NULL, ndims, dims, !isptr, hasptr, isunion, elsize); - jl_value_t *aty = jl_decode_value(s); - jl_set_typeof(a, aty); - if (a->flags.ptrarray) { - jl_value_t **data = (jl_value_t**)jl_array_data(a); - size_t i, numel = jl_array_len(a); + jl_genericmemory_t *m = jl_alloc_genericmemory(mty, nel); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)mty)->layout; + if (layout->arrayelem_isboxed) { + jl_value_t **data = (jl_value_t**)m->data; + size_t i, numel = m->length; for (i = 0; i < numel; i++) { data[i] = jl_decode_value(s); } - assert(jl_astaggedvalue(a)->bits.gc == GC_CLEAN); // gc is disabled - } - else if (a->flags.hasptr) { - size_t i, numel = jl_array_len(a); - char *data = (char*)jl_array_data(a); - uint16_t elsz = a->elsize; - jl_datatype_t *et = (jl_datatype_t*)jl_tparam0(jl_typeof(a)); - size_t j, np = et->layout->npointers; + assert(jl_astaggedvalue(m)->bits.gc == GC_CLEAN); // gc is disabled + } + else if (layout->first_ptr >= 0) { + size_t i, numel = m->length; + char *data = (char*)m->data; + uint16_t elsz = layout->size; + size_t j, np = layout->npointers; for (i = 0; i < numel; i++) { char *start = data; for (j = 0; j < np; j++) { - uint32_t ptr = jl_ptr_offset(et, j); + uint32_t ptr = jl_ptr_offset((jl_datatype_t*)mty, j); jl_value_t **fld = &((jl_value_t**)data)[ptr]; if ((char*)fld != start) ios_readall(s->s, start, (const char*)fld - start); @@ -522,13 +518,39 @@ static jl_value_t *jl_decode_value_array(jl_ircode_state *s, uint8_t tag) JL_GC_ if (data != start) ios_readall(s->s, start, data - start); } - assert(jl_astaggedvalue(a)->bits.gc == GC_CLEAN); // gc is disabled + assert(jl_astaggedvalue(m)->bits.gc == GC_CLEAN); // gc is disabled } else { - size_t extra = jl_array_isbitsunion(a) ? jl_array_len(a) : 0; - size_t tot = jl_array_len(a) * a->elsize + extra; - ios_readall(s->s, (char*)jl_array_data(a), tot); + size_t extra = jl_genericmemory_isbitsunion(m) ? m->length : 0; + size_t tot = m->length * layout->size + extra; + ios_readall(s->s, (char*)m->data, tot); } + return (jl_value_t*)m; +} + +JL_DLLEXPORT jl_array_t *jl_alloc_array_nd(jl_value_t *atype, size_t *dims, size_t ndims); + +static jl_value_t *jl_decode_value_array(jl_ircode_state *s, uint8_t tag) JL_GC_DISABLED +{ + int16_t i, ndims; + if (tag == TAG_ARRAY1D) + ndims = 1; + else + ndims = read_uint16(s->s); + size_t *dims = (size_t*)alloca(ndims * sizeof(size_t)); + size_t len = 1; + for (i = 0; i < ndims; i++) { + dims[i] = jl_unbox_long(jl_decode_value(s)); + len *= dims[i]; + } + jl_value_t *aty = jl_decode_value(s); + jl_array_t *a = jl_alloc_array_nd(aty, dims, ndims); + a->ref.mem = (jl_genericmemory_t*)jl_decode_value_memory(s, jl_field_type_concrete((jl_datatype_t*)jl_field_type_concrete((jl_datatype_t*)aty, 0), 1), len); + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(a->ref.mem))->layout; + if (layout->arrayelem_isunion || layout->size == 0) + a->ref.data = (void*)0; + else + a->ref.data = a->ref.mem->data; return (jl_value_t*)a; } @@ -553,7 +575,7 @@ static jl_value_t *jl_decode_value_expr(jl_ircode_state *s, uint8_t tag) JL_GC_D if (head == NULL) head = (jl_sym_t*)jl_decode_value(s); jl_expr_t *e = jl_exprn(head, len); - jl_value_t **data = (jl_value_t**)(e->args->data); + jl_value_t **data = jl_array_ptr_data(e->args); for (i = 0; i < len; i++) { data[i] = jl_decode_value(s); } @@ -573,11 +595,11 @@ static jl_value_t *jl_decode_value_phi(jl_ircode_state *s, uint8_t tag) JL_GC_DI jl_array_t *e = jl_alloc_array_1d(jl_array_int32_type, len_e); jl_array_t *v = jl_alloc_vec_any(len_v); jl_value_t *phi = jl_new_struct(jl_phinode_type, e, v); - int32_t *data_e = (int32_t*)(e->data); + int32_t *data_e = jl_array_data(e, int32_t); for (i = 0; i < len_e; i++) { data_e[i] = jl_unbox_int32(jl_decode_value(s)); } - jl_value_t **data_v = (jl_value_t**)(v->data); + jl_value_t **data_v = jl_array_ptr_data(v); for (i = 0; i < len_v; i++) { data_v[i] = jl_decode_value(s); } @@ -593,7 +615,7 @@ static jl_value_t *jl_decode_value_phic(jl_ircode_state *s, uint8_t tag) JL_GC_D len = read_int32(s->s); jl_array_t *v = jl_alloc_vec_any(len); jl_value_t *phic = jl_new_struct(jl_phicnode_type, v); - jl_value_t **data = (jl_value_t**)(v->data); + jl_value_t **data = jl_array_ptr_data(v); for (i = 0; i < len; i++) { data[i] = jl_decode_value(s); } @@ -678,6 +700,8 @@ static jl_value_t *jl_decode_value(jl_ircode_state *s) JL_GC_DISABLED return v; case TAG_ARRAY: JL_FALLTHROUGH; case TAG_ARRAY1D: return jl_decode_value_array(s, tag); + case TAG_MEMORYT: + return jl_decode_value_memory(s, jl_decode_value(s), jl_unbox_long(jl_decode_value(s))); case TAG_EXPR: JL_FALLTHROUGH; case TAG_LONG_EXPR: JL_FALLTHROUGH; case TAG_CALL1: JL_FALLTHROUGH; @@ -792,10 +816,10 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) write_uint8(s.s, code->purity.bits); write_uint16(s.s, code->inlining_cost); - size_t nslots = jl_array_len(code->slotflags); + size_t nslots = jl_array_nrows(code->slotflags); assert(nslots >= m->nargs && nslots < INT32_MAX); // required by generated functions write_int32(s.s, nslots); - ios_write(s.s, (char*)jl_array_data(code->slotflags), nslots); + ios_write(s.s, jl_array_data(code->slotflags, const char), nslots); // N.B.: The layout of everything before this point is explicitly referenced // by the various jl_ir_ accessors. Make sure to adjust those if you change @@ -825,20 +849,20 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) else jl_encode_value(&s, jl_nothing); - size_t nstmt = jl_array_len(code->code); - assert(nstmt == jl_array_len(code->codelocs)); - if (jl_array_len(code->linetable) < 256) { + size_t nstmt = jl_array_nrows(code->code); + assert(nstmt == jl_array_nrows(code->codelocs)); + if (jl_array_nrows(code->linetable) < 256) { for (i = 0; i < nstmt; i++) { - write_uint8(s.s, ((int32_t*)jl_array_data(code->codelocs))[i]); + write_uint8(s.s, jl_array_data(code->codelocs, int32_t)[i]); } } - else if (jl_array_len(code->linetable) < 65536) { + else if (jl_array_nrows(code->linetable) < 65536) { for (i = 0; i < nstmt; i++) { - write_uint16(s.s, ((int32_t*)jl_array_data(code->codelocs))[i]); + write_uint16(s.s, jl_array_data(code->codelocs, int32_t)[i]); } } else { - ios_write(s.s, (char*)jl_array_data(code->codelocs), nstmt * sizeof(int32_t)); + ios_write(s.s, (char*)jl_array_data(code->codelocs, int32_t), nstmt * sizeof(int32_t)); } write_uint8(s.s, s.relocatability); @@ -846,7 +870,7 @@ JL_DLLEXPORT jl_string_t *jl_compress_ir(jl_method_t *m, jl_code_info_t *code) ios_flush(s.s); jl_string_t *v = jl_pchar_to_string(s.s->buf, s.s->size); ios_close(s.s); - if (jl_array_len(m->roots) == 0) { + if (jl_array_nrows(m->roots) == 0) { m->roots = NULL; } JL_GC_PUSH1(&v); @@ -892,7 +916,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t size_t nslots = read_int32(&src); code->slotflags = jl_alloc_array_1d(jl_array_uint8_type, nslots); - ios_readall(s.s, (char*)jl_array_data(code->slotflags), nslots); + ios_readall(s.s, jl_array_data(code->slotflags, char), nslots); for (i = 0; i < 6; i++) { if (i == 1) // skip codelocs @@ -909,20 +933,20 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t slotnames = m->slot_syms; code->slotnames = jl_uncompress_argnames(slotnames); - size_t nstmt = jl_array_len(code->code); + size_t nstmt = jl_array_nrows(code->code); code->codelocs = (jl_value_t*)jl_alloc_array_1d(jl_array_int32_type, nstmt); - if (jl_array_len(code->linetable) < 256) { + if (jl_array_nrows(code->linetable) < 256) { for (i = 0; i < nstmt; i++) { - ((int32_t*)jl_array_data(code->codelocs))[i] = read_uint8(s.s); + jl_array_data(code->codelocs, int32_t)[i] = read_uint8(s.s); } } - else if (jl_array_len(code->linetable) < 65536) { + else if (jl_array_nrows(code->linetable) < 65536) { for (i = 0; i < nstmt; i++) { - ((int32_t*)jl_array_data(code->codelocs))[i] = read_uint16(s.s); + jl_array_data(code->codelocs, int32_t)[i] = read_uint16(s.s); } } else { - ios_readall(s.s, (char*)jl_array_data(code->codelocs), nstmt * sizeof(int32_t)); + ios_readall(s.s, (char*)jl_array_data(code->codelocs, int32_t), nstmt * sizeof(int32_t)); } (void) read_uint8(s.s); // relocatability @@ -984,7 +1008,7 @@ JL_DLLEXPORT uint16_t jl_ir_inlining_cost(jl_string_t *data) JL_DLLEXPORT jl_value_t *jl_compress_argnames(jl_array_t *syms) { - size_t nsyms = jl_array_len(syms); + size_t nsyms = jl_array_nrows(syms); size_t i, len = 0; for (i = 0; i < nsyms; i++) { jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(syms, i); @@ -1012,7 +1036,7 @@ JL_DLLEXPORT ssize_t jl_ir_nslots(jl_value_t *data) { if (jl_is_code_info(data)) { jl_code_info_t *func = (jl_code_info_t*)data; - return jl_array_len(func->slotnames); + return jl_array_nrows(func->slotnames); } else { assert(jl_is_string(data)); @@ -1024,8 +1048,10 @@ JL_DLLEXPORT ssize_t jl_ir_nslots(jl_value_t *data) JL_DLLEXPORT uint8_t jl_ir_slotflag(jl_string_t *data, size_t i) { assert(i < jl_ir_nslots(data)); - if (jl_is_code_info(data)) - return ((uint8_t*)((jl_code_info_t*)data)->slotflags->data)[i]; + if (jl_is_code_info(data)) { + jl_array_t *slotflags = ((jl_code_info_t*)data)->slotflags; + return jl_array_data(slotflags, uint8_t)[i]; + } assert(jl_is_string(data)); return jl_string_data(data)[2 + sizeof(uint16_t) + sizeof(int32_t) + i]; } @@ -1118,6 +1144,8 @@ void jl_init_serializer(void) jl_uint32_type, jl_uint64_type, jl_type_type_mt, jl_nonfunction_mt, jl_opaque_closure_type, + jl_memory_any_type, //TODO? jl_memory_symbol_type, + jl_memory_uint8_type, //TODO? jl_memory_int32_type, ct->ptls->root_task, @@ -1135,6 +1163,7 @@ void jl_init_serializer(void) deser_tag[TAG_SLOTNUMBER] = (jl_value_t*)jl_slotnumber_type; deser_tag[TAG_SVEC] = (jl_value_t*)jl_simplevector_type; deser_tag[TAG_ARRAY] = (jl_value_t*)jl_array_type; + deser_tag[TAG_MEMORYT] = (jl_value_t*)jl_genericmemory_type; deser_tag[TAG_EXPR] = (jl_value_t*)jl_expr_type; deser_tag[TAG_PHINODE] = (jl_value_t*)jl_phinode_type; deser_tag[TAG_PHICNODE] = (jl_value_t*)jl_phicnode_type; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index f0360c6addc95..cd4e2d1847a00 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -436,7 +436,7 @@ void jl_extern_c_impl(jl_value_t *declrt, jl_tupletype_t *sigt) jl_type_error("@ccallable", (jl_value_t*)jl_anytuple_type_type, (jl_value_t*)sigt); // check that f is a guaranteed singleton type jl_datatype_t *ft = (jl_datatype_t*)jl_tparam0(sigt); - if (!jl_is_datatype(ft) || ft->instance == NULL) + if (!jl_is_datatype(ft) || !jl_is_datatype_singleton(ft)) jl_error("@ccallable: function object must be a singleton"); // compute / validate return type diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 2acde218a104c..e43255f3dcea6 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -6,6 +6,7 @@ XX(jl_abstractstring_type) \ XX(jl_an_empty_string) \ XX(jl_an_empty_vec_any) \ + XX(jl_an_empty_memory_any) \ XX(jl_anytuple_type) \ XX(jl_anytuple_type_type) \ XX(jl_any_type) \ @@ -63,7 +64,17 @@ XX(jl_llvmpointer_typename) \ XX(jl_loaderror_type) \ XX(jl_main_module) \ + XX(jl_memory_any_type) \ XX(jl_memory_exception) \ + XX(jl_genericmemory_type) \ + XX(jl_genericmemory_typename) \ + XX(jl_memory_uint8_type) \ + XX(jl_memory_uint64_type) \ + XX(jl_memoryref_any_type) \ + XX(jl_genericmemoryref_type) \ + XX(jl_genericmemoryref_typename) \ + XX(jl_memoryref_uint8_type) \ + XX(jl_memoryref_uint64_type) \ XX(jl_methoderror_type) \ XX(jl_method_instance_type) \ XX(jl_method_match_type) \ diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 228c88109f322..55388fbfb9900 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -6,8 +6,6 @@ XX(jl_adopt_thread) \ XX(jl_alignment) \ XX(jl_alloc_array_1d) \ - XX(jl_alloc_array_2d) \ - XX(jl_alloc_array_3d) \ XX(jl_alloc_string) \ XX(jl_alloc_svec) \ XX(jl_alloc_svec_uninit) \ @@ -21,26 +19,15 @@ XX(jl_apply_type1) \ XX(jl_apply_type2) \ XX(jl_argument_datatype) \ - XX(jl_arraylen) \ - XX(jl_arrayref) \ - XX(jl_arrayset) \ - XX(jl_arrayunset) \ XX(jl_array_copy) \ - XX(jl_array_del_at) \ - XX(jl_array_del_beg) \ XX(jl_array_del_end) \ XX(jl_array_eltype) \ - XX(jl_array_grow_at) \ - XX(jl_array_grow_beg) \ XX(jl_array_grow_end) \ - XX(jl_array_isassigned) \ XX(jl_array_ptr) \ XX(jl_array_ptr_1d_append) \ XX(jl_array_ptr_1d_push) \ XX(jl_array_ptr_copy) \ XX(jl_array_rank) \ - XX(jl_array_size) \ - XX(jl_array_sizehint) \ XX(jl_array_to_string) \ XX(jl_array_typetagdata) \ XX(jl_array_validate_dims) \ @@ -342,7 +329,6 @@ XX(jl_module_uuid) \ XX(jl_native_alignment) \ XX(jl_nb_available) \ - XX(jl_new_array) \ XX(jl_new_bits) \ XX(jl_new_codeinst) \ XX(jl_new_code_info_uninit) \ @@ -393,7 +379,6 @@ XX(jl_profile_maxlen_data) \ XX(jl_profile_start_timer) \ XX(jl_profile_stop_timer) \ - XX(jl_ptrarrayref) \ XX(jl_ptr_to_array) \ XX(jl_ptr_to_array_1d) \ XX(jl_queue_work) \ @@ -404,7 +389,6 @@ XX(jl_read_verify_header) \ XX(jl_realloc) \ XX(jl_register_newmeth_tracer) \ - XX(jl_reshape_array) \ XX(jl_resolve_globals_in_ir) \ XX(jl_restore_excstack) \ XX(jl_restore_incremental) \ diff --git a/src/jlapi.c b/src/jlapi.c index 0dffaac627288..f0bf5162eacc7 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -41,12 +41,12 @@ JL_DLLEXPORT void jl_set_ARGS(int argc, char **argv) jl_set_const(jl_core_module, jl_symbol("ARGS"), (jl_value_t*)args); JL_GC_POP(); } - assert(jl_array_len(args) == 0); + assert(jl_array_nrows(args) == 0); jl_array_grow_end(args, argc); int i; for (i = 0; i < argc; i++) { jl_value_t *s = (jl_value_t*)jl_cstr_to_string(argv[i]); - jl_arrayset(args, s, i); + jl_array_ptr_set(args, i, s); } } } @@ -165,8 +165,11 @@ JL_DLLEXPORT int jl_array_rank(jl_value_t *a) return jl_array_ndims(a); } -JL_DLLEXPORT size_t jl_array_size(jl_value_t *a, int d) +JL_DLLEXPORT size_t jl_array_size(jl_array_t *a, int d) { + // n.b this functions only use was to violate the vector abstraction, so we have to continue to emulate that + if (d >= jl_array_ndims(a)) + return a->ref.mem->length; return jl_array_dim(a, d); } diff --git a/src/jltypes.c b/src/jltypes.c index f38197e49353d..4ee7fbbaaee2b 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -65,7 +65,9 @@ static int layout_uses_free_typevars(jl_value_t *v, jl_typeenv_t *env) } if (jl_is_datatype(v)) { jl_datatype_t *dt = (jl_datatype_t*)v; - if (dt->layout || dt->isconcretetype || !dt->name->mayinlinealloc) + if (dt->isconcretetype) + return 0; + if (dt->layout || !dt->name->mayinlinealloc) return 0; if (dt->name == jl_namedtuple_typename) return layout_uses_free_typevars(jl_tparam0(dt), env) || layout_uses_free_typevars(jl_tparam1(dt), env); @@ -290,7 +292,13 @@ JL_DLLEXPORT int jl_has_typevar_from_unionall(jl_value_t *t, jl_unionall_t *ua) int jl_has_fixed_layout(jl_datatype_t *dt) { - if (dt->layout || dt->isconcretetype) + if (dt->isconcretetype) + return 1; + if (jl_is_genericmemory_type(dt)) { // GenericMemory{isatomic,T} uses T for final layout, which is a parameter not a field however + // optionally: return !layout_uses_free_typevars(jl_tparam1(dt), env); + return 0; + } + if (dt->layout) return 1; if (dt->name->abstract) return 0; @@ -313,15 +321,15 @@ int jl_has_fixed_layout(jl_datatype_t *dt) int jl_type_mappable_to_c(jl_value_t *ty) { assert(!jl_is_typevar(ty) && jl_is_type(ty)); + if (jl_is_array_type(ty) || jl_is_genericmemory_type(ty) || + (jl_is_datatype(ty) && ((jl_datatype_t*)ty)->layout != NULL && + jl_is_layout_opaque(((jl_datatype_t*)ty)->layout))) + return 1; // as boxed if (jl_is_structtype(ty)) return jl_has_fixed_layout((jl_datatype_t*)ty) && ((jl_datatype_t*)ty)->name->atomicfields == NULL; if (jl_is_primitivetype(ty)) - return 1; - if (ty == (jl_value_t*)jl_any_type || ty == (jl_value_t*)jl_bottom_type) - return 1; // as boxed - if (jl_is_abstract_ref_type(ty) || jl_is_array_type(ty) || - (jl_is_datatype(ty) && ((jl_datatype_t*)ty)->layout != NULL && - jl_is_layout_opaque(((jl_datatype_t*)ty)->layout))) + return 1; // as isbits + if (ty == (jl_value_t*)jl_any_type || ty == (jl_value_t*)jl_bottom_type || jl_is_abstract_ref_type(ty)) return 1; // as boxed return 0; // refuse to map Union and UnionAll to C } @@ -2080,6 +2088,11 @@ static jl_value_t *inst_datatype_inner(jl_datatype_t *dt, jl_svec_t *p, jl_value ndt->types = jl_emptysvec; // XXX: this is essentially always false } } + else if (tn == jl_genericmemoryref_typename || tn == jl_genericmemory_typename) { + jl_value_t *isatomic = jl_svecref(p, 0); + if (!jl_is_typevar(isatomic) && !jl_is_symbol(isatomic)) + jl_type_error_rt("GenericMemory", "isatomic parameter", (jl_value_t*)jl_symbol_type, isatomic); + } jl_datatype_t *primarydt = ((jl_datatype_t*)jl_unwrap_unionall(tn->wrapper)); jl_precompute_memoized_dt(ndt, cacheable); @@ -2482,7 +2495,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! if (partial == NULL) return; if (n == 0) { - assert(jl_array_len(partial) == 0); + assert(jl_array_nrows(partial) == 0); return; } @@ -2493,7 +2506,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! env[i].prev = i == 0 ? NULL : &env[i - 1]; } - for (j = 0; j < jl_array_len(partial); j++) { + for (j = 0; j < jl_array_nrows(partial); j++) { jl_datatype_t *ndt = (jl_datatype_t*)jl_array_ptr_ref(partial, j); if (ndt == NULL) continue; @@ -2506,7 +2519,7 @@ void jl_reinstantiate_inner_types(jl_datatype_t *t) // can throw! } if (t->types != jl_emptysvec) { - for (j = 0; j < jl_array_len(partial); j++) { + for (j = 0; j < jl_array_nrows(partial); j++) { jl_datatype_t *ndt = (jl_datatype_t*)jl_array_ptr_ref(partial, j); if (ndt == NULL) continue; @@ -2849,7 +2862,21 @@ void jl_init_types(void) JL_GC_DISABLED jl_function_type->name->mt = NULL; // subtypes of Function have independent method tables jl_builtin_type->name->mt = NULL; // so they don't share the Any type table - jl_svec_t *tv = jl_svec2(tvar("T"), tvar("N")); + jl_svec_t *tv; + tv = jl_svec1(tvar("T")); + jl_ref_type = (jl_unionall_t*) + jl_new_abstracttype((jl_value_t*)jl_symbol("Ref"), core, jl_any_type, tv)->name->wrapper; + + tv = jl_svec1(tvar("T")); + jl_pointer_typename = + jl_new_primitivetype((jl_value_t*)jl_symbol("Ptr"), core, + (jl_datatype_t*)jl_apply_type((jl_value_t*)jl_ref_type, jl_svec_data(tv), 1), tv, + sizeof(void*)*8)->name; + jl_pointer_type = (jl_unionall_t*)jl_pointer_typename->wrapper; + jl_value_t *pointer_void = jl_apply_type1((jl_value_t*)jl_pointer_type, (jl_value_t*)jl_nothing_type); + jl_voidpointer_type = (jl_datatype_t*)pointer_void; + + tv = jl_svec2(tvar("T"), tvar("N")); jl_abstractarray_type = (jl_unionall_t*) jl_new_abstracttype((jl_value_t*)jl_symbol("AbstractArray"), core, jl_any_type, tv)->name->wrapper; @@ -2860,13 +2887,44 @@ void jl_init_types(void) JL_GC_DISABLED (jl_datatype_t*)jl_apply_type((jl_value_t*)jl_abstractarray_type, jl_svec_data(tv), 2), tv)->name->wrapper; + tv = jl_svec2(tvar("isatomic"), tvar("T")); + jl_datatype_t *jl_memory_supertype = (jl_datatype_t*)jl_apply_type2((jl_value_t*)jl_densearray_type, jl_svecref(tv, 1), jl_box_long(1)); + jl_datatype_t *memory_datatype = + jl_new_datatype(jl_symbol("GenericMemory"), core, jl_memory_supertype, tv, + jl_perm_symsvec(2, "length", "data"), + jl_svec(2, jl_long_type, pointer_void), + jl_emptysvec, 0, 1, 2); + jl_genericmemory_typename = memory_datatype->name; + jl_genericmemory_type = (jl_unionall_t*)jl_genericmemory_typename->wrapper; + const static uint32_t memory_constfields[1] = { 0x00000003 }; // (1<<1)|(1<<0) + memory_datatype->name->constfields = memory_constfields; + memory_datatype->ismutationfree = 0; + + jl_datatype_t *jl_memoryref_supertype = (jl_datatype_t*)jl_apply_type1((jl_value_t*)jl_ref_type, jl_svecref(tv, 1)); + jl_datatype_t *memoryref_datatype = + jl_new_datatype(jl_symbol("MemoryRef"), core, jl_memoryref_supertype, tv, + jl_perm_symsvec(2, "ptr", "mem"), + jl_svec(2, pointer_void, memory_datatype), + jl_emptysvec, 0, 0, 2); + jl_genericmemoryref_typename = memoryref_datatype->name; + jl_genericmemoryref_type = (jl_unionall_t*)jl_genericmemoryref_typename->wrapper; + memoryref_datatype->ismutationfree = 0; + + jl_memory_any_type = jl_apply_type2((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_any_type); + jl_memory_uint8_type = jl_apply_type2((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint8_type); + jl_memoryref_any_type = jl_apply_type2((jl_value_t*)jl_genericmemoryref_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_any_type); + jl_memoryref_uint8_type = jl_apply_type2((jl_value_t*)jl_genericmemoryref_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint8_type); + tv = jl_svec2(tvar("T"), tvar("N")); - jl_array_type = (jl_unionall_t*) - jl_new_datatype(jl_symbol("Array"), core, + jl_array_typename = jl_new_datatype(jl_symbol("Array"), core, (jl_datatype_t*)jl_apply_type((jl_value_t*)jl_densearray_type, jl_svec_data(tv), 2), - tv, jl_emptysvec, jl_emptysvec, jl_emptysvec, 0, 1, 0)->name->wrapper; - jl_array_typename = ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_array_type))->name; - jl_compute_field_offsets((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_array_type)); + tv, + jl_perm_symsvec(2, "ref", "size"), + jl_svec(2, + jl_apply_type2((jl_value_t*)jl_genericmemoryref_type, (jl_value_t*)jl_not_atomic_sym, jl_svecref(tv, 0)), + jl_apply_type1((jl_value_t*)jl_tuple_type, (jl_value_t*)jl_wrap_vararg((jl_value_t*)jl_long_type, jl_svecref(tv, 1), 0))), + jl_emptysvec, 0, 1, 2)->name; + jl_array_type = (jl_unionall_t*)jl_array_typename->wrapper; jl_array_any_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_any_type, jl_box_long(1)); jl_array_symbol_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_symbol_type, jl_box_long(1)); @@ -2875,8 +2933,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_array_int32_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_int32_type, jl_box_long(1)); jl_array_uint64_type = jl_apply_type2((jl_value_t*)jl_array_type, (jl_value_t*)jl_uint64_type, jl_box_long(1)); jl_an_empty_vec_any = (jl_value_t*)jl_alloc_vec_any(0); // used internally - jl_atomic_store_relaxed(&jl_nonfunction_mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); - jl_atomic_store_relaxed(&jl_type_type_mt->leafcache, (jl_array_t*)jl_an_empty_vec_any); + jl_an_empty_memory_any = (jl_value_t*)jl_alloc_memory_any(0); // used internally + jl_atomic_store_relaxed(&jl_nonfunction_mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&jl_type_type_mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_expr_type = jl_new_datatype(jl_symbol("Expr"), core, @@ -3185,17 +3244,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_intrinsic_type = jl_new_primitivetype((jl_value_t*)jl_symbol("IntrinsicFunction"), core, jl_builtin_type, jl_emptysvec, 32); - tv = jl_svec1(tvar("T")); - jl_ref_type = (jl_unionall_t*) - jl_new_abstracttype((jl_value_t*)jl_symbol("Ref"), core, jl_any_type, tv)->name->wrapper; - - tv = jl_svec1(tvar("T")); - jl_pointer_type = (jl_unionall_t*) - jl_new_primitivetype((jl_value_t*)jl_symbol("Ptr"), core, - (jl_datatype_t*)jl_apply_type((jl_value_t*)jl_ref_type, jl_svec_data(tv), 1), tv, - sizeof(void*)*8)->name->wrapper; - jl_pointer_typename = ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_pointer_type))->name; - // LLVMPtr{T, AS} where {T, AS} jl_tvar_t *elvar = tvar("T"); tv = jl_svec2(elvar, tvar("AS")); @@ -3283,8 +3331,6 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(3, jl_module_type, jl_symbol_type, jl_binding_type), jl_emptysvec, 0, 0, 3); - jl_value_t *pointer_void = jl_apply_type1((jl_value_t*)jl_pointer_type, (jl_value_t*)jl_nothing_type); - jl_voidpointer_type = (jl_datatype_t*)pointer_void; tv = jl_svec2(tvar("A"), tvar("R")); jl_opaque_closure_type = (jl_unionall_t*)jl_new_datatype(jl_symbol("OpaqueClosure"), core, jl_function_type, tv, // N.B.: OpaqueClosure call code relies on specptr being field 5. @@ -3345,6 +3391,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_symbol_type->ismutationfree = jl_symbol_type->isidentityfree = 1; jl_simplevector_type->ismutationfree = jl_simplevector_type->isidentityfree = 1; jl_datatype_type->ismutationfree = 1; + assert(((jl_datatype_t*)jl_array_any_type)->ismutationfree == 0); + assert(((jl_datatype_t*)jl_array_uint8_type)->ismutationfree == 0); // Technically not ismutationfree, but there's a separate system to deal // with mutations for global state. @@ -3352,15 +3400,6 @@ void jl_init_types(void) JL_GC_DISABLED // Module object identity is determined by its name and parent name. jl_module_type->isidentityfree = 1; - // Array's mutable data is hidden, so we need to override it - ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_array_type))->ismutationfree = 0; - ((jl_datatype_t*)jl_array_any_type)->ismutationfree = 0; - ((jl_datatype_t*)jl_array_symbol_type)->ismutationfree = 0; - ((jl_datatype_t*)jl_array_uint8_type)->ismutationfree = 0; - ((jl_datatype_t*)jl_array_uint32_type)->ismutationfree = 0; - ((jl_datatype_t*)jl_array_int32_type)->ismutationfree = 0; - ((jl_datatype_t*)jl_array_uint64_type)->ismutationfree = 0; - // override the preferred layout for a couple types jl_lineinfonode_type->name->mayinlinealloc = 0; // FIXME: assumed to be a pointer by codegen diff --git a/src/julia.h b/src/julia.h index 1b3b2c7a70138..9a65eefb80270 100644 --- a/src/julia.h +++ b/src/julia.h @@ -120,7 +120,8 @@ JL_DLLEXPORT jl_taggedvalue_t *_jl_astaggedvalue(jl_value_t *v JL_PROPAGATES_ROO jl_value_t *_jl_valueof(jl_taggedvalue_t *tv JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; #define jl_valueof(v) _jl_valueof((jl_taggedvalue_t*)(v)) JL_DLLEXPORT jl_value_t *_jl_typeof(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; -#define jl_typeof(v) _jl_typeof((jl_value_t*)(v)) +#define jl_typeof(v) (_jl_typeof((jl_value_t*)(v))) +#define jl_typetagof(v) ((uintptr_t)_jl_typeof((jl_value_t*)(v))) #else #define jl_astaggedvalue(v) \ ((jl_taggedvalue_t*)((char*)(v) - sizeof(jl_taggedvalue_t))) @@ -128,6 +129,8 @@ JL_DLLEXPORT jl_value_t *_jl_typeof(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFE ((jl_value_t*)((char*)(v) + sizeof(jl_taggedvalue_t))) #define jl_typeof(v) \ jl_to_typeof(jl_typetagof(v)) +#define jl_typetagof(v) \ + ((jl_astaggedvalue(v)->header) & ~(uintptr_t)15) #endif static inline void jl_set_typeof(void *v, void *t) JL_NOTSAFEPOINT { @@ -135,8 +138,6 @@ static inline void jl_set_typeof(void *v, void *t) JL_NOTSAFEPOINT jl_taggedvalue_t *tag = jl_astaggedvalue(v); jl_atomic_store_relaxed((_Atomic(jl_value_t*)*)&tag->type, (jl_value_t*)t); } -#define jl_typetagof(v) \ - ((jl_astaggedvalue(v)->header) & ~(uintptr_t)15) #define jl_typeis(v,t) (jl_typeof(v)==(jl_value_t*)(t)) #define jl_typetagis(v,t) (jl_typetagof(v)==(uintptr_t)(t)) #define jl_set_typetagof(v,t,gc) (jl_set_typeof((v), (void*)(((uintptr_t)(t) << 4) | (gc)))) @@ -167,47 +168,32 @@ typedef struct { // jl_value_t *data[]; } jl_svec_t; -typedef struct { - /* - how - allocation style - 0 = data is inlined, or a foreign pointer we don't manage - 1 = julia-allocated buffer that needs to be marked - 2 = malloc-allocated pointer this array object manages - 3 = has a pointer to the object that owns the data - */ - uint16_t how:2; - uint16_t ndims:9; - uint16_t pooled:1; - uint16_t ptrarray:1; // representation is pointer array - uint16_t hasptr:1; // representation has embedded pointers - uint16_t isshared:1; // data is shared by multiple Arrays - uint16_t isaligned:1; // data allocated with memalign -} jl_array_flags_t; +JL_EXTENSION typedef struct { + JL_DATA_TYPE + size_t length; + void *data; +#ifndef _P64 + size_t padding[2]; +#endif + // followed by padding and inline data, or owner pointer + // union { + // jl_value_t *owner + // T inl[]; + // }; +} jl_genericmemory_t; JL_EXTENSION typedef struct { JL_DATA_TYPE void *data; - size_t length; - jl_array_flags_t flags; - uint16_t elsize; // element size including alignment (dim 1 memory stride) - uint32_t offset; // for 1-d only. does not need to get big. - size_t nrows; - union { - // 1d - size_t maxsize; - // Nd - size_t ncols; - }; - // other dim sizes go here for ndims > 2 + jl_genericmemory_t *mem; +} jl_genericmemoryref_t; - // followed by alignment padding and inline data, or owner pointer +JL_EXTENSION typedef struct { + JL_DATA_TYPE + jl_genericmemoryref_t ref; + size_t dimsize[]; // length for 1-D, otherwise length is mem->length } jl_array_t; -// compute # of extra words needed to store dimensions -STATIC_INLINE int jl_array_ndimwords(uint32_t ndims) JL_NOTSAFEPOINT -{ - return (ndims < 3 ? 0 : ndims-2); -} typedef struct _jl_datatype_t jl_tupletype_t; struct _jl_code_instance_t; @@ -541,7 +527,10 @@ typedef struct { uint16_t alignment; // strictest alignment over all fields uint16_t haspadding : 1; // has internal undefined bytes uint16_t fielddesc_type : 2; // 0 -> 8, 1 -> 16, 2 -> 32, 3 -> foreign type - uint16_t padding : 13; + // metadata bit only for GenericMemory eltype layout + uint16_t arrayelem_isboxed : 1; + uint16_t arrayelem_isunion : 1; + uint16_t padding : 11; // union { // jl_fielddesc8_t field8[nfields]; // jl_fielddesc16_t field16[nfields]; @@ -664,10 +653,10 @@ typedef struct _jl_typemap_level_t { // next split may be on Type{T} as LeafTypes then TypeName's parents up to Any // next split may be on LeafType // next split may be on TypeName - _Atomic(jl_array_t*) arg1; // contains LeafType (in a map of non-abstract TypeName) - _Atomic(jl_array_t*) targ; // contains Type{LeafType} (in a map of non-abstract TypeName) - _Atomic(jl_array_t*) name1; // a map for a map for TypeName, for parents up to (excluding) Any - _Atomic(jl_array_t*) tname; // a map for Type{TypeName}, for parents up to (including) Any + _Atomic(jl_genericmemory_t*) arg1; // contains LeafType (in a map of non-abstract TypeName) + _Atomic(jl_genericmemory_t*) targ; // contains Type{LeafType} (in a map of non-abstract TypeName) + _Atomic(jl_genericmemory_t*) name1; // a map for a map for TypeName, for parents up to (excluding) Any + _Atomic(jl_genericmemory_t*) tname; // a map for Type{TypeName}, for parents up to (including) Any // next a linear list of things too complicated at this level for analysis (no more levels) _Atomic(jl_typemap_entry_t*) linear; // finally, start a new level if the type at offs is Any @@ -679,7 +668,7 @@ typedef struct _jl_methtable_t { JL_DATA_TYPE jl_sym_t *name; // sometimes used for debug printing _Atomic(jl_typemap_t*) defs; - _Atomic(jl_array_t*) leafcache; + _Atomic(jl_genericmemory_t*) leafcache; _Atomic(jl_typemap_t*) cache; _Atomic(intptr_t) max_args; // max # of non-vararg arguments in a signature jl_module_t *module; // sometimes used for debug printing @@ -813,6 +802,10 @@ extern JL_DLLIMPORT jl_unionall_t *jl_abstractarray_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_unionall_t *jl_densearray_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_unionall_t *jl_array_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_typename_t *jl_array_typename JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_unionall_t *jl_genericmemory_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_typename_t *jl_genericmemory_typename JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_unionall_t *jl_genericmemoryref_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_typename_t *jl_genericmemoryref_typename JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_weakref_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_abstractstring_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_string_type JL_GLOBALLY_ROOTED; @@ -833,6 +826,7 @@ extern JL_DLLIMPORT jl_value_t *jl_undefref_exception JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_interrupt_exception JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_boundserror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_an_empty_vec_any JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_an_empty_memory_any JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_an_empty_string JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_bool_type JL_GLOBALLY_ROOTED; @@ -871,6 +865,10 @@ extern JL_DLLIMPORT jl_value_t *jl_array_symbol_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_int32_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_uint32_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_uint64_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memory_uint8_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memory_any_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memoryref_uint8_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memoryref_any_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_expr_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_binding_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_globalref_type JL_GLOBALLY_ROOTED; @@ -1089,16 +1087,101 @@ STATIC_INLINE jl_value_t *jl_svecset( } #endif -#define jl_array_len(a) (((jl_array_t*)(a))->length) -#define jl_array_data(a) ((void*)((jl_array_t*)(a))->data) -#define jl_array_dim(a,i) ((&((jl_array_t*)(a))->nrows)[i]) -#define jl_array_dim0(a) (((jl_array_t*)(a))->nrows) -#define jl_array_nrows(a) (((jl_array_t*)(a))->nrows) -#define jl_array_ndims(a) ((int32_t)(((jl_array_t*)a)->flags.ndims)) -#define jl_array_data_owner_offset(ndims) (offsetof(jl_array_t,ncols) + sizeof(size_t)*(1+jl_array_ndimwords(ndims))) // in bytes -#define jl_array_data_owner(a) (*((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a))))) +#define jl_genericmemory_data_owner_field(a) (*(jl_value_t**)((jl_genericmemory_t*)(a) + 1)) + +#define jl_nparams(t) jl_svec_len(((jl_datatype_t*)(t))->parameters) +#define jl_tparam0(t) jl_svecref(((jl_datatype_t*)(t))->parameters, 0) +#define jl_tparam1(t) jl_svecref(((jl_datatype_t*)(t))->parameters, 1) +#define jl_tparam(t,i) jl_svecref(((jl_datatype_t*)(t))->parameters, i) +#define jl_array_data(a,t) ((t*)((jl_array_t*)(a))->ref.data) +#define jl_array_data_(a) ((void*)((jl_array_t*)(a))->ref.data) +#define jl_array_dim(a,i) (((jl_array_t*)(a))->dimsize[i]) +#define jl_array_dim0(a) (((jl_array_t*)(a))->dimsize[0]) +#define jl_array_nrows(a) (((jl_array_t*)(a))->dimsize[0]) +#define jl_array_ndims(a) (*(size_t*)jl_tparam1(jl_typetagof(a))) +#define jl_array_maxsize(a) (((jl_array_t*)(a))->ref.mem->length) +#define jl_array_len(a) (jl_array_ndims(a) == 1 ? jl_array_nrows(a) : jl_array_maxsize(a)) + +/* + how - allocation style + 0 = data is inlined + 1 = owns the gc-managed data, exclusively + 2 = malloc-allocated pointer (may or may not own it) + 3 = has a pointer to the object that owns the data pointer +*/ +STATIC_INLINE int jl_genericmemory_how(jl_genericmemory_t *m) JL_NOTSAFEPOINT +{ + if (m->data == (void*)(m + 1)) + return 0; + jl_value_t *owner = jl_genericmemory_data_owner_field(m); + if (owner == (jl_value_t*)m) + return 1; + if (owner == NULL) + return 2; + return 3; +} + +STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + if (jl_genericmemory_how(m) == 3) + return jl_genericmemory_data_owner_field(m); + return (jl_value_t*)m; +} JL_DLLEXPORT char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT; +JL_DLLEXPORT char *jl_genericmemory_typetagdata(jl_genericmemory_t *m) JL_NOTSAFEPOINT; + +#ifdef __clang_gcanalyzer__ +jl_value_t **jl_genericmemory_ptr_data(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +STATIC_INLINE jl_value_t *jl_genericmemory_ptr_ref(void *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; +STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set( + void *m JL_ROOTING_ARGUMENT, size_t i, + void *x JL_ROOTED_ARGUMENT) JL_NOTSAFEPOINT; +#else +#define jl_genericmemory_ptr_data(a) ((jl_value_t**)((jl_genericmemory_t*)(a))->data) +STATIC_INLINE jl_value_t *jl_genericmemory_ptr_ref(void *m JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT +{ + jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; + assert(((jl_datatype_t*)jl_typetagof(m_))->layout->arrayelem_isboxed); + assert(i < m_->length); + return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)(m_->data)) + i); +} +STATIC_INLINE jl_value_t *jl_genericmemory_ptr_set( + void *m JL_ROOTING_ARGUMENT, size_t i, + void *x JL_ROOTED_ARGUMENT) JL_NOTSAFEPOINT +{ + jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; + assert(((jl_datatype_t*)jl_typetagof(m_))->layout->arrayelem_isboxed); + assert(i < m_->length); + jl_atomic_store_release(((_Atomic(jl_value_t*)*)(m_->data)) + i, (jl_value_t*)x); + if (x) { + if (jl_genericmemory_how(m_) == 3) + m = (void*)jl_genericmemory_data_owner_field(m_); + jl_gc_wb(m, x); + } + return (jl_value_t*)x; +} +#endif + +STATIC_INLINE uint8_t jl_memory_uint8_ref(void *m, size_t i) JL_NOTSAFEPOINT +{ + jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; + assert(jl_typetagis(m_, jl_memory_uint8_type)); + assert(i < m_->length); + return ((uint8_t*)m_->data)[i]; +} +STATIC_INLINE void jl_memory_uint8_set(void *m, size_t i, uint8_t x) JL_NOTSAFEPOINT +{ + jl_genericmemory_t *m_ = (jl_genericmemory_t*)m; + assert(jl_typetagis(m_, jl_memory_uint8_type)); + assert(i < m_->length); + ((uint8_t*)m_->data)[i] = x; +} + +STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT +{ + return jl_genericmemory_owner(a->ref.mem); +} #ifdef __clang_gcanalyzer__ jl_value_t **jl_array_ptr_data(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; @@ -1107,25 +1190,22 @@ STATIC_INLINE jl_value_t *jl_array_ptr_set( void *a JL_ROOTING_ARGUMENT, size_t i, void *x JL_ROOTED_ARGUMENT) JL_NOTSAFEPOINT; #else -#define jl_array_ptr_data(a) ((jl_value_t**)((jl_array_t*)(a))->data) +#define jl_array_ptr_data(a) (jl_array_data(a, jl_value_t*)) STATIC_INLINE jl_value_t *jl_array_ptr_ref(void *a JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT { - assert(((jl_array_t*)a)->flags.ptrarray); + assert(((jl_datatype_t*)jl_typetagof(((jl_array_t*)a)->ref.mem))->layout->arrayelem_isboxed); assert(i < jl_array_len(a)); - return jl_atomic_load_relaxed(((_Atomic(jl_value_t*)*)(jl_array_data(a))) + i); + return jl_atomic_load_relaxed(jl_array_data(a, _Atomic(jl_value_t*)) + i); } STATIC_INLINE jl_value_t *jl_array_ptr_set( void *a JL_ROOTING_ARGUMENT, size_t i, void *x JL_ROOTED_ARGUMENT) JL_NOTSAFEPOINT { - assert(((jl_array_t*)a)->flags.ptrarray); + assert(((jl_datatype_t*)jl_typetagof(((jl_array_t*)a)->ref.mem))->layout->arrayelem_isboxed); assert(i < jl_array_len(a)); - jl_atomic_store_release(((_Atomic(jl_value_t*)*)(jl_array_data(a))) + i, (jl_value_t*)x); + jl_atomic_store_release(jl_array_data(a, _Atomic(jl_value_t*)) + i, (jl_value_t*)x); if (x) { - if (((jl_array_t*)a)->flags.how == 3) { - a = jl_array_data_owner(a); - } - jl_gc_wb(a, x); + jl_gc_wb(jl_array_owner((jl_array_t*)a), x); } return (jl_value_t*)x; } @@ -1133,32 +1213,26 @@ STATIC_INLINE jl_value_t *jl_array_ptr_set( STATIC_INLINE uint8_t jl_array_uint8_ref(void *a, size_t i) JL_NOTSAFEPOINT { - assert(i < jl_array_len(a)); assert(jl_typetagis(a, jl_array_uint8_type)); - return ((uint8_t*)(jl_array_data(a)))[i]; + assert(i < jl_array_len(a)); + return jl_array_data(a, uint8_t)[i]; } STATIC_INLINE void jl_array_uint8_set(void *a, size_t i, uint8_t x) JL_NOTSAFEPOINT { - assert(i < jl_array_len(a)); assert(jl_typetagis(a, jl_array_uint8_type)); - ((uint8_t*)(jl_array_data(a)))[i] = x; -} -STATIC_INLINE uint8_t jl_array_uint32_ref(void *a, size_t i) JL_NOTSAFEPOINT -{ assert(i < jl_array_len(a)); - assert(jl_typetagis(a, jl_array_uint32_type)); - return ((uint32_t*)(jl_array_data(a)))[i]; + jl_array_data(a, uint8_t)[i] = x; } STATIC_INLINE void jl_array_uint32_set(void *a, size_t i, uint8_t x) JL_NOTSAFEPOINT { assert(i < jl_array_len(a)); - assert(jl_typetagis(a, jl_array_uint32_type)); - ((uint32_t*)(jl_array_data(a)))[i] = x; + assert(jl_typetagis(a, jl_array_uint32_type) || jl_typetagis(a, jl_array_int32_type)); + jl_array_data(a, uint32_t)[i] = x; } #define jl_exprarg(e,n) jl_array_ptr_ref(((jl_expr_t*)(e))->args, n) #define jl_exprargset(e, n, v) jl_array_ptr_set(((jl_expr_t*)(e))->args, n, v) -#define jl_expr_nargs(e) jl_array_len(((jl_expr_t*)(e))->args) +#define jl_expr_nargs(e) jl_array_nrows(((jl_expr_t*)(e))->args) #define jl_fieldref(s,i) jl_get_nth_field(((jl_value_t*)(s)),i) #define jl_fieldref_noalloc(s,i) jl_get_nth_field_noalloc(((jl_value_t*)(s)),i) @@ -1177,11 +1251,6 @@ STATIC_INLINE void jl_array_uint32_set(void *a, size_t i, uint8_t x) JL_NOTSAFEP #define jl_quotenode_value(x) (((jl_value_t**)x)[0]) #define jl_returnnode_value(x) (((jl_value_t**)x)[0]) -#define jl_nparams(t) jl_svec_len(((jl_datatype_t*)(t))->parameters) -#define jl_tparam0(t) jl_svecref(((jl_datatype_t*)(t))->parameters, 0) -#define jl_tparam1(t) jl_svecref(((jl_datatype_t*)(t))->parameters, 1) -#define jl_tparam(t,i) jl_svecref(((jl_datatype_t*)(t))->parameters, i) - // get a pointer to the data in a datatype #define jl_data_ptr(v) ((jl_value_t**)v) @@ -1208,10 +1277,24 @@ STATIC_INLINE jl_value_t *jl_field_type_concrete(jl_datatype_t *st JL_PROPAGATES return jl_svecref(st->types, i); } -#define jl_datatype_size(t) (((jl_datatype_t*)t)->layout->size) -#define jl_datatype_align(t) (((jl_datatype_t*)t)->layout->alignment) -#define jl_datatype_nbits(t) ((((jl_datatype_t*)t)->layout->size)*8) -#define jl_datatype_nfields(t) (((jl_datatype_t*)(t))->layout->nfields) +STATIC_INLINE int jl_is_layout_opaque(const jl_datatype_layout_t *l) JL_NOTSAFEPOINT +{ + return l->nfields == 0 && l->npointers > 0; +} + +JL_DLLEXPORT jl_value_t *jl_unwrap_unionall(jl_value_t *v JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; + +#define jl_inlinedatatype_layout(t) (((jl_datatype_t*)t)->layout) +STATIC_INLINE const jl_datatype_layout_t *jl_datatype_layout(jl_datatype_t *t) JL_NOTSAFEPOINT +{ + if (jl_is_layout_opaque(t->layout)) // e.g. GenericMemory + t = (jl_datatype_t*)jl_unwrap_unionall(t->name->wrapper); + return t->layout; +} +#define jl_datatype_size(t) (jl_datatype_layout((jl_datatype_t*)(t))->size) +#define jl_datatype_align(t) (jl_datatype_layout((jl_datatype_t*)(t))->alignment) +#define jl_datatype_nbits(t) ((jl_datatype_layout((jl_datatype_t*)(t))->size)*8) +#define jl_datatype_nfields(t) (jl_datatype_layout((jl_datatype_t*)(t))->nfields) JL_DLLEXPORT void *jl_symbol_name(jl_sym_t *s); // inline version with strong type check to detect typos in a `->name` chain @@ -1246,7 +1329,7 @@ static inline const char *jl_dt_layout_ptrs(const jl_datatype_layout_t *l) JL_NO static inline uint32_t jl_field_##f(jl_datatype_t *st, \ int i) JL_NOTSAFEPOINT \ { \ - const jl_datatype_layout_t *ly = st->layout; \ + const jl_datatype_layout_t *ly = jl_datatype_layout(st); \ assert(i >= 0 && (size_t)i < ly->nfields); \ if (ly->fielddesc_type == 0) { \ return ((const jl_fielddesc8_t*)jl_dt_layout_fields(ly))[i].f; \ @@ -1266,14 +1349,14 @@ DEFINE_FIELD_ACCESSORS(size) static inline int jl_field_isptr(jl_datatype_t *st, int i) JL_NOTSAFEPOINT { - const jl_datatype_layout_t *ly = st->layout; + const jl_datatype_layout_t *ly = jl_datatype_layout(st); assert(i >= 0 && (size_t)i < ly->nfields); return ((const jl_fielddesc8_t*)(jl_dt_layout_fields(ly) + jl_fielddesc_size(ly->fielddesc_type) * i))->isptr; } static inline uint32_t jl_ptr_offset(jl_datatype_t *st, int i) JL_NOTSAFEPOINT { - const jl_datatype_layout_t *ly = st->layout; + const jl_datatype_layout_t *ly = st->layout; // NOT jl_datatype_layout(st) assert(i >= 0 && (size_t)i < ly->npointers); const void *ptrs = jl_dt_layout_ptrs(ly); if (ly->fielddesc_type == 0) { @@ -1312,11 +1395,6 @@ static inline int jl_field_isconst(jl_datatype_t *st, int i) JL_NOTSAFEPOINT } -static inline int jl_is_layout_opaque(const jl_datatype_layout_t *l) JL_NOTSAFEPOINT -{ - return l->nfields == 0 && l->npointers > 0; -} - // basic predicates ----------------------------------------------------------- #define jl_is_nothing(v) (((jl_value_t*)(v)) == ((jl_value_t*)jl_nothing)) #define jl_is_tuple(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_tuple_typename) @@ -1372,7 +1450,7 @@ static inline int jl_is_layout_opaque(const jl_datatype_layout_t *l) JL_NOTSAFEP #define jl_is_uint8pointer(v)jl_typetagis(v,jl_uint8pointer_type) #define jl_is_llvmpointer(v) (((jl_datatype_t*)jl_typeof(v))->name == jl_llvmpointer_typename) #define jl_is_intrinsic(v) jl_typetagis(v,jl_intrinsic_type) -#define jl_array_isbitsunion(a) (!(((jl_array_t*)(a))->flags.ptrarray) && jl_is_uniontype(jl_tparam0(jl_typeof(a)))) +#define jl_genericmemory_isbitsunion(a) (((jl_datatype_t*)jl_typetagof(a))->layout->arrayelem_isunion) JL_DLLEXPORT int jl_subtype(jl_value_t *a, jl_value_t *b); @@ -1408,23 +1486,23 @@ STATIC_INLINE int jl_is_structtype(void *v) JL_NOTSAFEPOINT STATIC_INLINE int jl_isbits(void *t) JL_NOTSAFEPOINT // corresponding to isbitstype() in julia { - return (jl_is_datatype(t) && ((jl_datatype_t*)t)->isbitstype); + return jl_is_datatype(t) && ((jl_datatype_t*)t)->isbitstype; } STATIC_INLINE int jl_is_datatype_singleton(jl_datatype_t *d) JL_NOTSAFEPOINT { - return (d->instance != NULL); + return d->instance != NULL && d->layout->size == 0 && d->layout->npointers == 0; } STATIC_INLINE int jl_is_abstracttype(void *v) JL_NOTSAFEPOINT { - return (jl_is_datatype(v) && ((jl_datatype_t*)(v))->name->abstract); + return jl_is_datatype(v) && ((jl_datatype_t*)(v))->name->abstract; } STATIC_INLINE int jl_is_array_type(void *t) JL_NOTSAFEPOINT { - return (jl_is_datatype(t) && - ((jl_datatype_t*)(t))->name == jl_array_typename); + return jl_is_datatype(t) && + ((jl_datatype_t*)(t))->name == jl_array_typename; } STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT @@ -1433,6 +1511,29 @@ STATIC_INLINE int jl_is_array(void *v) JL_NOTSAFEPOINT return jl_is_array_type(t); } +STATIC_INLINE int jl_is_genericmemory_type(void *t) JL_NOTSAFEPOINT +{ + return (jl_is_datatype(t) && + ((jl_datatype_t*)(t))->name == jl_genericmemory_typename); +} + +STATIC_INLINE int jl_is_genericmemory(void *v) JL_NOTSAFEPOINT +{ + jl_value_t *t = jl_typeof(v); + return jl_is_genericmemory_type(t); +} + +STATIC_INLINE int jl_is_genericmemoryref_type(void *t) JL_NOTSAFEPOINT +{ + return (jl_is_datatype(t) && + ((jl_datatype_t*)(t))->name == jl_genericmemoryref_typename); +} + +STATIC_INLINE int jl_is_genericmemoryref(void *v) JL_NOTSAFEPOINT +{ + jl_value_t *t = jl_typeof(v); + return jl_is_genericmemoryref_type(t); +} STATIC_INLINE int jl_is_opaque_closure_type(void *t) JL_NOTSAFEPOINT { @@ -1488,12 +1589,9 @@ STATIC_INLINE int jl_is_type_type(jl_value_t *v) JL_NOTSAFEPOINT ((jl_datatype_t*)(v))->name == ((jl_datatype_t*)jl_type_type->body)->name); } -STATIC_INLINE int jl_is_array_zeroinit(jl_array_t *a) JL_NOTSAFEPOINT +STATIC_INLINE int jl_is_genericmemory_zeroinit(jl_genericmemory_t *m) JL_NOTSAFEPOINT { - if (a->flags.ptrarray || a->flags.hasptr) - return 1; - jl_value_t *elty = jl_tparam0(jl_typeof(a)); - return jl_is_datatype(elty) && ((jl_datatype_t*)elty)->zeroinit; + return ((jl_datatype_t*)jl_typeof(m))->zeroinit; } // object identity @@ -1680,35 +1778,20 @@ int jl_uniontype_size(jl_value_t *ty, size_t *sz); JL_DLLEXPORT int jl_islayout_inline(jl_value_t *eltype, size_t *fsz, size_t *al); // arrays -JL_DLLEXPORT jl_array_t *jl_new_array(jl_value_t *atype, jl_value_t *dims); -JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, - jl_value_t *dims); JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel, int own_buffer); JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_value_t *dims, int own_buffer); JL_DLLEXPORT jl_array_t *jl_alloc_array_1d(jl_value_t *atype, size_t nr); -JL_DLLEXPORT jl_array_t *jl_alloc_array_2d(jl_value_t *atype, size_t nr, - size_t nc); -JL_DLLEXPORT jl_array_t *jl_alloc_array_3d(jl_value_t *atype, size_t nr, - size_t nc, size_t z); JL_DLLEXPORT jl_array_t *jl_pchar_to_array(const char *str, size_t len); JL_DLLEXPORT jl_value_t *jl_pchar_to_string(const char *str, size_t len); JL_DLLEXPORT jl_value_t *jl_cstr_to_string(const char *str); JL_DLLEXPORT jl_value_t *jl_alloc_string(size_t len); JL_DLLEXPORT jl_value_t *jl_array_to_string(jl_array_t *a); JL_DLLEXPORT jl_array_t *jl_alloc_vec_any(size_t n); -JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i); // 0-indexed -JL_DLLEXPORT jl_value_t *jl_ptrarrayref(jl_array_t *a JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; // 0-indexed -JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i); // 0-indexed -JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i); // 0-indexed -JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i); // 0-indexed JL_DLLEXPORT void jl_array_grow_end(jl_array_t *a, size_t inc); JL_DLLEXPORT void jl_array_del_end(jl_array_t *a, size_t dec); -JL_DLLEXPORT void jl_array_grow_beg(jl_array_t *a, size_t inc); -JL_DLLEXPORT void jl_array_del_beg(jl_array_t *a, size_t dec); -JL_DLLEXPORT void jl_array_sizehint(jl_array_t *a, size_t sz); JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item); JL_DLLEXPORT void jl_array_ptr_1d_append(jl_array_t *a, jl_array_t *a2); JL_DLLEXPORT jl_value_t *jl_apply_array_type(jl_value_t *type, size_t dim); @@ -1717,7 +1800,28 @@ JL_DLLEXPORT int jl_array_validate_dims(size_t *nel, size_t *tot, uint32_t ndims JL_DLLEXPORT void *jl_array_ptr(jl_array_t *a); JL_DLLEXPORT void *jl_array_eltype(jl_value_t *a); JL_DLLEXPORT int jl_array_rank(jl_value_t *a); -JL_DLLEXPORT size_t jl_array_size(jl_value_t *a, int d); + +// genericmemory +JL_DLLEXPORT jl_genericmemory_t *jl_new_genericmemory(jl_value_t *mtype, jl_value_t *dim); +JL_DLLEXPORT jl_genericmemory_t *jl_ptr_to_genericmemory(jl_value_t *mtype, void *data, + size_t nel, int own_buffer); +JL_DLLEXPORT jl_genericmemory_t *jl_alloc_genericmemory(jl_value_t *mtype, size_t nel); +JL_DLLEXPORT jl_genericmemory_t *jl_pchar_to_memory(const char *str, size_t len); +JL_DLLEXPORT jl_value_t *jl_genericmemory_to_string(jl_genericmemory_t *m, size_t len); +JL_DLLEXPORT jl_genericmemory_t *jl_alloc_memory_any(size_t n); +JL_DLLEXPORT jl_genericmemory_t *jl_genericmemory_slice(jl_genericmemory_t *mem, size_t offset, size_t len); +JL_DLLEXPORT jl_value_t *jl_genericmemoryref(jl_genericmemory_t *m, size_t i); // 0-indexed +JL_DLLEXPORT void jl_genericmemoryset(jl_genericmemory_t *m JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED, size_t i); // 0-indexed +JL_DLLEXPORT void jl_genericmemoryunset(jl_genericmemory_t *m, size_t i); // 0-indexed +JL_DLLEXPORT int jl_genericmemory_isassigned(jl_genericmemory_t *m, size_t i); // 0-indexed + +JL_DLLEXPORT jl_genericmemoryref_t *jl_new_memoryref(jl_value_t *typ, jl_genericmemory_t *mem, void *data); +JL_DLLEXPORT jl_value_t *jl_memoryrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT); +JL_DLLEXPORT jl_value_t *jl_ptrmemoryrefget(jl_genericmemoryref_t m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_memoryref_isassigned(jl_genericmemoryref_t m) JL_GLOBALLY_ROOTED; +JL_DLLEXPORT jl_genericmemoryref_t jl_memoryrefindex(jl_genericmemoryref_t m JL_PROPAGATES_ROOT, size_t idx) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_memoryrefset(jl_genericmemoryref_t m JL_ROOTING_ARGUMENT, jl_value_t *v JL_ROOTED_ARGUMENT JL_MAYBE_UNROOTED); +JL_DLLEXPORT void jl_memoryrefunset(jl_genericmemoryref_t m); // strings JL_DLLEXPORT const char *jl_string_ptr(jl_value_t *s); @@ -1774,10 +1878,10 @@ STATIC_INLINE jl_function_t *jl_get_function(jl_module_t *m, const char *name) } // eq hash tables -JL_DLLEXPORT jl_array_t *jl_eqtable_put(jl_array_t *h JL_ROOTING_ARGUMENT, jl_value_t *key, jl_value_t *val JL_ROOTED_ARGUMENT, int *inserted); -JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_array_t *h JL_PROPAGATES_ROOT, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; -JL_DLLEXPORT jl_value_t *jl_eqtable_pop(jl_array_t *h, jl_value_t *key, jl_value_t *deflt, int *found); -jl_value_t *jl_eqtable_getkey(jl_array_t *h JL_PROPAGATES_ROOT, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_genericmemory_t *jl_eqtable_put(jl_genericmemory_t *h JL_ROOTING_ARGUMENT, jl_value_t *key, jl_value_t *val JL_ROOTED_ARGUMENT, int *inserted); +JL_DLLEXPORT jl_value_t *jl_eqtable_get(jl_genericmemory_t *h JL_PROPAGATES_ROOT, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_value_t *jl_eqtable_pop(jl_genericmemory_t *h, jl_value_t *key, jl_value_t *deflt, int *found); +jl_value_t *jl_eqtable_getkey(jl_genericmemory_t *h JL_PROPAGATES_ROOT, jl_value_t *key, jl_value_t *deflt) JL_NOTSAFEPOINT; // system information JL_DLLEXPORT int jl_errno(void) JL_NOTSAFEPOINT; diff --git a/src/julia_internal.h b/src/julia_internal.h index 61e141c755a5e..ae7f86361a588 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -569,6 +569,7 @@ JL_DLLEXPORT void JL_NORETURN jl_throw_out_of_memory_error(void); JL_DLLEXPORT int64_t jl_gc_diff_total_bytes(void) JL_NOTSAFEPOINT; JL_DLLEXPORT int64_t jl_gc_sync_total_bytes(int64_t offset) JL_NOTSAFEPOINT; void jl_gc_track_malloced_array(jl_ptls_t ptls, jl_array_t *a) JL_NOTSAFEPOINT; +void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPOINT; void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT; void jl_gc_run_all_finalizers(jl_task_t *ct); void jl_release_task_stack(jl_ptls_t ptls, jl_task_t *task); @@ -777,15 +778,13 @@ JL_DLLEXPORT jl_methtable_t *jl_method_get_table( JL_DLLEXPORT int jl_pointer_egal(jl_value_t *t); JL_DLLEXPORT jl_value_t *jl_nth_slot_type(jl_value_t *sig JL_PROPAGATES_ROOT, size_t i) JL_NOTSAFEPOINT; void jl_compute_field_offsets(jl_datatype_t *st); -jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims, - int isunboxed, int hasptr, int isunion, int elsz); void jl_module_run_initializer(jl_module_t *m); JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, int alloc); JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *sym, jl_binding_t *b); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; -extern jl_array_t *jl_global_roots_table JL_GLOBALLY_ROOTED; +extern jl_genericmemory_t *jl_global_roots_table JL_GLOBALLY_ROOTED; JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val JL_MAYBE_UNROOTED); @@ -969,8 +968,8 @@ uint8_t jl_object_in_image(jl_value_t* v) JL_NOTSAFEPOINT; // the first argument to jl_idtable_rehash is used to return a value // make sure it is rooted if it is used after the function returns -JL_DLLEXPORT jl_array_t *jl_idtable_rehash(jl_array_t *a, size_t newsz); -_Atomic(jl_value_t*) *jl_table_peek_bp(jl_array_t *a, jl_value_t *key) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_genericmemory_t *jl_idtable_rehash(jl_genericmemory_t *a, size_t newsz); +_Atomic(jl_value_t*) *jl_table_peek_bp(jl_genericmemory_t *a, jl_value_t *key) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t*); @@ -1285,7 +1284,7 @@ JL_DLLEXPORT unsigned jl_intrinsic_nargs(int f) JL_NOTSAFEPOINT; STATIC_INLINE int is_valid_intrinsic_elptr(jl_value_t *ety) { - return ety == (jl_value_t*)jl_any_type || (jl_is_concrete_type(ety) && !jl_is_layout_opaque(((jl_datatype_t*)ety)->layout)); + return ety == (jl_value_t*)jl_any_type || (jl_is_concrete_type(ety) && !jl_is_layout_opaque(((jl_datatype_t*)ety)->layout) && !jl_is_array(ety)); } JL_DLLEXPORT jl_value_t *jl_bitcast(jl_value_t *ty, jl_value_t *v); JL_DLLEXPORT jl_value_t *jl_pointerref(jl_value_t *p, jl_value_t *i, jl_value_t *align); @@ -1380,7 +1379,6 @@ JL_DLLEXPORT jl_value_t *jl_arraylen(jl_value_t *a); JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *a); JL_DLLEXPORT int jl_stored_inline(jl_value_t *el_type); JL_DLLEXPORT jl_value_t *(jl_array_data_owner)(jl_array_t *a); -JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i); JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary); JL_DLLEXPORT uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) JL_NOTSAFEPOINT; diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index 6f39a1d5f715e..80788e67b9b47 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -79,6 +79,7 @@ static void removeGCPreserve(CallInst *call, Instruction *val) * * * load * * `pointer_from_objref` + * * `gc_loaded` * * Any real llvm intrinsics * * gc preserve intrinsics * * `ccall` gcroot array (`jl_roots` operand bundle) @@ -94,7 +95,6 @@ static void removeGCPreserve(CallInst *call, Instruction *val) * TODO: * * Return twice * * Handle phi node. - * * Look through `pointer_from_objref`. * * Handle jl_box* */ @@ -310,7 +310,9 @@ bool Optimizer::isSafepoint(Instruction *inst) return false; if (auto callee = call->getCalledFunction()) { // Known functions emitted in codegen that are not safepoints - if (callee == pass.pointer_from_objref_func || callee->getName() == "memcmp") { + if (callee == pass.pointer_from_objref_func + || callee == pass.gc_loaded_func + || callee->getName() == "memcmp") { return false; } } @@ -695,6 +697,11 @@ void Optimizer::moveToStack(CallInst *orig_inst, size_t sz, bool has_ref) call->eraseFromParent(); return; } + //if (pass.gc_loaded_func == callee) { + // call->replaceAllUsesWith(new_i); + // call->eraseFromParent(); + // return; + //} if (pass.typeof_func == callee) { ++RemovedTypeofs; call->replaceAllUsesWith(tag); diff --git a/src/llvm-codegen-shared.h b/src/llvm-codegen-shared.h index 0ab140b42b8b7..ff69ea4690921 100644 --- a/src/llvm-codegen-shared.h +++ b/src/llvm-codegen-shared.h @@ -92,7 +92,7 @@ struct CountTrackedPointers { unsigned count = 0; bool all = true; bool derived = false; - CountTrackedPointers(llvm::Type *T); + CountTrackedPointers(llvm::Type *T, bool ignore_loaded=false); }; unsigned TrackWithShadow(llvm::Value *Src, llvm::Type *T, bool isptr, llvm::Value *Dst, llvm::Type *DTy, llvm::IRBuilder<> &irbuilder); diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index e49cdc5c5b552..a57e674fac3e3 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -377,16 +377,18 @@ static bool isSpecialPtr(Type *Ty) { // return how many Special pointers are in T (count > 0), // and if there is anything else in T (all == false) -CountTrackedPointers::CountTrackedPointers(Type *T) { +CountTrackedPointers::CountTrackedPointers(Type *T, bool ignore_loaded) { if (isa(T)) { if (isSpecialPtr(T)) { + if (ignore_loaded && T->getPointerAddressSpace() == AddressSpace::Loaded) + return; count++; if (T->getPointerAddressSpace() != AddressSpace::Tracked) derived = true; } } else if (isa(T) || isa(T) || isa(T)) { for (Type *ElT : T->subtypes()) { - auto sub = CountTrackedPointers(ElT); + auto sub = CountTrackedPointers(ElT, ignore_loaded); count += sub.count; all &= sub.all; derived |= sub.derived; @@ -402,6 +404,20 @@ CountTrackedPointers::CountTrackedPointers(Type *T) { all = false; } +bool hasLoadedTy(Type *T) { + if (isa(T)) { + if (T->getPointerAddressSpace() == AddressSpace::Loaded) + return true; + } else if (isa(T) || isa(T) || isa(T)) { + for (Type *ElT : T->subtypes()) { + if (hasLoadedTy(ElT)) + return true; + } + } + return false; +} + + unsigned getCompositeNumElements(Type *T) { if (auto *ST = dyn_cast(T)) return ST->getNumElements(); @@ -483,18 +499,18 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac CurrentV = EEI->getVectorOperand(); } else if (auto LI = dyn_cast(CurrentV)) { - if (auto PtrT = dyn_cast(LI->getType()->getScalarType())) { - if (PtrT->getAddressSpace() == AddressSpace::Loaded) { - CurrentV = LI->getPointerOperand(); - fld_idx = -1; - if (!isSpecialPtr(CurrentV->getType())) { - // This could really be anything, but it's not loaded - // from a tracked pointer, so it doesn't matter what - // it is--just pick something simple. - CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); - } - continue; + if (hasLoadedTy(LI->getType())) { + // TODO: we need a better representation of this than Loaded, since this is a mutable value we loaded from, + // but the load is *paired with* the Tracked value + CurrentV = LI->getPointerOperand(); + fld_idx = -1; + if (!isSpecialPtr(CurrentV->getType())) { + // This could really be anything, but it's not loaded + // from a tracked pointer, so it doesn't matter what + // it is--just pick something simple. + CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); } + continue; } // In general a load terminates a walk break; @@ -516,36 +532,42 @@ static std::pair FindBaseValue(const State &S, Value *V, bool UseCac if (II->getIntrinsicID() == Intrinsic::masked_load || II->getIntrinsicID() == Intrinsic::masked_gather) { if (auto VTy = dyn_cast(II->getType())) { - if (auto PtrT = dyn_cast(VTy->getElementType())) { - if (PtrT->getAddressSpace() == AddressSpace::Loaded) { - Value *Mask = II->getOperand(2); - Value *Passthrough = II->getOperand(3); - if (!isa(Mask) || !cast(Mask)->isAllOnesValue()) { - assert(isa(Passthrough) && "unimplemented"); - (void)Passthrough; + if (hasLoadedTy(VTy->getElementType())) { + Value *Mask = II->getOperand(2); + Value *Passthrough = II->getOperand(3); + if (!isa(Mask) || !cast(Mask)->isAllOnesValue()) { + assert(isa(Passthrough) && "unimplemented"); + (void)Passthrough; + } + CurrentV = II->getOperand(0); + if (II->getIntrinsicID() == Intrinsic::masked_load) { + fld_idx = -1; + if (!isSpecialPtr(CurrentV->getType())) { + CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); } - CurrentV = II->getOperand(0); - if (II->getIntrinsicID() == Intrinsic::masked_load) { - fld_idx = -1; - if (!isSpecialPtr(CurrentV->getType())) { + } else { + if (auto VTy2 = dyn_cast(CurrentV->getType())) { + if (!isSpecialPtr(VTy2->getElementType())) { CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); - } - } else { - if (auto VTy2 = dyn_cast(CurrentV->getType())) { - if (!isSpecialPtr(VTy2->getElementType())) { - CurrentV = ConstantPointerNull::get(Type::getInt8PtrTy(V->getContext())); - fld_idx = -1; - } + fld_idx = -1; } } - continue; } + continue; } } // In general a load terminates a walk break; } } + else if (auto CI = dyn_cast(CurrentV)) { + auto callee = CI->getCalledFunction(); + if (callee && callee->getName() == "julia.gc_loaded") { + CurrentV = CI->getArgOperand(0); + continue; + } + break; + } else { break; } @@ -637,15 +659,12 @@ void LateLowerGCFrame::LiftSelect(State &S, SelectInst *SI) { // already visited here--nothing to do return; } + assert(!isTrackedValue(SI)); std::vector Numbers; unsigned NumRoots = 1; - if (auto VTy = dyn_cast(SI->getType())) { - ElementCount EC = VTy->getElementCount(); - Numbers.resize(EC.getKnownMinValue(), -1); - } - else - assert(isa(SI->getType()) && "unimplemented"); - assert(!isTrackedValue(SI)); + Type *STy = SI->getType(); + if (!isa(STy)) + Numbers.resize(CountTrackedPointers(STy).count, -1); // find the base root for the arguments Value *TrueBase = MaybeExtractScalar(S, FindBaseValue(S, SI->getTrueValue(), false), SI); Value *FalseBase = MaybeExtractScalar(S, FindBaseValue(S, SI->getFalseValue(), false), SI); @@ -722,20 +741,17 @@ void LateLowerGCFrame::LiftPhi(State &S, PHINode *Phi) { SmallVector lifted; std::vector Numbers; unsigned NumRoots = 1; - if (auto VTy = dyn_cast(Phi->getType())) { - NumRoots = VTy->getNumElements(); + Type *PTy = Phi->getType(); + if (!isa(PTy)) { + NumRoots = CountTrackedPointers(PTy).count; Numbers.resize(NumRoots); } - else { - // TODO: SVE - assert(isa(Phi->getType()) && "unimplemented"); - } for (unsigned i = 0; i < NumRoots; ++i) { PHINode *lift = PHINode::Create(T_prjlvalue, Phi->getNumIncomingValues(), "gclift", Phi); int Number = ++S.MaxPtrNumber; S.AllPtrNumbering[lift] = Number; S.ReversePtrNumbering[Number] = lift; - if (!isa(Phi->getType())) + if (isa(PTy)) S.AllPtrNumbering[Phi] = Number; else Numbers[i] = Number; @@ -1173,7 +1189,7 @@ static bool isLoadFromImmut(LoadInst *LI) if (LI->getMetadata(LLVMContext::MD_invariant_load)) return true; MDNode *TBAA = LI->getMetadata(LLVMContext::MD_tbaa); - if (isTBAA(TBAA, {"jtbaa_immut", "jtbaa_const", "jtbaa_datatype"})) + if (isTBAA(TBAA, {"jtbaa_immut", "jtbaa_const", "jtbaa_datatype", "jtbaa_memoryptr", "jtbaa_memorylen", "jtbaa_memoryown"})) return true; return false; } @@ -1228,6 +1244,10 @@ static bool isLoadFromConstGV(Value *v, bool &task_local, PhiSet *seen = nullptr task_local = true; return true; } + if (callee && callee->getName() == "julia.gc_loaded") { + return isLoadFromConstGV(call->getArgOperand(0), task_local, seen) && + isLoadFromConstGV(call->getArgOperand(1), task_local, seen); + } } if (isa(v)) { task_local = true; @@ -1252,8 +1272,7 @@ static bool isLoadFromConstGV(LoadInst *LI, bool &task_local, PhiSet *seen) auto load_base = LI->getPointerOperand()->stripInBoundsOffsets(); assert(load_base); // Static analyzer auto gv = dyn_cast(load_base); - if (isTBAA(LI->getMetadata(LLVMContext::MD_tbaa), - {"jtbaa_immut", "jtbaa_const", "jtbaa_datatype"})) { + if (isLoadFromImmut(LI)) { if (gv) return true; return isLoadFromConstGV(load_base, task_local, seen); @@ -1488,20 +1507,18 @@ State LateLowerGCFrame::LocalScan(Function &F) { if (II->getIntrinsicID() == Intrinsic::masked_load || II->getIntrinsicID() == Intrinsic::masked_gather) { if (auto VTy = dyn_cast(II->getType())) { - if (auto PtrT = dyn_cast(VTy->getElementType())) { - if (isSpecialPtr(PtrT)) { - // LLVM sometimes tries to materialize these operations with undefined pointers in our non-integral address space. - // Hopefully LLVM didn't already propagate that information and poison our users. Set those to NULL now. - Value *passthru = II->getArgOperand(3); - if (isa(passthru)) { - II->setArgOperand(3, Constant::getNullValue(passthru->getType())); - } - if (PtrT->getAddressSpace() == AddressSpace::Loaded) { - // These are not real defs - continue; - } + if (CountTrackedPointers(VTy->getElementType(), true).count) { + // LLVM sometimes tries to materialize these operations with undefined pointers in our non-integral address space. + // Hopefully LLVM didn't already propagate that information and poison our users. Set those to NULL now. + Value *passthru = II->getArgOperand(3); + if (isa(passthru)) { + II->setArgOperand(3, Constant::getNullValue(passthru->getType())); } } + if (hasLoadedTy(VTy->getElementType())) { + // These are not real defs + continue; + } } } } @@ -1509,13 +1526,16 @@ State LateLowerGCFrame::LocalScan(Function &F) { if (callee && callee == typeof_func) { MaybeNoteDef(S, BBS, CI, BBS.Safepoints, SmallVector{-2}); } + else if (callee && callee->getName() == "julia.gc_loaded") { + continue; + } else { MaybeNoteDef(S, BBS, CI, BBS.Safepoints); } if (CI->hasStructRetAttr()) { Type *ElT = getAttributeAtIndex(CI->getAttributes(), 1, Attribute::StructRet).getValueAsType(); assert(cast(CI->getArgOperand(0)->getType())->isOpaqueOrPointeeTypeMatches(getAttributeAtIndex(CI->getAttributes(), 1, Attribute::StructRet).getValueAsType())); - auto tracked = CountTrackedPointers(ElT); + auto tracked = CountTrackedPointers(ElT, true); if (tracked.count) { AllocaInst *SRet = dyn_cast((CI->arg_begin()[0])->stripInBoundsOffsets()); assert(SRet); @@ -1596,7 +1616,7 @@ State LateLowerGCFrame::LocalScan(Function &F) { callee == gc_preserve_end_func || callee == typeof_func || callee == pgcstack_getter || callee->getName() == XSTR(jl_egal__unboxed) || callee->getName() == XSTR(jl_lock_value) || callee->getName() == XSTR(jl_unlock_value) || - callee == write_barrier_func || + callee == write_barrier_func || callee == gc_loaded_func || callee->getName() == "memcmp") { continue; } @@ -1660,9 +1680,8 @@ State LateLowerGCFrame::LocalScan(Function &F) { // task but we do need to issue write barriers for when the current task dies. RefinedPtr.push_back(task_local ? -1 : -2); } - if (!Ty->isPointerTy() || Ty->getPointerAddressSpace() != AddressSpace::Loaded) { + if (!hasLoadedTy(Ty)) MaybeNoteDef(S, BBS, LI, BBS.Safepoints, std::move(RefinedPtr)); - } NoteOperandUses(S, BBS, I); } else if (auto *LI = dyn_cast(&I)) { Type *Ty = LI->getNewValOperand()->getType()->getScalarType(); @@ -1824,7 +1843,8 @@ std::vector ExtractTrackedValues(Value *Src, Type *STy, bool isptr, IRBu if (ignore_field(Idxs)) continue; Value *Elem = ExtractScalar(Src, STy, isptr, Idxs, irbuilder); - Ptrs.push_back(Elem); + if (isTrackedValue(Elem)) // ignore addrspace Loaded when it appears + Ptrs.push_back(Elem); } return Ptrs; } @@ -2291,7 +2311,7 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { if (I->getMetadata(LLVMContext::MD_invariant_load)) I->setMetadata(LLVMContext::MD_invariant_load, NULL); if (MDNode *TBAA = I->getMetadata(LLVMContext::MD_tbaa)) { - if (TBAA->getNumOperands() == 4 && isTBAA(TBAA, {"jtbaa_const"})) { + if (TBAA->getNumOperands() == 4 && isTBAA(TBAA, {"jtbaa_const", "jtbaa_memoryptr", "jtbaa_memorylen", "tbaa_memoryown"})) { MDNode *MutableTBAA = createMutableTBAAAccessTag(TBAA); if (MutableTBAA != TBAA) I->setMetadata(LLVMContext::MD_tbaa, MutableTBAA); @@ -2320,7 +2340,13 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { /* No replacement */ } else if (pointer_from_objref_func != nullptr && callee == pointer_from_objref_func) { auto *obj = CI->getOperand(0); - auto *ASCI = new AddrSpaceCastInst(obj, JuliaType::get_pjlvalue_ty(obj->getContext()), "", CI); + auto *ASCI = new AddrSpaceCastInst(obj, CI->getType(), "", CI); + ASCI->takeName(CI); + CI->replaceAllUsesWith(ASCI); + UpdatePtrNumbering(CI, ASCI, S); + } else if (gc_loaded_func != nullptr && callee == gc_loaded_func) { + auto *obj = CI->getOperand(1); + auto *ASCI = new AddrSpaceCastInst(obj, CI->getType(), "", CI); ASCI->takeName(CI); CI->replaceAllUsesWith(ASCI); UpdatePtrNumbering(CI, ASCI, S); diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 543cdb137e570..63eeb96f0a03e 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -25,7 +25,7 @@ JuliaPassContext::JuliaPassContext() pgcstack_getter(nullptr), adoptthread_func(nullptr), gc_flush_func(nullptr), gc_preserve_begin_func(nullptr), gc_preserve_end_func(nullptr), - pointer_from_objref_func(nullptr), alloc_obj_func(nullptr), + pointer_from_objref_func(nullptr), gc_loaded_func(nullptr), alloc_obj_func(nullptr), typeof_func(nullptr), write_barrier_func(nullptr), call_func(nullptr), call2_func(nullptr), module(nullptr) { @@ -48,6 +48,7 @@ void JuliaPassContext::initFunctions(Module &M) gc_preserve_begin_func = M.getFunction("llvm.julia.gc_preserve_begin"); gc_preserve_end_func = M.getFunction("llvm.julia.gc_preserve_end"); pointer_from_objref_func = M.getFunction("julia.pointer_from_objref"); + gc_loaded_func = M.getFunction("julia.gc_loaded"); typeof_func = M.getFunction("julia.typeof"); write_barrier_func = M.getFunction("julia.write_barrier"); alloc_obj_func = M.getFunction("julia.gc_alloc_obj"); diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index 97cc2a03415b2..3478f1ece0b5e 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -56,6 +56,7 @@ struct JuliaPassContext { llvm::Function *gc_preserve_begin_func; llvm::Function *gc_preserve_end_func; llvm::Function *pointer_from_objref_func; + llvm::Function *gc_loaded_func; llvm::Function *alloc_obj_func; llvm::Function *typeof_func; llvm::Function *write_barrier_func; diff --git a/src/method.c b/src/method.c index 68110d6995bbf..fdc4040d37f9f 100644 --- a/src/method.c +++ b/src/method.c @@ -94,7 +94,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve // ignore these } else { - size_t i = 0, nargs = jl_array_len(e->args); + size_t i = 0, nargs = jl_array_nrows(e->args); if (e->head == jl_opaque_closure_method_sym) { if (nargs != 5) { jl_error("opaque_closure_method: invalid syntax"); @@ -266,7 +266,7 @@ static jl_value_t *resolve_globals(jl_value_t *expr, jl_module_t *module, jl_sve JL_DLLEXPORT void jl_resolve_globals_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, int binding_effects) { - size_t i, l = jl_array_len(stmts); + size_t i, l = jl_array_nrows(stmts); for (i = 0; i < l; i++) { jl_value_t *stmt = jl_array_ptr_ref(stmts, i); jl_array_ptr_set(stmts, i, resolve_globals(stmt, m, sparam_vals, binding_effects, 0)); @@ -286,18 +286,17 @@ static void jl_code_info_set_ir(jl_code_info_t *li, jl_expr_t *ir) jl_expr_t *bodyex = (jl_expr_t*)jl_exprarg(ir, 2); jl_value_t *codelocs = jl_exprarg(ir, 3); li->linetable = jl_exprarg(ir, 4); - size_t nlocs = jl_array_len(codelocs); + size_t nlocs = jl_array_nrows(codelocs); li->codelocs = (jl_value_t*)jl_alloc_array_1d(jl_array_int32_type, nlocs); size_t j; for (j = 0; j < nlocs; j++) { - jl_arrayset((jl_array_t*)li->codelocs, jl_box_int32(jl_unbox_long(jl_arrayref((jl_array_t*)codelocs, j))), - j); + jl_array_uint32_set((jl_array_t*)li->codelocs, j, jl_unbox_long(jl_array_ptr_ref((jl_array_t*)codelocs, j))); } assert(jl_is_expr(bodyex)); jl_array_t *body = bodyex->args; li->code = body; jl_gc_wb(li, li->code); - size_t n = jl_array_len(body); + size_t n = jl_array_nrows(body); jl_value_t **bd = (jl_value_t**)jl_array_ptr_data((jl_array_t*)li->code); li->ssaflags = jl_alloc_array_1d(jl_array_uint32_type, n); jl_gc_wb(li, li->ssaflags); @@ -409,7 +408,7 @@ static void jl_code_info_set_ir(jl_code_info_t *li, jl_expr_t *ir) free(inline_flags); jl_array_t *vinfo = (jl_array_t*)jl_exprarg(ir, 1); jl_array_t *vis = (jl_array_t*)jl_array_ptr_ref(vinfo, 0); - size_t nslots = jl_array_len(vis); + size_t nslots = jl_array_nrows(vis); jl_value_t *ssavalue_types = jl_array_ptr_ref(vinfo, 2); assert(jl_is_long(ssavalue_types)); size_t nssavalue = jl_unbox_long(ssavalue_types); @@ -507,7 +506,7 @@ void jl_add_function_to_lineinfo(jl_code_info_t *ci, jl_value_t *func) { // func may contain jl_symbol (function name), jl_method_t, or jl_method_instance_t jl_array_t *li = (jl_array_t*)ci->linetable; - size_t i, n = jl_array_len(li); + size_t i, n = jl_array_nrows(li); jl_value_t *rt = NULL, *lno = NULL, *inl = NULL; JL_GC_PUSH3(&rt, &lno, &inl); for (i = 0; i < n; i++) { @@ -618,7 +617,7 @@ JL_DLLEXPORT jl_code_info_t *jl_code_for_staged(jl_method_instance_t *linfo, siz // If this generated function has an opaque closure, cache it for // correctness of method identity - for (int i = 0; i < jl_array_len(func->code); ++i) { + for (int i = 0; i < jl_array_nrows(func->code); ++i) { jl_value_t *stmt = jl_array_ptr_ref(func->code, i); if (jl_is_expr(stmt) && ((jl_expr_t*)stmt)->head == jl_new_opaque_closure_sym) { if (jl_options.incremental && jl_generating_output()) @@ -700,7 +699,7 @@ JL_DLLEXPORT void jl_method_set_source(jl_method_t *m, jl_code_info_t *src) JL_GC_PUSH3(©, &sparam_vars, &src); assert(jl_typetagis(src->code, jl_array_any_type)); jl_array_t *stmts = (jl_array_t*)src->code; - size_t i, n = jl_array_len(stmts); + size_t i, n = jl_array_nrows(stmts); copy = jl_alloc_vec_any(n); for (i = 0; i < n; i++) { jl_value_t *st = jl_array_ptr_ref(stmts, i); @@ -1110,7 +1109,7 @@ JL_DLLEXPORT jl_method_t* jl_method_def(jl_svec_t *argdata, jl_all_methods = jl_alloc_vec_any(0); #endif if (jl_all_methods != NULL) { - while (jl_array_len(jl_all_methods) < m->primary_world) + while (jl_array_nrows(jl_all_methods) < m->primary_world) jl_array_ptr_1d_push(jl_all_methods, NULL); jl_array_ptr_1d_push(jl_all_methods, (jl_value_t*)m); } @@ -1170,10 +1169,10 @@ static uint64_t current_root_id(jl_array_t *root_blocks) if (!root_blocks) return 0; assert(jl_is_array(root_blocks)); - size_t nx2 = jl_array_len(root_blocks); + size_t nx2 = jl_array_nrows(root_blocks); if (nx2 == 0) return 0; - uint64_t *blocks = (uint64_t*)jl_array_data(root_blocks); + uint64_t *blocks = jl_array_data(root_blocks, uint64_t); return blocks[nx2-2]; } @@ -1182,8 +1181,8 @@ static void add_root_block(jl_array_t *root_blocks, uint64_t modid, size_t len) { assert(jl_is_array(root_blocks)); jl_array_grow_end(root_blocks, 2); - uint64_t *blocks = (uint64_t*)jl_array_data(root_blocks); - int nx2 = jl_array_len(root_blocks); + uint64_t *blocks = jl_array_data(root_blocks, uint64_t); + int nx2 = jl_array_nrows(root_blocks); blocks[nx2-2] = modid; blocks[nx2-1] = len; } @@ -1213,7 +1212,7 @@ JL_DLLEXPORT void jl_add_method_root(jl_method_t *m, jl_module_t *mod, jl_value_ assert(jl_is_method(m)); prepare_method_for_roots(m, modid); if (current_root_id(m->root_blocks) != modid) - add_root_block(m->root_blocks, modid, jl_array_len(m->roots)); + add_root_block(m->root_blocks, modid, jl_array_nrows(m->roots)); jl_array_ptr_1d_push(m->roots, root); JL_GC_POP(); } @@ -1225,7 +1224,7 @@ void jl_append_method_roots(jl_method_t *m, uint64_t modid, jl_array_t* roots) assert(jl_is_method(m)); assert(jl_is_array(roots)); prepare_method_for_roots(m, modid); - add_root_block(m->root_blocks, modid, jl_array_len(m->roots)); + add_root_block(m->root_blocks, modid, jl_array_nrows(m->roots)); jl_array_ptr_1d_append(m->roots, roots); JL_GC_POP(); } @@ -1239,7 +1238,7 @@ int get_root_reference(rle_reference *rr, jl_method_t *m, size_t i) rr->index = i; return i < m->nroots_sysimg; } - rle_index_to_reference(rr, i, (uint64_t*)jl_array_data(m->root_blocks), jl_array_len(m->root_blocks), 0); + rle_index_to_reference(rr, i, jl_array_data(m->root_blocks, uint64_t), jl_array_nrows(m->root_blocks), 0); if (rr->key) return 1; return i < m->nroots_sysimg; @@ -1254,7 +1253,7 @@ jl_value_t *lookup_root(jl_method_t *m, uint64_t key, int index) return jl_array_ptr_ref(m->roots, index); } rle_reference rr = {key, index}; - size_t i = rle_reference_to_index(&rr, (uint64_t*)jl_array_data(m->root_blocks), jl_array_len(m->root_blocks), 0); + size_t i = rle_reference_to_index(&rr, jl_array_data(m->root_blocks, uint64_t), jl_array_nrows(m->root_blocks), 0); return jl_array_ptr_ref(m->roots, i); } @@ -1263,11 +1262,11 @@ int nroots_with_key(jl_method_t *m, uint64_t key) { size_t nroots = 0; if (m->roots) - nroots = jl_array_len(m->roots); + nroots = jl_array_nrows(m->roots); if (!m->root_blocks) return key == 0 ? nroots : 0; - uint64_t *rletable = (uint64_t*)jl_array_data(m->root_blocks); - size_t j, nblocks2 = jl_array_len(m->root_blocks); + uint64_t *rletable = jl_array_data(m->root_blocks, uint64_t); + size_t j, nblocks2 = jl_array_nrows(m->root_blocks); int nwithkey = 0; for (j = 0; j < nblocks2; j+=2) { if (rletable[j] == key) diff --git a/src/precompile.c b/src/precompile.c index f6266e252f609..7f4c96ab3e8e9 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -35,7 +35,7 @@ void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { // uint64: length of src text // char*: src text // At the end we write int32(0) as a terminal sentinel. - size_t len = jl_array_len(udeps); + size_t len = jl_array_nrows(udeps); ios_t srctext; for (size_t i = 0; i < len; i++) { jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); @@ -86,9 +86,9 @@ JL_DLLEXPORT void jl_write_compiler_output(void) jl_array_t *udeps = NULL; JL_GC_PUSH2(&worklist, &udeps); jl_module_init_order = jl_alloc_vec_any(0); - int i, l = jl_array_len(worklist); + int i, l = jl_array_nrows(worklist); for (i = 0; i < l; i++) { - jl_value_t *m = jl_ptrarrayref(worklist, i); + jl_value_t *m = jl_array_ptr_ref(worklist, i); jl_value_t *f = jl_get_global((jl_module_t*)m, jl_symbol("__init__")); if (f) { jl_array_ptr_1d_push(jl_module_init_order, m); diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 055ec4b3330f1..a94794135ef79 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -154,7 +154,7 @@ static void jl_compile_all_defs(jl_array_t *mis) jl_foreach_reachable_mtable(compile_all_collect_, allmeths); - size_t i, l = jl_array_len(allmeths); + size_t i, l = jl_array_nrows(allmeths); for (i = 0; i < l; i++) { jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i); if (jl_is_datatype(m->sig) && jl_isa_compileable_sig((jl_tupletype_t*)m->sig, jl_emptysvec, m)) { @@ -243,7 +243,7 @@ static void *jl_precompile_(jl_array_t *m, int external_linkage) jl_method_instance_t *mi = NULL; JL_GC_PUSH2(&m2, &mi); m2 = jl_alloc_vec_any(0); - for (size_t i = 0; i < jl_array_len(m); i++) { + for (size_t i = 0; i < jl_array_nrows(m); i++) { jl_value_t *item = jl_array_ptr_ref(m, i); if (jl_is_method_instance(item)) { mi = (jl_method_instance_t*)item; @@ -287,13 +287,13 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met // type signatures that were inferred but haven't been compiled jl_array_t *m = jl_alloc_vec_any(0); JL_GC_PUSH1(&m); - size_t i, n = jl_array_len(worklist); + size_t i, n = jl_array_nrows(worklist); for (i = 0; i < n; i++) { jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); assert(jl_is_module(mod)); foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); } - n = jl_array_len(extext_methods); + n = jl_array_nrows(extext_methods); for (i = 0; i < n; i++) { jl_method_t *method = (jl_method_t*)jl_array_ptr_ref(extext_methods, i); assert(jl_is_method(method)); @@ -310,7 +310,7 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met } } } - n = jl_array_len(new_specializations); + n = jl_array_nrows(new_specializations); for (i = 0; i < n; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_specializations, i); precompile_enq_specialization_(ci->def, m); diff --git a/src/processor.cpp b/src/processor.cpp index f9c8648866318..917383d3efb11 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -994,7 +994,7 @@ extern "C" JL_DLLEXPORT jl_value_t* jl_reflect_clone_targets() { } jl_value_t *arr = (jl_value_t*)jl_alloc_array_1d(jl_array_uint8_type, data.size()); - uint8_t *out = (uint8_t*)jl_array_data(arr); + uint8_t *out = jl_array_data(arr, uint8_t); memcpy(out, data.data(), data.size()); return arr; } diff --git a/src/rtutils.c b/src/rtutils.c index 35ab89d856783..ed60ad94f5a29 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -365,7 +365,10 @@ JL_DLLEXPORT void *(jl_symbol_name)(jl_sym_t *s) // WARNING: THIS FUNCTION IS NEVER CALLED BUT INLINE BY CCALL JL_DLLEXPORT void *jl_array_ptr(jl_array_t *a) { - return a->data; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(a->ref.mem))->layout; + if (layout->arrayelem_isunion || layout->size == 0) + return (char*)a->ref.mem->data + (size_t)jl_array_data_(a); + return jl_array_data_(a); } JL_DLLEXPORT jl_value_t *jl_value_ptr(jl_value_t *a) { @@ -1031,7 +1034,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt } else if (vt == jl_expr_type) { jl_expr_t *e = (jl_expr_t*)v; - if (e->head == jl_assign_sym && jl_array_len(e->args) == 2) { + if (e->head == jl_assign_sym && jl_array_nrows(e->args) == 2) { n += jl_static_show_x(out, jl_exprarg(e,0), depth, ctx); n += jl_printf(out, " = "); n += jl_static_show_x(out, jl_exprarg(e,1), depth, ctx); @@ -1039,7 +1042,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt else { char sep = ' '; n += jl_printf(out, "Expr(:%s", jl_symbol_name(e->head)); - size_t i, len = jl_array_len(e->args); + size_t i, len = jl_array_nrows(e->args); for (i = 0; i < len; i++) { n += jl_printf(out, ",%c", sep); n += jl_static_show_x(out, jl_exprarg(e,i), depth, ctx); @@ -1049,24 +1052,48 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt } else if (jl_array_type && jl_is_array_type(vt)) { n += jl_printf(out, "Array{"); - n += jl_static_show_x(out, (jl_value_t*)jl_tparam0(vt), depth, ctx); - n += jl_printf(out, ", ("); + jl_value_t *el_type = jl_tparam0(vt); + n += jl_static_show_x(out, el_type, depth, ctx); + jl_array_t *av = (jl_array_t*)v; size_t i, ndims = jl_array_ndims(v); + size_t offset = (size_t)av->ref.data; + if (av->ref.mem) { + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typeof(av->ref.mem))->layout; + if (layout->size != 0 && !layout->arrayelem_isunion) + offset = ((char*)offset - (char*)av->ref.mem->data) / layout->size; + } + n += jl_printf(out, ", %" PRIdPTR "}(offset=%" PRIdPTR ", dims=(", ndims, offset); if (ndims == 1) n += jl_printf(out, "%" PRIdPTR ",", jl_array_dim0(v)); else for (i = 0; i < ndims; i++) n += jl_printf(out, (i > 0 ? ", %" PRIdPTR : "%" PRIdPTR), jl_array_dim(v, i)); - n += jl_printf(out, ")}["); - size_t j, tlen = jl_array_len(v); - jl_array_t *av = (jl_array_t*)v; - jl_value_t *el_type = jl_tparam0(vt); - char *typetagdata = (!av->flags.ptrarray && jl_is_uniontype(el_type)) ? jl_array_typetagdata(av) : NULL; + n += jl_printf(out, "), mem="); + n += jl_static_show_x(out, (jl_value_t*)av->ref.mem, depth, ctx); + n += jl_printf(out, ")"); + } + else if (jl_genericmemory_type && jl_is_genericmemory_type(vt)) { + jl_genericmemory_t *m = (jl_genericmemory_t*)v; + n += jl_printf(out, "GenericMemory{"); + jl_value_t *isatomic = jl_tparam0(vt); + n += jl_static_show_x(out, isatomic, depth, ctx); + n += jl_printf(out, ", "); + jl_value_t *el_type = jl_tparam1(vt); + n += jl_static_show_x(out, el_type, depth, ctx); + size_t j, tlen = m->length; + n += jl_printf(out, "}(%" PRIdPTR ", %p)[", tlen, m->data); +//#ifdef _P64 +// n += jl_printf(out, "0x%016" PRIx64, tlen); +//#else +// n += jl_printf(out, "0x%08" PRIx32, tlen); +//#endif + const jl_datatype_layout_t *layout = vt->layout; int nlsep = 0; - if (av->flags.ptrarray) { + const char *typetagdata = NULL; + if (layout->arrayelem_isboxed) { // print arrays with newlines, unless the elements are probably small for (j = 0; j < tlen; j++) { - jl_value_t **ptr = ((jl_value_t**)av->data) + j; + jl_value_t **ptr = ((jl_value_t**)m->data) + j; jl_value_t *p = *ptr; if (p != NULL && (uintptr_t)p >= 4096U) { jl_value_t *p_ty = jl_typeof(p); @@ -1079,17 +1106,20 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt } } } + else if (jl_is_uniontype(el_type)) { + typetagdata = jl_genericmemory_typetagdata(m); + } if (nlsep && tlen > 1) n += jl_printf(out, "\n "); - for (j = 0; j < tlen; j++) { - if (av->flags.ptrarray) { - jl_value_t **ptr = ((jl_value_t**)av->data) + j; + for (size_t j = 0; j < tlen; j++) { + if (layout->arrayelem_isboxed) { + jl_value_t **ptr = ((jl_value_t**)m->data) + j; n += jl_static_show_x(out, *ptr, depth, ctx); } else { - char *ptr = ((char*)av->data) + j * av->elsize; + char *ptr = ((char*)m->data) + j * layout->size; n += jl_static_show_x_(out, (jl_value_t*)ptr, - typetagdata ? (jl_datatype_t*)jl_nth_union_component(el_type, typetagdata[j]) : (jl_datatype_t*)el_type, + (jl_datatype_t*)(typetagdata ? jl_nth_union_component(el_type, typetagdata[j]) : el_type), depth, ctx); } if (j != tlen - 1) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index ed320aa9a6c35..b2d783fc4b4c3 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -1455,12 +1455,6 @@ un_fintrinsic(rint_float,rint_llvm) un_fintrinsic(sqrt_float,sqrt_llvm) un_fintrinsic(sqrt_float,sqrt_llvm_fast) -JL_DLLEXPORT jl_value_t *jl_arraylen(jl_value_t *a) -{ - JL_TYPECHK(arraylen, array, a); - return jl_box_long(jl_array_len((jl_array_t*)a)); -} - JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *typ) { JL_TYPECHK(have_fma, datatype, typ); diff --git a/src/serialize.h b/src/serialize.h index afcdcc31d66c4..1bd29e9cc5911 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -64,8 +64,9 @@ extern "C" { #define TAG_ARGUMENT 56 #define TAG_RELOC_METHODROOT 57 #define TAG_BINDING 58 +#define TAG_MEMORYT 59 -#define LAST_TAG 58 +#define LAST_TAG 59 #define write_uint8(s, n) ios_putc((n), (s)) #define read_uint8(s) ((uint8_t)ios_getc((s))) diff --git a/src/smallintset.c b/src/smallintset.c index fa647b57e7d3e..b3af6e45c7834 100644 --- a/src/smallintset.c +++ b/src/smallintset.c @@ -28,11 +28,11 @@ static inline size_t jl_intref(const jl_array_t *arr, size_t idx) JL_NOTSAFEPOIN { jl_value_t *el = jl_tparam0(jl_typeof(arr)); if (el == (jl_value_t*)jl_uint8_type) - return jl_atomic_load_relaxed(&((_Atomic(uint8_t)*)jl_array_data(arr))[idx]); + return jl_atomic_load_relaxed(&jl_array_data(arr, _Atomic(uint8_t))[idx]); else if (el == (jl_value_t*)jl_uint16_type) - return jl_atomic_load_relaxed(&((_Atomic(uint16_t)*)jl_array_data(arr))[idx]); + return jl_atomic_load_relaxed(&jl_array_data(arr, _Atomic(uint16_t))[idx]); else if (el == (jl_value_t*)jl_uint32_type) - return jl_atomic_load_relaxed(&((_Atomic(uint32_t)*)jl_array_data(arr))[idx]); + return jl_atomic_load_relaxed(&jl_array_data(arr, _Atomic(uint32_t))[idx]); else abort(); } @@ -41,11 +41,11 @@ static inline size_t jl_intref_acquire(const jl_array_t *arr, size_t idx) JL_NOT { jl_value_t *el = jl_tparam0(jl_typeof(arr)); if (el == (jl_value_t*)jl_uint8_type) - return jl_atomic_load_acquire(&((_Atomic(uint8_t)*)jl_array_data(arr))[idx]); + return jl_atomic_load_acquire(&jl_array_data(arr, _Atomic(uint8_t))[idx]); else if (el == (jl_value_t*)jl_uint16_type) - return jl_atomic_load_acquire(&((_Atomic(uint16_t)*)jl_array_data(arr))[idx]); + return jl_atomic_load_acquire(&jl_array_data(arr, _Atomic(uint16_t))[idx]); else if (el == (jl_value_t*)jl_uint32_type) - return jl_atomic_load_acquire(&((_Atomic(uint32_t)*)jl_array_data(arr))[idx]); + return jl_atomic_load_acquire(&jl_array_data(arr, _Atomic(uint32_t))[idx]); else abort(); } @@ -54,11 +54,11 @@ static inline void jl_intset_release(const jl_array_t *arr, size_t idx, size_t v { jl_value_t *el = jl_tparam0(jl_typeof(arr)); if (el == (jl_value_t*)jl_uint8_type) - jl_atomic_store_release(&((_Atomic(uint8_t)*)jl_array_data(arr))[idx], val); + jl_atomic_store_release(&jl_array_data(arr, _Atomic(uint8_t))[idx], val); else if (el == (jl_value_t*)jl_uint16_type) - jl_atomic_store_release(&((_Atomic(uint16_t)*)jl_array_data(arr))[idx], val); + jl_atomic_store_release(&jl_array_data(arr, _Atomic(uint16_t))[idx], val); else if (el == (jl_value_t*)jl_uint32_type) - jl_atomic_store_release(&((_Atomic(uint32_t)*)jl_array_data(arr))[idx], val); + jl_atomic_store_release(&jl_array_data(arr, _Atomic(uint32_t))[idx], val); else abort(); } @@ -81,14 +81,17 @@ static inline size_t jl_max_int(const jl_array_t *arr) static jl_array_t *jl_alloc_int_1d(size_t np, size_t len) { jl_value_t *ty; + size_t elsize; if (np < 0xFF) { ty = jl_array_uint8_type; + elsize = sizeof(uint8_t); } else if (np < 0xFFFF) { static jl_value_t *int16 JL_ALWAYS_LEAFTYPE = NULL; if (int16 == NULL) int16 = jl_apply_array_type((jl_value_t*)jl_uint16_type, 1); ty = int16; + elsize = sizeof(uint16_t); } else { assert(np < 0x7FFFFFFF); @@ -96,15 +99,16 @@ static jl_array_t *jl_alloc_int_1d(size_t np, size_t len) if (int32 == NULL) int32 = jl_apply_array_type((jl_value_t*)jl_uint32_type, 1); ty = int32; + elsize = sizeof(uint32_t); } jl_array_t *a = jl_alloc_array_1d(ty, len); - memset(a->data, 0, len * a->elsize); + memset(jl_array_data(a, char), 0, len * elsize); return a; } ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv) { - size_t sz = jl_array_len(cache); + size_t sz = jl_array_nrows(cache); if (sz == 0) return -1; JL_GC_PUSH1(&cache); @@ -131,7 +135,7 @@ ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void * static int smallintset_insert_(jl_array_t *a, uint_t hv, size_t val1) { - size_t sz = jl_array_len(a); + size_t sz = jl_array_nrows(a); if (sz <= 1) return 0; size_t orig, index, iter; @@ -156,7 +160,7 @@ void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, sma { jl_array_t *a = jl_atomic_load_relaxed(pcache); if (val + 1 > jl_max_int(a)) - smallintset_rehash(pcache, parent, hash, data, jl_array_len(a), val + 1); + smallintset_rehash(pcache, parent, hash, data, jl_array_nrows(a), val + 1); while (1) { a = jl_atomic_load_relaxed(pcache); if (smallintset_insert_(a, hash(val, data), val + 1)) @@ -168,7 +172,7 @@ void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, sma /* lots of time rehashing all the keys over and over. */ size_t newsz; a = jl_atomic_load_relaxed(pcache); - size_t sz = jl_array_len(a); + size_t sz = jl_array_nrows(a); if (sz < HT_N_INLINE) newsz = HT_N_INLINE; else if (sz >= (1 << 19) || (sz <= (1 << 8))) @@ -182,7 +186,7 @@ void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, sma static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np) { jl_array_t *a = jl_atomic_load_relaxed(pcache); - size_t sz = jl_array_len(a); + size_t sz = jl_array_nrows(a); size_t i; for (i = 0; i < sz; i += 1) { size_t val = jl_intref(a, i); diff --git a/src/stackwalk.c b/src/stackwalk.c index 18bf4b2126938..f10981b51be64 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -260,21 +260,21 @@ JL_DLLEXPORT jl_value_t *jl_backtrace_from_here(int returnsp, int skip) uintptr_t *sp_ptr = NULL; if (returnsp) { jl_array_grow_end(sp, maxincr); - sp_ptr = (uintptr_t*)jl_array_data(sp) + offset; + sp_ptr = jl_array_data(sp, uintptr_t) + offset; } size_t size_incr = 0; - have_more_frames = jl_unw_stepn(&cursor, (jl_bt_element_t*)jl_array_data(ip) + offset, + have_more_frames = jl_unw_stepn(&cursor, jl_array_data(ip, jl_bt_element_t) + offset, &size_incr, sp_ptr, maxincr, skip, &pgcstack, 0); skip = 0; offset += size_incr; } - jl_array_del_end(ip, jl_array_len(ip) - offset); + jl_array_del_end(ip, jl_array_nrows(ip) - offset); if (returnsp) - jl_array_del_end(sp, jl_array_len(sp) - offset); + jl_array_del_end(sp, jl_array_nrows(sp) - offset); size_t n = 0; - jl_bt_element_t *bt_data = (jl_bt_element_t*)jl_array_data(ip); - while (n < jl_array_len(ip)) { + jl_bt_element_t *bt_data = jl_array_data(ip, jl_bt_element_t); + while (n < jl_array_nrows(ip)) { jl_bt_element_t *bt_entry = bt_data + n; if (!jl_bt_is_native(bt_entry)) { size_t njlvals = jl_bt_num_jlvals(bt_entry); @@ -303,7 +303,7 @@ static void decode_backtrace(jl_bt_element_t *bt_data, size_t bt_size, bt = *btout = jl_alloc_array_1d(array_ptr_void_type, bt_size); static_assert(sizeof(jl_bt_element_t) == sizeof(void*), "jl_bt_element_t is presented as Ptr{Cvoid} on julia side"); - memcpy(bt->data, bt_data, bt_size * sizeof(jl_bt_element_t)); + memcpy(jl_array_data(bt, jl_bt_element_t), bt_data, bt_size * sizeof(jl_bt_element_t)); bt2 = *bt2out = jl_alloc_array_1d(jl_array_any_type, 0); // Scan the backtrace buffer for any gc-managed values for (size_t i = 0; i < bt_size; i += jl_bt_entry_size(bt_data + i)) { @@ -669,7 +669,7 @@ void jl_print_bt_entry_codeloc(jl_bt_element_t *bt_entry) JL_NOTSAFEPOINT jl_code_info_t *src = (jl_code_info_t*)code; // See also the debug info handling in codegen.cpp. // NB: debuginfoloc is 1-based! - intptr_t debuginfoloc = ((int32_t*)jl_array_data(src->codelocs))[ip]; + intptr_t debuginfoloc = jl_array_data(src->codelocs, int32_t)[ip]; while (debuginfoloc != 0) { jl_line_info_node_t *locinfo = (jl_line_info_node_t*) jl_array_ptr_ref(src->linetable, debuginfoloc - 1); diff --git a/src/staticdata.c b/src/staticdata.c index 536ca4cd6c3aa..0d8ad4b9625fd 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -99,7 +99,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 159 +#define NUM_TAGS 172 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -198,6 +198,12 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_number_type); INSERT_TAG(jl_signed_type); INSERT_TAG(jl_pair_type); + INSERT_TAG(jl_genericmemory_type); + INSERT_TAG(jl_memory_any_type); + INSERT_TAG(jl_memory_uint8_type); + INSERT_TAG(jl_genericmemoryref_type); + INSERT_TAG(jl_memoryref_any_type); + INSERT_TAG(jl_memoryref_uint8_type); // special typenames INSERT_TAG(jl_tuple_typename); @@ -208,6 +214,8 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_namedtuple_typename); INSERT_TAG(jl_vecelement_typename); INSERT_TAG(jl_opaque_closure_typename); + INSERT_TAG(jl_genericmemory_typename); + INSERT_TAG(jl_genericmemoryref_typename); // special exceptions INSERT_TAG(jl_errorexception_type); @@ -233,6 +241,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_true); INSERT_TAG(jl_an_empty_string); INSERT_TAG(jl_an_empty_vec_any); + INSERT_TAG(jl_an_empty_memory_any); INSERT_TAG(jl_module_init_order); INSERT_TAG(jl_core_module); INSERT_TAG(jl_base_module); @@ -264,10 +273,10 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_builtin_modifyfield); INSERT_TAG(jl_builtin_replacefield); INSERT_TAG(jl_builtin_fieldtype); - INSERT_TAG(jl_builtin_arrayref); - INSERT_TAG(jl_builtin_const_arrayref); - INSERT_TAG(jl_builtin_arrayset); - INSERT_TAG(jl_builtin_arraysize); + INSERT_TAG(jl_builtin_memoryref); + INSERT_TAG(jl_builtin_memoryrefget); + INSERT_TAG(jl_builtin_memoryrefset); + INSERT_TAG(jl_builtin_memoryref_isassigned); INSERT_TAG(jl_builtin_apply_type); INSERT_TAG(jl_builtin_applicable); INSERT_TAG(jl_builtin_invoke); @@ -444,8 +453,8 @@ static const jl_fptr_args_t id_to_fptrs[] = { &jl_f__call_latest, &jl_f__call_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, - &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, - &jl_f_arrayref, &jl_f_const_arrayref, &jl_f_arrayset, &jl_f_arraysize, &jl_f_apply_type, + &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, + &jl_f_memoryref, &jl_f_memoryrefget, &jl_f_memoryrefset, &jl_f_memoryref_isassigned, &jl_f_applicable, &jl_f_invoke, &jl_f_sizeof, &jl_f__expr, &jl_f__typevar, &jl_f_ifelse, &jl_f__structtype, &jl_f__abstracttype, &jl_f__primitivetype, &jl_f__typebody, &jl_f__setsuper, &jl_f__equiv_typedef, &jl_f_get_binding_type, @@ -460,6 +469,7 @@ typedef struct { ios_t *relocs; // for (de)serializing relocs_list and gctags_list ios_t *gvar_record; // serialized array mapping gvid => spos ios_t *fptr_record; // serialized array mapping fptrid => spos + arraylist_t memref_list; // a list of memoryref locations arraylist_t relocs_list; // a list of (location, target) pairs, see description at top arraylist_t gctags_list; // " arraylist_t uniquing_types; // a list of locations that reference types that must be de-duplicated @@ -747,9 +757,8 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ jl_queue_for_serialization_(s, (jl_value_t*)dt->super, 1, 1); // ensure all type parameters are recached jl_queue_for_serialization_(s, (jl_value_t*)dt->parameters, 1, 1); - jl_value_t *singleton = dt->instance; - if (singleton && needs_uniquing(singleton)) { - assert(jl_needs_serialization(s, singleton)); // should be true, since we visited dt + if (jl_is_datatype_singleton(dt) && needs_uniquing(dt->instance)) { + assert(jl_needs_serialization(s, dt->instance)); // should be true, since we visited dt // do not visit dt->instance for our template object as it leads to unwanted cycles here // (it may get serialized from elsewhere though) record_field_change(&dt->instance, jl_nothing); @@ -828,22 +837,29 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ } else if (jl_is_array(v)) { jl_array_t *ar = (jl_array_t*)v; - const char *data = (const char*)jl_array_data(ar); - if (ar->flags.ptrarray) { - size_t i, l = jl_array_len(ar); + jl_value_t *mem = get_replaceable_field((jl_value_t**)&ar->ref.mem, 1); + jl_queue_for_serialization_(s, mem, 1, immediate); + } + else if (jl_is_genericmemory(v)) { + jl_genericmemory_t *m = (jl_genericmemory_t*)v; + const char *data = (const char*)m->data; + if (jl_genericmemory_how(m) == 3) { + jl_queue_for_serialization_(s, jl_genericmemory_data_owner_field(v), 1, immediate); + } + else if (layout->arrayelem_isboxed) { + size_t i, l = m->length; for (i = 0; i < l; i++) { jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[i], 1); jl_queue_for_serialization_(s, fld, 1, immediate); } } - else if (ar->flags.hasptr) { - uint16_t elsz = ar->elsize; - size_t i, l = jl_array_len(ar); - jl_datatype_t *et = (jl_datatype_t*)jl_tparam0(jl_typeof(ar)); - size_t j, np = et->layout->npointers; + else if (layout->first_ptr >= 0) { + uint16_t elsz = layout->size; + size_t i, l = m->length; + size_t j, np = layout->npointers; for (i = 0; i < l; i++) { for (j = 0; j < np; j++) { - uint32_t ptr = jl_ptr_offset(et, j); + uint32_t ptr = jl_ptr_offset(t, j); jl_value_t *fld = get_replaceable_field(&((jl_value_t**)data)[ptr], 1); jl_queue_for_serialization_(s, fld, 1, immediate); } @@ -994,8 +1010,8 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ size_t offset = (uintptr_t)v - (uintptr_t)jl_linkage_blobs.items[2*i]; offset /= sizeof(void*); assert(offset < ((uintptr_t)1 << DEPS_IDX_OFFSET) && "offset to external image too large"); - assert(n_linkage_blobs() == jl_array_len(s->buildid_depmods_idxs)); - size_t depsidx = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[i]; // map from build_id_idx -> deps_idx + assert(n_linkage_blobs() == jl_array_nrows(s->buildid_depmods_idxs)); + size_t depsidx = jl_array_data(s->buildid_depmods_idxs, uint32_t)[i]; // map from build_id_idx -> deps_idx assert(depsidx < INT32_MAX); if (depsidx < ((uintptr_t)1 << (RELOC_TAG_OFFSET - DEPS_IDX_OFFSET)) && offset < ((uintptr_t)1 << DEPS_IDX_OFFSET)) // if it fits in a SysimageLinkage type, use that representation @@ -1003,8 +1019,8 @@ static uintptr_t add_external_linkage(jl_serializer_state *s, jl_value_t *v, jl_ // otherwise, we store the image key in `link_ids` assert(link_ids && jl_is_array(link_ids)); jl_array_grow_end(link_ids, 1); - uint32_t *link_id_data = (uint32_t*)jl_array_data(link_ids); // wait until after the `grow` - link_id_data[jl_array_len(link_ids) - 1] = depsidx; + uint32_t *link_id_data = jl_array_data(link_ids, uint32_t); // wait until after the `grow` + link_id_data[jl_array_nrows(link_ids) - 1] = depsidx; return ((uintptr_t)ExternalLinkage << RELOC_TAG_OFFSET) + offset; } return 0; @@ -1158,6 +1174,36 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t assert(ios_pos(s->s) - reloc_offset == tot); } +static void record_memoryref(jl_serializer_state *s, size_t reloc_offset, jl_genericmemoryref_t ref) { + ios_t *f = s->s; + // make some header modifications in-place + jl_genericmemoryref_t *newref = (jl_genericmemoryref_t*)&f->buf[reloc_offset]; + const jl_datatype_layout_t *layout = ((jl_datatype_t*)jl_typetagof(ref.mem))->layout; + if (!layout->arrayelem_isunion && layout->size != 0) { + newref->data = (void*)((char*)ref.data - (char*)ref.mem->data); // relocation offset (bytes) + arraylist_push(&s->memref_list, (void*)reloc_offset); // relocation location + arraylist_push(&s->memref_list, NULL); // relocation target (ignored) + } +} + +static void record_memoryrefs_inside(jl_serializer_state *s, jl_datatype_t *t, size_t reloc_offset, const char *data) +{ + assert(jl_is_datatype(t)); + size_t i, nf = jl_datatype_nfields(t); + for (i = 0; i < nf; i++) { + size_t offset = jl_field_offset(t, i); + if (jl_field_isptr(t, i)) + continue; + jl_value_t *ft = jl_field_type_concrete(t, i); + if (jl_is_uniontype(ft)) + continue; + if (jl_is_genericmemoryref_type(ft)) + record_memoryref(s, reloc_offset + offset, *(jl_genericmemoryref_t*)(data + offset)); + else + record_memoryrefs_inside(s, (jl_datatype_t*)ft, reloc_offset + offset, data + offset); + } +} + static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFEPOINT { for (size_t i = 0; i < globals->len; i++) @@ -1200,7 +1246,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED JL_GC_PROMISE_ROOTED(v); assert(!(s->incremental && jl_object_in_image(v))); jl_datatype_t *t = (jl_datatype_t*)jl_typeof(v); - assert((t->instance == NULL || t->instance == v) && "detected singleton construction corruption"); + assert((!jl_is_datatype_singleton(t) || t->instance == v) && "detected singleton construction corruption"); ios_t *f = s->s; if (t->smalltag) { if (t->layout->npointers == 0 || t == jl_string_type) { @@ -1259,102 +1305,142 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED if (jl_is_array(v)) { assert(f == s->s); // Internal data for types in julia.h with `jl_array_t` field(s) -#define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes) jl_array_t *ar = (jl_array_t*)v; - jl_value_t *et = jl_tparam0(jl_typeof(v)); - size_t alen = jl_array_len(ar); - size_t datasize = alen * ar->elsize; - size_t tot = datasize; - int isbitsunion = jl_array_isbitsunion(ar); - if (isbitsunion) - tot += alen; - else if (ar->elsize == 1) - tot += 1; - int ndimwords = jl_array_ndimwords(ar->flags.ndims); - size_t headersize = sizeof(jl_array_t) + ndimwords*sizeof(size_t); // copy header + size_t headersize = sizeof(jl_array_t) + jl_array_ndims(ar)*sizeof(size_t); ios_write(f, (char*)v, headersize); - size_t alignment_amt = JL_SMALL_BYTE_ALIGNMENT; - if (tot >= ARRAY_CACHE_ALIGN_THRESHOLD) - alignment_amt = JL_CACHE_BYTE_ALIGNMENT; // make some header modifications in-place jl_array_t *newa = (jl_array_t*)&f->buf[reloc_offset]; - if (newa->flags.ndims == 1) - newa->maxsize = alen; - newa->offset = 0; - newa->flags.how = 0; - newa->flags.pooled = 0; - newa->flags.isshared = 0; - - // write data - if (!ar->flags.ptrarray && !ar->flags.hasptr) { - // Non-pointer eltypes get encoded in the const_data section - uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), alignment_amt); - write_padding(s->const_data, data - ios_pos(s->const_data)); - // write data and relocations - newa->data = NULL; // relocation offset - data /= sizeof(void*); - assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large"); - arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location - arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target - if (jl_is_cpointer_type(et)) { - // reset Ptr fields to C_NULL (but keep MAP_FAILED / INVALID_HANDLE) - const intptr_t *data = (const intptr_t*)jl_array_data(ar); - size_t i; - for (i = 0; i < alen; i++) { - if (data[i] != -1) - write_pointer(s->const_data); - else - ios_write(s->const_data, (char*)&data[i], sizeof(data[i])); - } - } - else { - if (isbitsunion) { - ios_write(s->const_data, (char*)jl_array_data(ar), datasize); - ios_write(s->const_data, jl_array_typetagdata(ar), alen); + newa->ref.mem = NULL; // relocation offset + arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, ref.mem))); // relocation location + jl_value_t *mem = get_replaceable_field((jl_value_t**)&ar->ref.mem, 1); + arraylist_push(&s->relocs_list, (void*)backref_id(s, mem, s->link_ids_relocs)); // relocation target + record_memoryref(s, reloc_offset + offsetof(jl_array_t, ref), ar->ref); + } + else if (jl_is_genericmemory(v)) { + assert(f == s->s); + // Internal data for types in julia.h with `jl_genericmemory_t` field(s) + jl_genericmemory_t *m = (jl_genericmemory_t*)v; + const jl_datatype_layout_t *layout = t->layout; + size_t len = m->length; + if (jl_genericmemory_how(m) == 3 && jl_is_genericmemory(jl_genericmemory_data_owner_field(m))) { + jl_genericmemory_t *owner = (jl_genericmemory_t*)jl_genericmemory_data_owner_field(m); + size_t data = ((char*)m->data - (char*)owner->data); // relocation offset (bytes) + write_uint(f, len); + write_uint(f, data); + write_pointerfield(s, (jl_value_t*)owner); + // perform record_memoryref + arraylist_push(&s->memref_list, (void*)(reloc_offset + offsetof(jl_genericmemory_t, data))); // relocation location + arraylist_push(&s->memref_list, NULL); // relocation target (ignored) + } + // else if (jl_genericmemory_how(m) == 3) { + // jl_value_t *owner = jl_genericmemory_data_owner_field(m); + // write_uint(f, len); + // write_pointerfield(s, owner); + // write_pointerfield(s, owner); + // jl_genericmemory_t *new_mem = (jl_genericmemory_t*)&f->buf[reloc_offset]; + // assert(new_mem->data == NULL); + // new_mem->data = (void*)((char*)m->data - (char*)owner); // relocation offset + // } + else { + size_t datasize = len * layout->size; + size_t tot = datasize; + int isbitsunion = layout->arrayelem_isunion; + if (isbitsunion) + tot += len; + size_t headersize = sizeof(jl_genericmemory_t); + // copy header + ios_write(f, (char*)v, headersize); + size_t alignment_amt = JL_SMALL_BYTE_ALIGNMENT; + if (tot >= ARRAY_CACHE_ALIGN_THRESHOLD) + alignment_amt = JL_CACHE_BYTE_ALIGNMENT; + // write data + if (!layout->arrayelem_isboxed && layout->first_ptr < 0) { + // set owner to NULL + headersize += sizeof(void*); + write_pointer(f); + // Non-pointer eltypes get encoded in the const_data section + uintptr_t data = LLT_ALIGN(ios_pos(s->const_data), alignment_amt); + write_padding(s->const_data, data - ios_pos(s->const_data)); + // write data and relocations + jl_genericmemory_t *new_mem = (jl_genericmemory_t*)&f->buf[reloc_offset]; + new_mem->data = NULL; // relocation offset + data /= sizeof(void*); + assert(data < ((uintptr_t)1 << RELOC_TAG_OFFSET) && "offset to constant data too large"); + arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_genericmemory_t, data))); // relocation location + arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + data)); // relocation target + jl_value_t *et = jl_tparam1(t); + if (jl_is_cpointer_type(et)) { + // reset Ptr fields to C_NULL (but keep MAP_FAILED / INVALID_HANDLE) + const intptr_t *data = (const intptr_t*)m->data; + size_t i; + for (i = 0; i < len; i++) { + if (data[i] != -1) + write_pointer(s->const_data); + else + ios_write(s->const_data, (char*)&data[i], sizeof(data[i])); + } } else { - ios_write(s->const_data, (char*)jl_array_data(ar), tot); + if (isbitsunion) { + ios_write(s->const_data, (char*)m->data, datasize); + ios_write(s->const_data, jl_genericmemory_typetagdata(m), len); + } + else { + ios_write(s->const_data, (char*)m->data, tot); + } } + if (len == 0) // TODO: should we have a zero-page, instead of writing each type's fragment separately? + write_padding(s->const_data, layout->size ? layout->size : isbitsunion); + else if (jl_genericmemory_how(m) == 3 && jl_is_string(jl_genericmemory_data_owner_field(m))) + write_padding(s->const_data, 1); } - } - else { - // Pointer eltypes are encoded in the mutable data section - size_t data = LLT_ALIGN(ios_pos(f), alignment_amt); - size_t padding_amt = data - ios_pos(f); - headersize += padding_amt; - newa->data = (void*)headersize; // relocation offset - arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_array_t, data))); // relocation location - arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target - write_padding(f, padding_amt); - if (ar->flags.hasptr) { - // copy all of the data first - const char *data = (const char*)jl_array_data(ar); - ios_write(f, data, datasize); - // the rewrite all of the embedded pointers to null+relocation - uint16_t elsz = ar->elsize; - size_t j, np = ((jl_datatype_t*)et)->layout->npointers; - size_t i; - for (i = 0; i < alen; i++) { - for (j = 0; j < np; j++) { - size_t offset = i * elsz + jl_ptr_offset(((jl_datatype_t*)et), j) * sizeof(jl_value_t*); - jl_value_t *fld = get_replaceable_field((jl_value_t**)&data[offset], 1); - size_t fld_pos = reloc_offset + headersize + offset; - if (fld != NULL) { - arraylist_push(&s->relocs_list, (void*)(uintptr_t)fld_pos); // relocation location - arraylist_push(&s->relocs_list, (void*)backref_id(s, fld, s->link_ids_relocs)); // relocation target - record_uniquing(s, fld, fld_pos); + else { + // Pointer eltypes are encoded in the mutable data section + size_t data = LLT_ALIGN(ios_pos(f), alignment_amt); + size_t padding_amt = data - ios_pos(f); + jl_genericmemory_t *new_mem = (jl_genericmemory_t*)&f->buf[reloc_offset]; + if (padding_amt != 0) { + // set owner to NULL if we want to increase alignment + // from the default of 16 + assert(alignment_amt > 16); + assert(padding_amt >= sizeof(void*)); + padding_amt -= sizeof(void*); + headersize += sizeof(void*); + write_pointer(f); + } + new_mem->data = (void*)headersize; // relocation offset + arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_genericmemory_t, data))); // relocation location + arraylist_push(&s->relocs_list, (void*)(((uintptr_t)DataRef << RELOC_TAG_OFFSET) + item)); // relocation target + if (!layout->arrayelem_isboxed) { + // copy all of the data first + const char *data = (const char*)m->data; + ios_write(f, data, datasize); + // the rewrite all of the embedded pointers to null+relocation + uint16_t elsz = layout->size; + size_t j, np = layout->first_ptr < 0 ? 0 : layout->npointers; + size_t i; + for (i = 0; i < len; i++) { + for (j = 0; j < np; j++) { + size_t offset = i * elsz + jl_ptr_offset(t, j) * sizeof(jl_value_t*); + jl_value_t *fld = get_replaceable_field((jl_value_t**)&data[offset], 1); + size_t fld_pos = reloc_offset + headersize + offset; + if (fld != NULL) { + arraylist_push(&s->relocs_list, (void*)(uintptr_t)fld_pos); // relocation location + arraylist_push(&s->relocs_list, (void*)backref_id(s, fld, s->link_ids_relocs)); // relocation target + record_uniquing(s, fld, fld_pos); + } + memset(&f->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) } - memset(&f->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) } } - } - else { - jl_value_t **data = (jl_value_t**)jl_array_data(ar); - size_t i; - for (i = 0; i < alen; i++) { - jl_value_t *e = get_replaceable_field(&data[i], 1); - write_pointerfield(s, e); + else { + jl_value_t **data = (jl_value_t**)m->data; + size_t i; + for (i = 0; i < len; i++) { + jl_value_t *e = get_replaceable_field(&data[i], 1); + write_pointerfield(s, e); + } } } } @@ -1384,7 +1470,6 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } else if (jl_datatype_nfields(t) == 0) { // The object has no fields, so we just snapshot its byte representation - assert(!t->layout->npointers); assert(t->layout->npointers == 0); ios_write(f, (char*)v, jl_datatype_size(t)); } @@ -1418,7 +1503,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED write_padding(f, offset - tot); tot = offset; size_t fsz = jl_field_size(t, i); - if (t->name->mutabl && jl_is_cpointer_type(jl_field_type(t, i)) && *(intptr_t*)slot != -1) { + if (t->name->mutabl && jl_is_cpointer_type(jl_field_type_concrete(t, i)) && *(intptr_t*)slot != -1) { // reset Ptr fields to C_NULL (but keep MAP_FAILED / INVALID_HANDLE) assert(!jl_field_isptr(t, i)); write_pointer(f); @@ -1445,8 +1530,10 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED memset(&f->buf[fld_pos], 0, sizeof(fld)); // relocation offset (none) } - // A few objects need additional handling beyond the generic serialization above + // Need do a tricky fieldtype walk an record all memoryref we find inlined in this value + record_memoryrefs_inside(s, t, reloc_offset, data); + // A few objects need additional handling beyond the generic serialization above if (s->incremental && jl_typetagis(v, jl_typemap_entry_type)) { assert(f == s->s); jl_typemap_entry_t *newentry = (jl_typemap_entry_t*)&s->s->buf[reloc_offset]; @@ -1639,6 +1726,10 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED // will need to rehash this, later (after types are fully constructed) arraylist_push(&s->fixup_objs, (void*)reloc_offset); } + else if (jl_is_genericmemoryref(v)) { + assert(f == s->s); + record_memoryref(s, reloc_offset, *(jl_genericmemoryref_t*)v); + } else { write_padding(f, jl_datatype_size(t) - tot); } @@ -1790,7 +1881,7 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas size_t depsidx = 0; #endif assert(s->buildid_depmods_idxs && depsidx < jl_array_len(s->buildid_depmods_idxs)); - size_t i = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[depsidx]; + size_t i = jl_array_data(s->buildid_depmods_idxs, uint32_t)[depsidx]; assert(2*i < jl_linkage_blobs.len); return (uintptr_t)jl_linkage_blobs.items[2*i] + offset*sizeof(void*); } @@ -1798,10 +1889,10 @@ static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t bas assert(link_ids); assert(link_index); assert(0 <= *link_index && *link_index < jl_array_len(link_ids)); - uint32_t depsidx = ((uint32_t*)jl_array_data(link_ids))[*link_index]; + uint32_t depsidx = jl_array_data(link_ids, uint32_t)[*link_index]; *link_index += 1; assert(depsidx < jl_array_len(s->buildid_depmods_idxs)); - size_t i = ((uint32_t*)jl_array_data(s->buildid_depmods_idxs))[depsidx]; + size_t i = jl_array_data(s->buildid_depmods_idxs, uint32_t)[depsidx]; assert(2*i < jl_linkage_blobs.len); return (uintptr_t)jl_linkage_blobs.items[2*i] + offset*sizeof(void*); } @@ -1890,6 +1981,37 @@ static void jl_read_reloclist(jl_serializer_state *s, jl_array_t *link_ids, uint assert(!link_ids || link_index == jl_array_len(link_ids)); } +static void jl_read_memreflist(jl_serializer_state *s) +{ + uintptr_t base = (uintptr_t)s->s->buf; + uintptr_t last_pos = 0; + uint8_t *current = (uint8_t *)(s->relocs->buf + s->relocs->bpos); + while (1) { + // Read the offset of the next object + size_t pos_diff = 0; + size_t cnt = 0; + while (1) { + assert(s->relocs->bpos <= s->relocs->size); + assert((char *)current <= (char *)(s->relocs->buf + s->relocs->size)); + int8_t c = *current++; + s->relocs->bpos += 1; + + pos_diff |= ((size_t)c & 0x7F) << (7 * cnt++); + if ((c >> 7) == 0) + break; + } + if (pos_diff == 0) + break; + + uintptr_t pos = last_pos + pos_diff; + last_pos = pos; + jl_genericmemoryref_t *pv = (jl_genericmemoryref_t*)(base + pos); + size_t offset = (size_t)pv->data; + pv->data = (void*)((char*)pv->mem->data + offset); + } +} + + static void jl_read_arraylist(ios_t *s, arraylist_t *list) { size_t list_len = read_uint(s); @@ -2187,7 +2309,7 @@ static jl_value_t *strip_codeinfo_meta(jl_method_t *m, jl_value_t *ci_, int orig ci = (jl_code_info_t*)ci_; } // leave codelocs length the same so the compiler can assume that; just zero it - memset(jl_array_data(ci->codelocs), 0, jl_array_len(ci->codelocs)*sizeof(int32_t)); + memset(jl_array_data(ci->codelocs, int32_t), 0, jl_array_len(ci->codelocs)*sizeof(int32_t)); // empty linetable if (jl_is_array(ci->linetable)) jl_array_del_end((jl_array_t*)ci->linetable, jl_array_len(ci->linetable)); @@ -2297,7 +2419,7 @@ static void jl_strip_all_codeinfos(void) // --- entry points --- -jl_array_t *jl_global_roots_table; +jl_genericmemory_t *jl_global_roots_table; jl_mutex_t global_roots_lock; JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT @@ -2358,7 +2480,7 @@ static void jl_prepare_serialization_data(jl_array_t *mod_array, jl_array_t *new // Collect method extensions and edges data JL_GC_PUSH1(&edges_map); if (edges) - edges_map = jl_alloc_vec_any(0); + edges_map = jl_alloc_memory_any(0); *extext_methods = jl_alloc_vec_any(0); jl_collect_methtable_from_mod(jl_type_type_mt, *extext_methods); jl_collect_methtable_from_mod(jl_nonfunction_mt, *extext_methods); @@ -2428,6 +2550,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, s.gvar_record = &gvar_record; s.fptr_record = &fptr_record; s.ptls = jl_current_task->ptls; + arraylist_new(&s.memref_list, 0); arraylist_new(&s.relocs_list, 0); arraylist_new(&s.gctags_list, 0); arraylist_new(&s.uniquing_types, 0); @@ -2591,6 +2714,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_finish_relocs(base + sysimg_offset, sysimg_size, &s.relocs_list); jl_write_offsetlist(s.relocs, sysimg_size, &s.gctags_list); jl_write_offsetlist(s.relocs, sysimg_size, &s.relocs_list); + jl_write_offsetlist(s.relocs, sysimg_size, &s.memref_list); if (s.incremental) { jl_write_arraylist(s.relocs, &s.uniquing_types); jl_write_arraylist(s.relocs, &s.uniquing_objs); @@ -2648,13 +2772,13 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_write_value(&s, edges); } write_uint32(f, jl_array_len(s.link_ids_gctags)); - ios_write(f, (char*)jl_array_data(s.link_ids_gctags), jl_array_len(s.link_ids_gctags) * sizeof(uint32_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_gctags, uint32_t), jl_array_len(s.link_ids_gctags) * sizeof(uint32_t)); write_uint32(f, jl_array_len(s.link_ids_relocs)); - ios_write(f, (char*)jl_array_data(s.link_ids_relocs), jl_array_len(s.link_ids_relocs) * sizeof(uint32_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_relocs, uint32_t), jl_array_len(s.link_ids_relocs) * sizeof(uint32_t)); write_uint32(f, jl_array_len(s.link_ids_gvars)); - ios_write(f, (char*)jl_array_data(s.link_ids_gvars), jl_array_len(s.link_ids_gvars) * sizeof(uint32_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_gvars, uint32_t), jl_array_len(s.link_ids_gvars) * sizeof(uint32_t)); write_uint32(f, jl_array_len(s.link_ids_external_fnvars)); - ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars), jl_array_len(s.link_ids_external_fnvars) * sizeof(uint32_t)); + ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars, uint32_t), jl_array_len(s.link_ids_external_fnvars) * sizeof(uint32_t)); write_uint32(f, external_fns_begin); jl_write_arraylist(s.s, &s.ccallable_list); } @@ -2664,6 +2788,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, arraylist_free(&serialization_queue); arraylist_free(&layout_table); arraylist_free(&s.ccallable_list); + arraylist_free(&s.memref_list); arraylist_free(&s.relocs_list); arraylist_free(&s.gctags_list); arraylist_free(&gvars); @@ -2932,7 +3057,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl JL_SMALL_TYPEOF(XX) #undef XX export_jl_small_typeof(); - jl_global_roots_table = (jl_array_t*)jl_read_value(&s); + jl_global_roots_table = (jl_genericmemory_t*)jl_read_value(&s); // set typeof extra-special values now that we have the type set by tags above jl_astaggedvalue(jl_nothing)->header = (uintptr_t)jl_nothing_type | jl_astaggedvalue(jl_nothing)->header; s.ptls->root_task->tls = jl_read_value(&s); @@ -2959,22 +3084,22 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl size_t nlinks_gctags = read_uint32(f); if (nlinks_gctags > 0) { s.link_ids_gctags = jl_alloc_array_1d(jl_array_int32_type, nlinks_gctags); - ios_read(f, (char*)jl_array_data(s.link_ids_gctags), nlinks_gctags * sizeof(uint32_t)); + ios_read(f, (char*)jl_array_data(s.link_ids_gctags, uint32_t), nlinks_gctags * sizeof(uint32_t)); } size_t nlinks_relocs = read_uint32(f); if (nlinks_relocs > 0) { s.link_ids_relocs = jl_alloc_array_1d(jl_array_int32_type, nlinks_relocs); - ios_read(f, (char*)jl_array_data(s.link_ids_relocs), nlinks_relocs * sizeof(uint32_t)); + ios_read(f, (char*)jl_array_data(s.link_ids_relocs, uint32_t), nlinks_relocs * sizeof(uint32_t)); } size_t nlinks_gvars = read_uint32(f); if (nlinks_gvars > 0) { s.link_ids_gvars = jl_alloc_array_1d(jl_array_int32_type, nlinks_gvars); - ios_read(f, (char*)jl_array_data(s.link_ids_gvars), nlinks_gvars * sizeof(uint32_t)); + ios_read(f, (char*)jl_array_data(s.link_ids_gvars, uint32_t), nlinks_gvars * sizeof(uint32_t)); } size_t nlinks_external_fnvars = read_uint32(f); if (nlinks_external_fnvars > 0) { s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_int32_type, nlinks_external_fnvars); - ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars), nlinks_external_fnvars * sizeof(uint32_t)); + ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars, uint32_t), nlinks_external_fnvars * sizeof(uint32_t)); } uint32_t external_fns_begin = read_uint32(f); jl_read_arraylist(s.s, ccallable_list ? ccallable_list : &s.ccallable_list); @@ -3007,6 +3132,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl size_t sizeof_tags = ios_pos(&relocs); (void)sizeof_tags; jl_read_reloclist(&s, s.link_ids_relocs, 0); // general relocs + jl_read_memreflist(&s); // memoryref relocs // s.link_ids_gvars will be processed in `jl_update_all_gvars` // s.link_ids_external_fns will be processed in `jl_update_all_gvars` jl_update_all_gvars(&s, image, external_fns_begin); // gvars relocs @@ -3092,8 +3218,9 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl // leave most fields undefined for now, but we may need instance later, // and we overwrite the name field (field 0) now so preserve it too if (dt->instance) { - assert(dt->instance == jl_nothing); - newdt->instance = dt->instance = jl_gc_permobj(0, newdt); + if (dt->instance == jl_nothing) + dt->instance = jl_gc_permobj(0, newdt); + newdt->instance = dt->instance; } static_assert(offsetof(jl_datatype_t, name) == 0, ""); newdt->name = dt->name; @@ -3108,9 +3235,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl } else { assert(!(image_base < (char*)otyp && (char*)otyp <= image_base + sizeof_sysimg)); - assert(jl_is_datatype_singleton((jl_datatype_t*)otyp) && "unreachable"); newobj = ((jl_datatype_t*)otyp)->instance; - assert(newobj != jl_nothing); + assert(newobj && newobj != jl_nothing); arraylist_push(&cleanup_list, (void*)obj); } if (tag) @@ -3271,9 +3397,9 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl else { // rehash IdDict //assert(((jl_datatype_t*)(jl_typeof(obj)))->name == jl_idtable_typename); - jl_array_t **a = (jl_array_t**)obj; - assert(jl_typetagis(*a, jl_array_any_type)); - *a = jl_idtable_rehash(*a, jl_array_len(*a)); + jl_genericmemory_t **a = (jl_genericmemory_t**)obj; + assert(jl_typetagis(*a, jl_memory_any_type)); + *a = jl_idtable_rehash(*a, (*a)->length); jl_gc_wb(obj, *a); } } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index a4cbc3fd5ebc4..8cc38dc966660 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -1,5 +1,5 @@ // inverse of backedges graph (caller=>callees hash) -jl_array_t *edges_map JL_GLOBALLY_ROOTED = NULL; // rooted for the duration of our uses of this +jl_genericmemory_t *edges_map JL_GLOBALLY_ROOTED = NULL; // rooted for the duration of our uses of this static void write_float64(ios_t *s, double x) JL_NOTSAFEPOINT { @@ -74,7 +74,7 @@ int must_be_new_dt(jl_value_t *t, htable_t *news, char *image_base, size_t sizeo static uint64_t jl_worklist_key(jl_array_t *worklist) JL_NOTSAFEPOINT { assert(jl_is_array(worklist)); - size_t len = jl_array_len(worklist); + size_t len = jl_array_nrows(worklist); if (len > 0) { jl_module_t *topmod = (jl_module_t*)jl_array_ptr_ref(worklist, len-1); assert(jl_is_module(topmod)); @@ -98,9 +98,9 @@ JL_DLLEXPORT void jl_set_newly_inferred(jl_value_t* _newly_inferred) JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci) { JL_LOCK(&newly_inferred_mutex); - size_t end = jl_array_len(newly_inferred); + size_t end = jl_array_nrows(newly_inferred); jl_array_grow_end(newly_inferred, 1); - jl_arrayset(newly_inferred, ci, end); + jl_array_ptr_set(newly_inferred, end, ci); JL_UNLOCK(&newly_inferred_mutex); } @@ -177,7 +177,7 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, arraylist_push(stack, (void*)mi); int depth = stack->len; *bp = (void*)((char*)HT_NOTFOUND + 4 + depth); // preliminarily mark as in-progress - size_t i = 0, n = jl_array_len(mi->backedges); + size_t i = 0, n = jl_array_nrows(mi->backedges); int cycle = depth; while (i < n) { jl_method_instance_t *be; @@ -222,7 +222,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list) htable_t visited; arraylist_t stack; assert(jl_is_array(list)); - size_t n0 = jl_array_len(list); + size_t n0 = jl_array_nrows(list); htable_new(&visited, n0); arraylist_new(&stack, 0); jl_array_t *new_specializations = jl_alloc_vec_any(0); @@ -251,8 +251,8 @@ static jl_array_t *queue_external_cis(jl_array_t *list) arraylist_free(&stack); JL_GC_POP(); // reverse new_specializations - n0 = jl_array_len(new_specializations); - jl_value_t **news = (jl_value_t**)jl_array_data(new_specializations); + n0 = jl_array_nrows(new_specializations); + jl_value_t **news = jl_array_data(new_specializations, jl_value_t*); for (i = 0; i < n0; i++) { jl_value_t *temp = news[i]; news[i] = news[n0 - i - 1]; @@ -266,7 +266,7 @@ static void jl_collect_new_roots(jl_array_t *roots, jl_array_t *new_specializati { htable_t mset; htable_new(&mset, 0); - size_t l = new_specializations ? jl_array_len(new_specializations) : 0; + size_t l = new_specializations ? jl_array_nrows(new_specializations) : 0; for (size_t i = 0; i < l; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_specializations, i); assert(jl_is_code_instance(ci)); @@ -289,10 +289,10 @@ static void jl_collect_new_roots(jl_array_t *roots, jl_array_t *new_specializati jl_array_ptr_1d_push(roots, (jl_value_t*)newroots); rle_iter_state rootiter = rle_iter_init(0); uint64_t *rletable = NULL; - size_t nblocks2 = 0, nroots = jl_array_len(m->roots), k = 0; + size_t nblocks2 = 0, nroots = jl_array_nrows(m->roots), k = 0; if (m->root_blocks) { - rletable = (uint64_t*)jl_array_data(m->root_blocks); - nblocks2 = jl_array_len(m->root_blocks); + rletable = jl_array_data(m->root_blocks, uint64_t); + nblocks2 = jl_array_nrows(m->root_blocks); } while (rle_iter_increment(&rootiter, nroots, rletable, nblocks2)) if (rootiter.key == key) @@ -314,7 +314,7 @@ static void jl_collect_missing_backedges(jl_methtable_t *mt) { jl_array_t *backedges = mt->backedges; if (backedges) { - size_t i, l = jl_array_len(backedges); + size_t i, l = jl_array_nrows(backedges); for (i = 1; i < l; i += 2) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(backedges, i); jl_value_t *missing_callee = jl_array_ptr_ref(backedges, i - 1); // signature of abstract callee @@ -337,7 +337,7 @@ static void collect_backedges(jl_method_instance_t *callee, int internal) { jl_array_t *backedges = callee->backedges; if (backedges) { - size_t i = 0, l = jl_array_len(backedges); + size_t i = 0, l = jl_array_nrows(backedges); while (i < l) { jl_value_t *invokeTypes; jl_method_instance_t *caller; @@ -411,7 +411,7 @@ static void jl_record_edges(jl_method_instance_t *caller, arraylist_t *wq, jl_ar if (callees != NULL) { jl_array_ptr_1d_push(edges, (jl_value_t*)caller); jl_array_ptr_1d_push(edges, (jl_value_t*)callees); - size_t i, l = jl_array_len(callees); + size_t i, l = jl_array_nrows(callees); for (i = 1; i < l; i += 2) { jl_method_instance_t *c = (jl_method_instance_t*)jl_array_ptr_ref(callees, i); if (c && jl_is_method_instance(c)) { @@ -431,7 +431,7 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra htable_t external_mis; htable_new(&external_mis, 0); if (external_cis) { - for (size_t i = 0; i < jl_array_len(external_cis); i++) { + for (size_t i = 0; i < jl_array_nrows(external_cis); i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(external_cis, i); jl_method_instance_t *mi = ci->def; ptrhash_put(&external_mis, (void*)mi, (void*)mi); @@ -439,10 +439,10 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra } arraylist_t wq; arraylist_new(&wq, 0); - void **table = (void**)jl_array_data(edges_map); // edges_map is caller => callees - size_t table_size = jl_array_len(edges_map); + void **table = (void**) edges_map->data; // edges_map is caller => callees + size_t table_size = edges_map->length; for (size_t i = 0; i < table_size; i += 2) { - assert(table == jl_array_data(edges_map) && table_size == jl_array_len(edges_map) && + assert(table == edges_map->data && table_size == edges_map->length && "edges_map changed during iteration"); jl_method_instance_t *caller = (jl_method_instance_t*)table[i]; jl_array_t *callees = (jl_array_t*)table[i + 1]; @@ -464,7 +464,7 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra htable_t edges_map2; htable_new(&edges_map2, 0); htable_t edges_ids; - size_t l = edges ? jl_array_len(edges) : 0; + size_t l = edges ? jl_array_nrows(edges) : 0; htable_new(&edges_ids, l); for (size_t i = 0; i < l / 2; i++) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, i * 2); @@ -479,9 +479,9 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra JL_GC_PUSH3(&matches, &callee_ids, &sig); for (size_t i = 0; i < l; i += 2) { jl_array_t *callees = (jl_array_t*)jl_array_ptr_ref(edges, i + 1); - size_t l = jl_array_len(callees); + size_t l = jl_array_nrows(callees); callee_ids = jl_alloc_array_1d(jl_array_int32_type, l + 1); - int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + int32_t *idxs = jl_array_data(callee_ids, int32_t); idxs[0] = 0; size_t nt = 0; for (size_t j = 0; j < l; j += 2) { @@ -534,7 +534,7 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra break; } size_t k; - for (k = 0; k < jl_array_len(matches); k++) { + for (k = 0; k < jl_array_nrows(matches); k++) { jl_method_match_t *match = (jl_method_match_t *)jl_array_ptr_ref(matches, k); jl_array_ptr_set(matches, k, match->method); } @@ -542,7 +542,7 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra jl_array_ptr_1d_push(ext_targets, invokeTypes); jl_array_ptr_1d_push(ext_targets, callee); jl_array_ptr_1d_push(ext_targets, matches); - target = (void*)((char*)HT_NOTFOUND + jl_array_len(ext_targets) / 3); + target = (void*)((char*)HT_NOTFOUND + jl_array_nrows(ext_targets) / 3); ptrhash_put(&edges_map2, (void*)callee, target); } idxs[++nt] = (char*)target - (char*)HT_NOTFOUND - 1; @@ -574,7 +574,7 @@ static void jl_collect_edges(jl_array_t *edges, jl_array_t *ext_targets, jl_arra static void write_mod_list(ios_t *s, jl_array_t *a) { size_t i; - size_t len = jl_array_len(a); + size_t len = jl_array_nrows(a); for (i = 0; i < len; i++) { jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(a, i); assert(jl_is_module(m)); @@ -656,7 +656,7 @@ static int64_t write_header(ios_t *s, uint8_t pkgimage) // serialize information about the result of deserializing this file static void write_worklist_for_header(ios_t *s, jl_array_t *worklist) { - int i, l = jl_array_len(worklist); + int i, l = jl_array_nrows(worklist); for (i = 0; i < l; i++) { jl_module_t *workmod = (jl_module_t*)jl_array_ptr_ref(worklist, i); if (workmod->parent == jl_main_module || workmod->parent == workmod) { @@ -710,7 +710,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // dependencies if we don't need them initial_pos = ios_pos(s); write_uint64(s, 0); - size_t i, l = udeps ? jl_array_len(udeps) : 0; + size_t i, l = udeps ? jl_array_nrows(udeps) : 0; for (i = 0; i < l; i++) { jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath @@ -723,7 +723,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t while (depmod_top->parent != jl_main_module && depmod_top->parent != depmod_top) depmod_top = depmod_top->parent; unsigned provides = 0; - size_t j, lj = jl_array_len(worklist); + size_t j, lj = jl_array_nrows(worklist); for (j = 0; j < lj; j++) { jl_module_t *workmod = (jl_module_t*)jl_array_ptr_ref(worklist, j); if (workmod->parent == jl_main_module || workmod->parent == workmod) { @@ -770,7 +770,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // If we successfully got the preferences, write it out, otherwise write `0` for this `.ji` file. if (prefs_hash != NULL && prefs_list != NULL) { - size_t i, l = jl_array_len(prefs_list); + size_t i, l = jl_array_nrows(prefs_list); for (i = 0; i < l; i++) { jl_value_t *pref_name = jl_array_ptr_ref(prefs_list, i); size_t slen = jl_string_len(pref_name); @@ -806,7 +806,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t* worklist, jl_array_t // Add methods to external (non-worklist-owned) functions static void jl_insert_methods(jl_array_t *list) { - size_t i, l = jl_array_len(list); + size_t i, l = jl_array_nrows(list); for (i = 0; i < l; i++) { jl_method_t *meth = (jl_method_t*)jl_array_ptr_ref(list, i); assert(jl_is_method(meth)); @@ -819,7 +819,7 @@ static void jl_insert_methods(jl_array_t *list) static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) { - size_t i, l = jl_array_len(method_roots_list); + size_t i, l = jl_array_nrows(method_roots_list); for (i = 0; i < l; i+=2) { jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(method_roots_list, i); jl_array_t *roots = (jl_array_t*)jl_array_ptr_ref(method_roots_list, i+1); @@ -835,12 +835,12 @@ static void jl_copy_roots(jl_array_t *method_roots_list, uint64_t key) static jl_array_t *jl_verify_edges(jl_array_t *targets, size_t minworld) { JL_TIMING(VERIFY_IMAGE, VERIFY_Edges); - size_t i, l = jl_array_len(targets) / 3; + size_t i, l = jl_array_nrows(targets) / 3; static jl_value_t *ulong_array JL_ALWAYS_LEAFTYPE = NULL; if (ulong_array == NULL) ulong_array = jl_apply_array_type((jl_value_t*)jl_ulong_type, 1); jl_array_t *maxvalids = jl_alloc_array_1d(ulong_array, l); - memset(jl_array_data(maxvalids), 0, l * sizeof(size_t)); + memset(jl_array_data(maxvalids, size_t), 0, l * sizeof(size_t)); jl_value_t *loctag = NULL; jl_value_t *matches = NULL; jl_value_t *sig = NULL; @@ -891,7 +891,7 @@ static jl_array_t *jl_verify_edges(jl_array_t *targets, size_t minworld) // TODO: possibly need to included ambiguities too (for the optimizer correctness)? // len + 1 is to allow us to log causes of invalidation (SnoopCompile's @snoopr) matches = jl_matching_methods((jl_tupletype_t*)sig, jl_nothing, - _jl_debug_method_invalidation ? INT32_MAX : jl_array_len(expected), + _jl_debug_method_invalidation ? INT32_MAX : jl_array_nrows(expected), 0, minworld, &min_valid, &max_valid, &ambig); sig = NULL; if (matches == jl_nothing) { @@ -900,12 +900,12 @@ static jl_array_t *jl_verify_edges(jl_array_t *targets, size_t minworld) else { // setdiff!(matches, expected) size_t j, k, ins = 0; - if (jl_array_len(matches) != jl_array_len(expected)) { + if (jl_array_nrows(matches) != jl_array_nrows(expected)) { max_valid = 0; } - for (k = 0; k < jl_array_len(matches); k++) { + for (k = 0; k < jl_array_nrows(matches); k++) { jl_method_t *match = ((jl_method_match_t*)jl_array_ptr_ref(matches, k))->method; - size_t l = jl_array_len(expected); + size_t l = jl_array_nrows(expected); for (j = 0; j < l; j++) if (match == (jl_method_t*)jl_array_ptr_ref(expected, j)) break; @@ -920,10 +920,10 @@ static jl_array_t *jl_verify_edges(jl_array_t *targets, size_t minworld) } } if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) - jl_array_del_end((jl_array_t*)matches, jl_array_len(matches) - ins); + jl_array_del_end((jl_array_t*)matches, jl_array_nrows(matches) - ins); } } - ((size_t*)(jl_array_data(maxvalids)))[i] = max_valid; + jl_array_data(maxvalids, size_t)[i] = max_valid; if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, invokesig ? (jl_value_t*)invokesig : callee); loctag = jl_cstr_to_string("insert_backedges_callee"); @@ -947,9 +947,9 @@ static jl_array_t *jl_verify_methods(jl_array_t *edges, jl_array_t *maxvalids) jl_value_t *loctag = NULL; jl_array_t *maxvalids2 = NULL; JL_GC_PUSH2(&loctag, &maxvalids2); - size_t i, l = jl_array_len(edges) / 2; + size_t i, l = jl_array_nrows(edges) / 2; maxvalids2 = jl_alloc_array_1d(jl_typeof(maxvalids), l); - size_t *maxvalids2_data = (size_t*)jl_array_data(maxvalids2); + size_t *maxvalids2_data = jl_array_data(maxvalids2, size_t); memset(maxvalids2_data, 0, l * sizeof(size_t)); for (i = 0; i < l; i++) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); @@ -961,12 +961,12 @@ static jl_array_t *jl_verify_methods(jl_array_t *edges, jl_array_t *maxvalids) maxvalids2_data[i] = 0; } else { - int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + int32_t *idxs = jl_array_data(callee_ids, int32_t); size_t j; maxvalids2_data[i] = ~(size_t)0; for (j = 0; j < idxs[0]; j++) { int32_t idx = idxs[j + 1]; - size_t max_valid = ((size_t*)(jl_array_data(maxvalids)))[idx]; + size_t max_valid = jl_array_data(maxvalids, size_t)[idx]; if (max_valid != ~(size_t)0 && _jl_debug_method_invalidation) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)caller); loctag = jl_cstr_to_string("verify_methods"); @@ -1006,8 +1006,8 @@ static int jl_verify_graph_edge(size_t *maxvalids2_data, jl_array_t *edges, size visited->items[idx] = (void*)(1 + depth); jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, idx * 2 + 1); assert(jl_typetagis((jl_value_t*)callee_ids, jl_array_int32_type)); - int32_t *idxs = (int32_t*)jl_array_data(callee_ids); - size_t i, n = jl_array_len(callee_ids); + int32_t *idxs = jl_array_data(callee_ids, int32_t); + size_t i, n = jl_array_nrows(callee_ids); cycle = depth; for (i = idxs[0] + 1; i < n; i++) { int32_t childidx = idxs[i]; @@ -1061,10 +1061,10 @@ static void jl_verify_graph(jl_array_t *edges, jl_array_t *maxvalids2) JL_TIMING(VERIFY_IMAGE, VERIFY_Graph); arraylist_t stack, visited; arraylist_new(&stack, 0); - size_t i, n = jl_array_len(edges) / 2; + size_t i, n = jl_array_nrows(edges) / 2; arraylist_new(&visited, n); memset(visited.items, 0, n * sizeof(size_t)); - size_t *maxvalids2_data = (size_t*)jl_array_data(maxvalids2); + size_t *maxvalids2_data = jl_array_data(maxvalids2, size_t); for (i = 0; i < n; i++) { assert(visited.items[i] == (void*)0 || visited.items[i] == (void*)1); int child_cycle = jl_verify_graph_edge(maxvalids2_data, edges, i, &visited, &stack); @@ -1089,7 +1089,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a size_t i, l; // next build a map from external MethodInstances to their CodeInstance for insertion - l = jl_array_len(ci_list); + l = jl_array_nrows(ci_list); htable_t visited; htable_new(&visited, l); for (i = 0; i < l; i++) { @@ -1110,14 +1110,14 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a } // next enable any applicable new codes - l = jl_array_len(edges) / 2; + l = jl_array_nrows(edges) / 2; for (i = 0; i < l; i++) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); - size_t maxvalid = ((size_t*)(jl_array_data(valids)))[i]; + size_t maxvalid = jl_array_data(valids, size_t)[i]; if (maxvalid == ~(size_t)0) { // if this callee is still valid, add all the backedges jl_array_t *callee_ids = (jl_array_t*)jl_array_ptr_ref(edges, 2 * i + 1); - int32_t *idxs = (int32_t*)jl_array_data(callee_ids); + int32_t *idxs = jl_array_data(callee_ids, int32_t); for (size_t j = 0; j < idxs[0]; j++) { int32_t idx = idxs[j + 1]; jl_value_t *invokesig = jl_array_ptr_ref(ext_targets, idx * 3); @@ -1158,7 +1158,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_targets, jl_a static void classify_callers(htable_t *callers_with_edges, jl_array_t *edges) { - size_t l = edges ? jl_array_len(edges) / 2 : 0; + size_t l = edges ? jl_array_nrows(edges) / 2 : 0; for (size_t i = 0; i < l; i++) { jl_method_instance_t *caller = (jl_method_instance_t*)jl_array_ptr_ref(edges, 2 * i); ptrhash_put(callers_with_edges, (void*)caller, (void*)caller); @@ -1171,7 +1171,7 @@ static jl_value_t *read_verify_mod_list(ios_t *s, jl_array_t *depmods) return jl_get_exceptionf(jl_errorexception_type, "Main module uuid state is invalid for module deserialization."); } - size_t i, l = jl_array_len(depmods); + size_t i, l = jl_array_nrows(depmods); for (i = 0; ; i++) { size_t len = read_int32(s); if (len == 0 && i == l) @@ -1233,11 +1233,11 @@ static jl_array_t *image_to_depmodidx(jl_array_t *depmods) { if (!depmods) return NULL; - assert(jl_array_len(depmods) < INT32_MAX && "too many dependencies to serialize"); + assert(jl_array_nrows(depmods) < INT32_MAX && "too many dependencies to serialize"); size_t lbids = n_linkage_blobs(); - size_t ldeps = jl_array_len(depmods); + size_t ldeps = jl_array_nrows(depmods); jl_array_t *depmodidxs = jl_alloc_array_1d(jl_array_int32_type, lbids); - int32_t *dmidxs = (int32_t*)jl_array_data(depmodidxs); + int32_t *dmidxs = jl_array_data(depmodidxs, int32_t); memset(dmidxs, -1, lbids * sizeof(int32_t)); dmidxs[0] = 0; // the sysimg can also be found at idx 0, by construction for (size_t i = 0, j = 0; i < ldeps; i++) { @@ -1257,9 +1257,9 @@ static jl_array_t *depmod_to_imageidx(jl_array_t *depmods) { if (!depmods) return NULL; - size_t ldeps = jl_array_len(depmods); + size_t ldeps = jl_array_nrows(depmods); jl_array_t *imageidxs = jl_alloc_array_1d(jl_array_int32_type, ldeps + 1); - int32_t *imgidxs = (int32_t*)jl_array_data(imageidxs); + int32_t *imgidxs = jl_array_data(imageidxs, int32_t); imgidxs[0] = 0; for (size_t i = 0; i < ldeps; i++) { jl_value_t *depmod = jl_array_ptr_ref(depmods, i); diff --git a/src/subtype.c b/src/subtype.c index c67beecae9dbd..83a35eb03cb7a 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -846,7 +846,7 @@ static jl_value_t *fix_inferred_var_bound(jl_tvar_t *var, jl_value_t *ty JL_MAYB JL_GC_PUSH2(&ans, &vs); vs = jl_find_free_typevars(ty); int i; - for (i = 0; i < jl_array_len(vs); i++) { + for (i = 0; i < jl_array_nrows(vs); i++) { ans = jl_type_unionall((jl_tvar_t*)jl_array_ptr_ref(vs, i), ans); } ans = (jl_value_t*)jl_new_typevar(var->name, jl_bottom_type, ans); @@ -2989,7 +2989,7 @@ static jl_value_t *finish_unionall(jl_value_t *res JL_MAYBE_UNROOTED, jl_varbind } if (vb->innervars != NULL) { - for (size_t i = 0; i < jl_array_len(vb->innervars); i++) { + for (size_t i = 0; i < jl_array_nrows(vb->innervars); i++) { jl_tvar_t *var = (jl_tvar_t*)jl_array_ptr_ref(vb->innervars, i); // the `btemp->prev` walk is only giving a sort of post-order guarantee (since we are // iterating 2 trees at once), so once we set `wrap`, there might remain other branches diff --git a/src/sys.c b/src/sys.c index e8d24ae051cc6..22994cc69372c 100644 --- a/src/sys.c +++ b/src/sys.c @@ -280,14 +280,15 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint return str; } a = jl_alloc_array_1d(jl_array_uint8_type, n - nchomp); - memcpy(jl_array_data(a), s->buf + s->bpos, n - nchomp); + memcpy(jl_array_data(a, uint8_t), s->buf + s->bpos, n - nchomp); s->bpos += n; } else { a = jl_alloc_array_1d(jl_array_uint8_type, 80); ios_t dest; ios_mem(&dest, 0); - ios_setbuf(&dest, (char*)a->data, 80, 0); + char *mem = jl_array_data(a, char); + ios_setbuf(&dest, (char*)mem, 80, 0); size_t n = ios_copyuntil(&dest, s, delim, 1); if (chomp && n > 0 && dest.buf[n - 1] == delim) { n--; @@ -298,13 +299,12 @@ JL_DLLEXPORT jl_value_t *jl_readuntil(ios_t *s, uint8_t delim, uint8_t str, uint assert(truncret == 0); (void)truncret; // ensure the variable is used to avoid warnings } - if (dest.buf != a->data) { + if (dest.buf != mem) { a = jl_take_buffer(&dest); } else { - a->length = n; - a->nrows = n; - ((char*)a->data)[n] = '\0'; + a->dimsize[0] = n; + ((char*)mem)[n] = '\0'; } if (str) { JL_GC_PUSH1(&a); diff --git a/src/toplevel.c b/src/toplevel.c index ca2033e58727d..ba90fe92574f1 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -121,7 +121,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex { jl_task_t *ct = jl_current_task; assert(ex->head == jl_module_sym); - if (jl_array_len(ex->args) != 3 || !jl_is_expr(jl_exprarg(ex, 2))) { + if (jl_array_nrows(ex->args) != 3 || !jl_is_expr(jl_exprarg(ex, 2))) { jl_error("syntax: malformed module expression"); } @@ -188,7 +188,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex jl_array_t *exprs = ((jl_expr_t*)jl_exprarg(ex, 2))->args; int lineno = 0; const char *filename = "none"; - if (jl_array_len(exprs) > 0) { + if (jl_array_nrows(exprs) > 0) { jl_value_t *lineex = jl_array_ptr_ref(exprs, 0); if (jl_is_linenode(lineex)) { lineno = jl_linenode_line(lineex); @@ -207,7 +207,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex form = NULL; } - for (int i = 0; i < jl_array_len(exprs); i++) { + for (int i = 0; i < jl_array_nrows(exprs); i++) { // process toplevel form ct->world_age = jl_atomic_load_acquire(&jl_world_counter); form = jl_expand_stmt_with_loc(jl_array_ptr_ref(exprs, i), newm, jl_filename, jl_lineno); @@ -254,7 +254,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex form = NULL; if (!jl_generating_output()) { if (!ptrhash_has(&jl_current_modules, (void*)newm->parent)) { - size_t i, l = jl_array_len(jl_module_init_order); + size_t i, l = jl_array_nrows(jl_module_init_order); size_t ns = 0; form = (jl_value_t*)jl_alloc_vec_any(0); for (i = 0; i < l; i++) { @@ -273,7 +273,7 @@ static jl_value_t *jl_eval_module_expr(jl_module_t *parent_module, jl_expr_t *ex JL_UNLOCK(&jl_modules_mutex); if (form) { - size_t i, l = jl_array_len(form); + size_t i, l = jl_array_nrows(form); for (i = 0; i < l; i++) { jl_module_t *m = (jl_module_t*)jl_array_ptr_ref(form, i); JL_GC_PROMISE_ROOTED(m); @@ -311,7 +311,7 @@ static jl_value_t *jl_eval_dot_expr(jl_module_t *m, jl_value_t *x, jl_value_t *f void jl_eval_global_expr(jl_module_t *m, jl_expr_t *ex, int set_type) { // create uninitialized mutable binding for "global x" decl sometimes or probably - size_t i, l = jl_array_len(ex->args); + size_t i, l = jl_array_nrows(ex->args); for (i = 0; i < l; i++) { jl_value_t *arg = jl_exprarg(ex, i); jl_module_t *gm; @@ -414,7 +414,7 @@ static void expr_attributes(jl_value_t *v, int *has_ccall, int *has_defs, int *h return; } int i; - for (i = 0; i < jl_array_len(e->args); i++) { + for (i = 0; i < jl_array_nrows(e->args); i++) { jl_value_t *a = jl_exprarg(e, i); if (jl_is_expr(a)) expr_attributes(a, has_ccall, has_defs, has_opaque); @@ -429,7 +429,7 @@ int jl_code_requires_compiler(jl_code_info_t *src, int include_force_compile) int has_ccall = 0, has_defs = 0, has_opaque = 0; if (include_force_compile && jl_has_meta(body, jl_force_compile_sym)) return 1; - for(i=0; i < jl_array_len(body); i++) { + for(i=0; i < jl_array_nrows(body); i++) { jl_value_t *stmt = jl_array_ptr_ref(body,i); expr_attributes(stmt, &has_ccall, &has_defs, &has_opaque); if (has_ccall) @@ -442,7 +442,7 @@ static void body_attributes(jl_array_t *body, int *has_ccall, int *has_defs, int { size_t i; *has_loops = 0; - for(i=0; i < jl_array_len(body); i++) { + for(i=0; i < jl_array_nrows(body); i++) { jl_value_t *stmt = jl_array_ptr_ref(body,i); if (!*has_loops) { if (jl_is_gotonode(stmt)) { @@ -493,7 +493,7 @@ static jl_module_t *call_require(jl_module_t *mod, jl_sym_t *var) JL_GLOBALLY_RO static jl_module_t *eval_import_path(jl_module_t *where, jl_module_t *from JL_PROPAGATES_ROOT, jl_array_t *args, jl_sym_t **name, const char *keyword) JL_GLOBALLY_ROOTED { - if (jl_array_len(args) == 0) + if (jl_array_nrows(args) == 0) jl_errorf("malformed \"%s\" statement", keyword); jl_sym_t *var = (jl_sym_t*)jl_array_ptr_ref(args, 0); size_t i = 1; @@ -517,14 +517,14 @@ static jl_module_t *eval_import_path(jl_module_t *where, jl_module_t *from JL_PR else { m = call_require(where, var); } - if (i == jl_array_len(args)) + if (i == jl_array_nrows(args)) return m; } else { // `.A.B.C`: strip off leading dots by following parent links m = where; while (1) { - if (i >= jl_array_len(args)) + if (i >= jl_array_nrows(args)) jl_error("invalid module path"); var = (jl_sym_t*)jl_array_ptr_ref(args, i); if (var != jl_dot_sym) @@ -541,7 +541,7 @@ static jl_module_t *eval_import_path(jl_module_t *where, jl_module_t *from JL_PR jl_type_error(keyword, (jl_value_t*)jl_symbol_type, (jl_value_t*)var); if (var == jl_dot_sym) jl_errorf("invalid %s path: \".\" in identifier path", keyword); - if (i == jl_array_len(args)-1) + if (i == jl_array_nrows(args)-1) break; m = (jl_module_t*)jl_eval_global_var(m, var); JL_GC_PROMISE_ROOTED(m); @@ -582,7 +582,7 @@ int jl_needs_lowering(jl_value_t *e) JL_NOTSAFEPOINT return 0; } if (head == jl_global_sym || head == jl_const_sym) { - size_t i, l = jl_array_len(ex->args); + size_t i, l = jl_array_nrows(ex->args); for (i = 0; i < l; i++) { jl_value_t *a = jl_exprarg(ex, i); if (!jl_is_symbol(a) && !jl_is_globalref(a)) @@ -841,7 +841,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int } else if (head == jl_export_sym || head == jl_public_sym) { int exp = (head == jl_export_sym); - for (size_t i = 0; i < jl_array_len(ex->args); i++) { + for (size_t i = 0; i < jl_array_nrows(ex->args); i++) { jl_sym_t *name = (jl_sym_t*)jl_array_ptr_ref(ex->args, i); if (!jl_is_symbol(name)) jl_eval_errorf(m, exp ? "syntax: malformed \"export\" statement" : @@ -877,7 +877,7 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *JL_NONNULL m, jl_value_t *e, int else if (head == jl_toplevel_sym) { jl_value_t *res = jl_nothing; int i; - for (i = 0; i < jl_array_len(ex->args); i++) { + for (i = 0; i < jl_array_nrows(ex->args); i++) { res = jl_toplevel_eval_flex(m, jl_array_ptr_ref(ex->args, i), fast, 0); } JL_GC_POP(); @@ -957,7 +957,7 @@ JL_DLLEXPORT void jl_check_top_level_effect(jl_module_t *m, char *fname) JL_LOCK(&jl_modules_mutex); int open = ptrhash_has(&jl_current_modules, (void*)m); if (!open && jl_module_init_order != NULL) { - size_t i, l = jl_array_len(jl_module_init_order); + size_t i, l = jl_array_nrows(jl_module_init_order); for (i = 0; i < l; i++) { if (m == (jl_module_t*)jl_array_ptr_ref(jl_module_init_order, i)) { open = 1; diff --git a/src/typemap.c b/src/typemap.c index 1bdbe52a974dd..06273b83c25ad 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -277,21 +277,21 @@ static int is_cache_leaf(jl_value_t *ty, int tparam) return (jl_is_concrete_type(ty) && (tparam || !jl_is_kind(ty))); } -static _Atomic(jl_value_t*) *mtcache_hash_lookup_bp(jl_array_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT +static _Atomic(jl_value_t*) *mtcache_hash_lookup_bp(jl_genericmemory_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT { - if (cache == (jl_array_t*)jl_an_empty_vec_any) + if (cache == (jl_genericmemory_t*)jl_an_empty_memory_any) return NULL; _Atomic(jl_value_t*) *pml = jl_table_peek_bp(cache, ty); JL_GC_PROMISE_ROOTED(pml); // clang-sa doesn't trust our JL_PROPAGATES_ROOT claim return pml; } -static void mtcache_hash_insert(_Atomic(jl_array_t*) *cache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) +static void mtcache_hash_insert(_Atomic(jl_genericmemory_t*) *cache, jl_value_t *parent, jl_value_t *key, jl_typemap_t *val) { int inserted = 0; - jl_array_t *a = jl_atomic_load_relaxed(cache); - if (a == (jl_array_t*)jl_an_empty_vec_any) { - a = jl_alloc_vec_any(16); + jl_genericmemory_t *a = jl_atomic_load_relaxed(cache); + if (a == (jl_genericmemory_t*)jl_an_empty_memory_any) { + a = jl_alloc_memory_any(16); jl_atomic_store_release(cache, a); if (parent) jl_gc_wb(parent, a); @@ -305,9 +305,9 @@ static void mtcache_hash_insert(_Atomic(jl_array_t*) *cache, jl_value_t *parent, } } -static jl_typemap_t *mtcache_hash_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT +static jl_typemap_t *mtcache_hash_lookup(jl_genericmemory_t *cache JL_PROPAGATES_ROOT, jl_value_t *ty) JL_NOTSAFEPOINT { - if (cache == (jl_array_t*)jl_an_empty_vec_any) + if (cache == (jl_genericmemory_t*)jl_an_empty_memory_any) return (jl_typemap_t*)jl_nothing; jl_typemap_t *ml = (jl_typemap_t*)jl_eqtable_get(cache, ty, jl_nothing); return ml; @@ -315,17 +315,17 @@ static jl_typemap_t *mtcache_hash_lookup(jl_array_t *cache JL_PROPAGATES_ROOT, j // ----- Sorted Type Signature Lookup Matching ----- // -static int jl_typemap_array_visitor(jl_array_t *a, jl_typemap_visitor_fptr fptr, void *closure) +static int jl_typemap_memory_visitor(jl_genericmemory_t *a, jl_typemap_visitor_fptr fptr, void *closure) { - size_t i, l = jl_array_len(a); - _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_data(a); + size_t i, l = a->length; + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*) a->data; for (i = 1; i < l; i += 2) { jl_value_t *d = jl_atomic_load_relaxed(&data[i]); JL_GC_PROMISE_ROOTED(d); if (d == NULL) continue; - if (jl_is_array(d)) { - if (!jl_typemap_array_visitor((jl_array_t*)d, fptr, closure)) + if (jl_is_genericmemory(d)) { + if (!jl_typemap_memory_visitor((jl_genericmemory_t*)d, fptr, closure)) return 0; } else { @@ -352,23 +352,23 @@ int jl_typemap_visitor(jl_typemap_t *cache, jl_typemap_visitor_fptr fptr, void * { if (jl_typeof(cache) == (jl_value_t*)jl_typemap_level_type) { jl_typemap_level_t *node = (jl_typemap_level_t*)cache; - jl_array_t *a; + jl_genericmemory_t *a; JL_GC_PUSH1(&a); a = jl_atomic_load_relaxed(&node->targ); - if (a != (jl_array_t*)jl_an_empty_vec_any) - if (!jl_typemap_array_visitor(a, fptr, closure)) + if (a != (jl_genericmemory_t*)jl_an_empty_memory_any) + if (!jl_typemap_memory_visitor(a, fptr, closure)) goto exit; a = jl_atomic_load_relaxed(&node->arg1); - if (a != (jl_array_t*)jl_an_empty_vec_any) - if (!jl_typemap_array_visitor(a, fptr, closure)) + if (a != (jl_genericmemory_t*)jl_an_empty_memory_any) + if (!jl_typemap_memory_visitor(a, fptr, closure)) goto exit; a = jl_atomic_load_relaxed(&node->tname); - if (a != (jl_array_t*)jl_an_empty_vec_any) - if (!jl_typemap_array_visitor(a, fptr, closure)) + if (a != (jl_genericmemory_t*)jl_an_empty_memory_any) + if (!jl_typemap_memory_visitor(a, fptr, closure)) goto exit; a = jl_atomic_load_relaxed(&node->name1); - if (a != (jl_array_t*)jl_an_empty_vec_any) - if (!jl_typemap_array_visitor(a, fptr, closure)) + if (a != (jl_genericmemory_t*)jl_an_empty_memory_any) + if (!jl_typemap_memory_visitor(a, fptr, closure)) goto exit; if (!jl_typemap_node_visitor(jl_atomic_load_relaxed(&node->linear), fptr, closure)) goto exit; @@ -451,12 +451,12 @@ static int concrete_intersects(jl_value_t *t, jl_value_t *ty, int8_t tparam) // tparam bit 0 is ::Type{T} (vs. T) // tparam bit 1 is typename(T) (vs. T) -static int jl_typemap_intersection_array_visitor(jl_array_t *a, jl_value_t *ty, int8_t tparam, +static int jl_typemap_intersection_memory_visitor(jl_genericmemory_t *a, jl_value_t *ty, int8_t tparam, int8_t offs, struct typemap_intersection_env *closure) { JL_GC_PUSH1(&a); - size_t i, l = jl_array_len(a); - _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_data(a); + size_t i, l = a->length; + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*) a->data; unsigned height = 0; jl_datatype_t *tydt = jl_any_type; if (tparam & 2) { @@ -492,8 +492,8 @@ static int jl_typemap_intersection_array_visitor(jl_array_t *a, jl_value_t *ty, tname_intersection_dt(tydt, (jl_typename_t*)t, height)) { if ((tparam & 1) && t == (jl_value_t*)jl_typeofbottom_type->name) // skip Type{Union{}} and Type{typeof(Union{})}, since the caller should have already handled those continue; - if (jl_is_array(ml)) { - if (!jl_typemap_intersection_array_visitor((jl_array_t*)ml, ty, tparam & ~2, offs, closure)) + if (jl_is_genericmemory(ml)) { + if (!jl_typemap_intersection_memory_visitor((jl_genericmemory_t*)ml, ty, tparam & ~2, offs, closure)) goto exit; } else { @@ -627,15 +627,15 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, if (jl_has_free_typevars(ty)) ty = jl_rewrap_unionall(ty, closure->type); JL_GC_PUSH1(&ty); - jl_array_t *targ = jl_atomic_load_relaxed(&cache->targ); - jl_array_t *tname = jl_atomic_load_relaxed(&cache->tname); + jl_genericmemory_t *targ = jl_atomic_load_relaxed(&cache->targ); + jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); int maybe_type = 0; int maybe_kind = 0; int exclude_typeofbottom = 0; jl_value_t *typetype = NULL; jl_value_t *name = NULL; // pre-check: optimized pre-intersection test to see if `ty` could intersect with any Type or Kind - if (targ != (jl_array_t*)jl_an_empty_vec_any || tname != (jl_array_t*)jl_an_empty_vec_any) { + if (targ != (jl_genericmemory_t*)jl_an_empty_memory_any || tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { maybe_kind = jl_has_intersect_kind_not_type(ty); maybe_type = maybe_kind || jl_has_intersect_type_not_kind(ty); if (maybe_type && !maybe_kind) { @@ -651,7 +651,7 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } // First check for intersections with methods defined on Type{T}, where T was a concrete type - if (targ != (jl_array_t*)jl_an_empty_vec_any && maybe_type && + if (targ != (jl_genericmemory_t*)jl_an_empty_memory_any && maybe_type && (!typetype || jl_has_free_typevars(typetype) || is_cache_leaf(typetype, 1))) { // otherwise cannot contain this particular kind, so don't bother with checking if (!exclude_typeofbottom) { // detect Type{Union{}}, Type{Type{Union{}}}, and Type{typeof(Union{}} and do those early here @@ -680,18 +680,18 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, // attempt semi-direct lookup of types via their names // consider the type name first jl_value_t *ml = mtcache_hash_lookup(targ, (jl_value_t*)name); - if (jl_is_array(ml)) { + if (jl_is_genericmemory(ml)) { if (typetype && !jl_has_free_typevars(typetype)) { // direct lookup of leaf types if (is_cache_leaf(typetype, 1)) { - ml = mtcache_hash_lookup((jl_array_t*)ml, typetype); + ml = mtcache_hash_lookup((jl_genericmemory_t*)ml, typetype); if (ml != jl_nothing) { if (!jl_typemap_intersection_visitor((jl_typemap_t*)ml, offs+1, closure)) { JL_GC_POP(); return 0; } } } } else { - if (!jl_typemap_intersection_array_visitor((jl_array_t*)ml, ty, 1, offs, closure)) { JL_GC_POP(); return 0; } + if (!jl_typemap_intersection_memory_visitor((jl_genericmemory_t*)ml, ty, 1, offs, closure)) { JL_GC_POP(); return 0; } } } else if (ml != jl_nothing) { @@ -699,19 +699,19 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } else { - // else an array scan is required to consider all the possible subtypes - if (!jl_typemap_intersection_array_visitor(targ, exclude_typeofbottom && !maybe_kind ? ty : (jl_value_t*)jl_any_type, 3, offs, closure)) { JL_GC_POP(); return 0; } + // else a scan is required to consider all the possible subtypes + if (!jl_typemap_intersection_memory_visitor(targ, exclude_typeofbottom && !maybe_kind ? ty : (jl_value_t*)jl_any_type, 3, offs, closure)) { JL_GC_POP(); return 0; } } } } - jl_array_t *cachearg1 = jl_atomic_load_relaxed(&cache->arg1); - if (cachearg1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *cachearg1 = jl_atomic_load_relaxed(&cache->arg1); + if (cachearg1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { if (is_cache_leaf(ty, 0)) { jl_typename_t *name = ty == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)ty)->name; // direct lookup of leaf types jl_value_t *ml = mtcache_hash_lookup(cachearg1, (jl_value_t*)name); - if (jl_is_array(ml)) - ml = mtcache_hash_lookup((jl_array_t*)ml, ty); + if (jl_is_genericmemory(ml)) + ml = mtcache_hash_lookup((jl_genericmemory_t*)ml, ty); if (ml != jl_nothing) { if (!jl_typemap_intersection_visitor(ml, offs+1, closure)) { JL_GC_POP(); return 0; } } @@ -721,21 +721,21 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, if (name && jl_type_extract_name_precise(ty, 0)) { // direct lookup of leaf types jl_value_t *ml = mtcache_hash_lookup(cachearg1, name); - if (jl_is_array(ml)) { - if (!jl_typemap_intersection_array_visitor((jl_array_t*)ml, ty, 0, offs, closure)) { JL_GC_POP(); return 0; } + if (jl_is_genericmemory(ml)) { + if (!jl_typemap_intersection_memory_visitor((jl_genericmemory_t*)ml, ty, 0, offs, closure)) { JL_GC_POP(); return 0; } } else { if (!jl_typemap_intersection_visitor((jl_typemap_t*)ml, offs+1, closure)) { JL_GC_POP(); return 0; } } } else { - // else an array scan is required to check subtypes - if (!jl_typemap_intersection_array_visitor(cachearg1, ty, 2, offs, closure)) { JL_GC_POP(); return 0; } + // else a scan is required to check subtypes + if (!jl_typemap_intersection_memory_visitor(cachearg1, ty, 2, offs, closure)) { JL_GC_POP(); return 0; } } } } // Next check for intersections with methods defined on Type{T}, where T was not concrete (it might even have been a TypeVar), but had an extractable TypeName - if (tname != (jl_array_t*)jl_an_empty_vec_any && maybe_type) { + if (tname != (jl_genericmemory_t*)jl_an_empty_memory_any && maybe_type) { if (!exclude_typeofbottom || (!typetype && jl_isa((jl_value_t*)jl_typeofbottom_type, ty))) { // detect Type{Union{}}, Type{Type{Union{}}}, and Type{typeof(Union{}} and do those early here // otherwise the possibility of encountering `Type{Union{}}` in this intersection may @@ -775,13 +775,13 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } else { - // else an array scan is required to check subtypes of typetype too + // else a scan is required to check subtypes of typetype too tname = jl_atomic_load_relaxed(&cache->tname); // may be GC'd earlier - if (!jl_typemap_intersection_array_visitor(tname, exclude_typeofbottom && !maybe_kind ? ty : (jl_value_t*)jl_any_type, 3, offs, closure)) { JL_GC_POP(); return 0; } + if (!jl_typemap_intersection_memory_visitor(tname, exclude_typeofbottom && !maybe_kind ? ty : (jl_value_t*)jl_any_type, 3, offs, closure)) { JL_GC_POP(); return 0; } } } - jl_array_t *name1 = jl_atomic_load_relaxed(&cache->name1); - if (name1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); + if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { jl_value_t *name = jl_type_extract_name(ty); if (name && jl_type_extract_name_precise(ty, 0)) { jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)name)->wrapper); @@ -798,8 +798,8 @@ int jl_typemap_intersection_visitor(jl_typemap_t *map, int offs, } } else { - // else an array scan is required to check subtypes - if (!jl_typemap_intersection_array_visitor(name1, ty, 2, offs, closure)) { JL_GC_POP(); return 0; } + // else a scan is required to check subtypes + if (!jl_typemap_intersection_memory_visitor(name1, ty, 2, offs, closure)) { JL_GC_POP(); return 0; } } } JL_GC_POP(); @@ -989,12 +989,12 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( if (jl_is_type_type(ty)) { jl_value_t *a0 = jl_tparam0(ty); if (is_cache_leaf(a0, 1)) { - jl_array_t *targ = jl_atomic_load_relaxed(&cache->targ); - if (targ != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *targ = jl_atomic_load_relaxed(&cache->targ); + if (targ != (jl_genericmemory_t*)jl_an_empty_memory_any) { jl_typename_t *name = a0 == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)a0)->name; jl_value_t *ml = mtcache_hash_lookup(targ, (jl_value_t*)name); - if (jl_is_array(ml)) - ml = mtcache_hash_lookup((jl_array_t*)ml, a0); + if (jl_is_genericmemory(ml)) + ml = mtcache_hash_lookup((jl_genericmemory_t*)ml, a0); if (ml != jl_nothing) { jl_typemap_entry_t *li = jl_typemap_assoc_by_type((jl_typemap_t*)ml, search, offs + 1, subtype); if (li) return li; @@ -1004,12 +1004,12 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( } } if (is_cache_leaf(ty, 0)) { - jl_array_t *cachearg1 = jl_atomic_load_relaxed(&cache->arg1); - if (cachearg1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *cachearg1 = jl_atomic_load_relaxed(&cache->arg1); + if (cachearg1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { jl_typename_t *name = ty == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)ty)->name; jl_value_t *ml = mtcache_hash_lookup(cachearg1, (jl_value_t*)name); - if (jl_is_array(ml)) - ml = mtcache_hash_lookup((jl_array_t*)ml, ty); + if (jl_is_genericmemory(ml)) + ml = mtcache_hash_lookup((jl_genericmemory_t*)ml, ty); if (ml != jl_nothing) { jl_typemap_entry_t *li = jl_typemap_assoc_by_type((jl_typemap_t*)ml, search, offs + 1, subtype); if (li) return li; @@ -1020,8 +1020,8 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( } if (ty || subtype) { // now look at the optimized TypeName caches - jl_array_t *tname = jl_atomic_load_relaxed(&cache->tname); - if (tname != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); + if (tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { jl_value_t *a0 = ty && jl_is_type_type(ty) ? jl_type_extract_name(jl_tparam0(ty)) : NULL; if (a0) { // TODO: if we start analyzing Union types in jl_type_extract_name, then a0 might be over-approximated here, leading us to miss possible subtypes jl_datatype_t *super = (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper); @@ -1039,9 +1039,10 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( } else { if (!ty || !jl_has_empty_intersection((jl_value_t*)jl_type_type, ty)) { + jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); // reload after type-intersect // couldn't figure out unique `a0` initial point, so scan all for matches - size_t i, l = jl_array_len(tname); - _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_ptr_data(tname); + size_t i, l = tname->length; + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*) tname->data; JL_GC_PUSH1(&tname); for (i = 1; i < l; i += 2) { jl_typemap_t *ml = jl_atomic_load_relaxed(&data[i]); @@ -1057,8 +1058,8 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( } } } - jl_array_t *name1 = jl_atomic_load_relaxed(&cache->name1); - if (name1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); + if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { if (ty) { jl_value_t *a0 = jl_type_extract_name(ty); if (a0) { // TODO: if we start analyzing Union types in jl_type_extract_name, then a0 might be over-approximated here, leading us to miss possible subtypes @@ -1079,8 +1080,8 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type( } else { // doing subtype, but couldn't figure out unique `ty`, so scan all for supertypes - size_t i, l = jl_array_len(name1); - _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_ptr_data(name1); + size_t i, l = name1->length; + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*) name1->data; JL_GC_PUSH1(&name1); for (i = 1; i < l; i += 2) { jl_typemap_t *ml = jl_atomic_load_relaxed(&data[i]); @@ -1198,26 +1199,26 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v jl_value_t *a1 = (offs == 0 ? arg1 : args[offs - 1]); jl_value_t *ty = jl_typeof(a1); assert(jl_is_datatype(ty)); - jl_array_t *targ = jl_atomic_load_relaxed(&cache->targ); - if (targ != (jl_array_t*)jl_an_empty_vec_any && is_cache_leaf(a1, 1)) { + jl_genericmemory_t *targ = jl_atomic_load_relaxed(&cache->targ); + if (targ != (jl_genericmemory_t*)jl_an_empty_memory_any && is_cache_leaf(a1, 1)) { jl_typename_t *name = a1 == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)a1)->name; jl_value_t *ml_or_cache = mtcache_hash_lookup(targ, (jl_value_t*)name); - if (jl_is_array(ml_or_cache)) - ml_or_cache = mtcache_hash_lookup((jl_array_t*)ml_or_cache, a1); + if (jl_is_genericmemory(ml_or_cache)) + ml_or_cache = mtcache_hash_lookup((jl_genericmemory_t*)ml_or_cache, a1); jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, arg1, args, n, offs+1, world); if (ml) return ml; } - jl_array_t *cachearg1 = jl_atomic_load_relaxed(&cache->arg1); - if (cachearg1 != (jl_array_t*)jl_an_empty_vec_any && is_cache_leaf(ty, 0)) { + jl_genericmemory_t *cachearg1 = jl_atomic_load_relaxed(&cache->arg1); + if (cachearg1 != (jl_genericmemory_t*)jl_an_empty_memory_any && is_cache_leaf(ty, 0)) { jl_typename_t *name = ty == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)ty)->name; jl_value_t *ml_or_cache = mtcache_hash_lookup(cachearg1, (jl_value_t*)name); - if (jl_is_array(ml_or_cache)) - ml_or_cache = mtcache_hash_lookup((jl_array_t*)ml_or_cache, ty); + if (jl_is_genericmemory(ml_or_cache)) + ml_or_cache = mtcache_hash_lookup((jl_genericmemory_t*)ml_or_cache, ty); jl_typemap_entry_t *ml = jl_typemap_assoc_exact((jl_typemap_t*)ml_or_cache, arg1, args, n, offs+1, world); if (ml) return ml; } - jl_array_t *tname = jl_atomic_load_relaxed(&cache->tname); - if (jl_is_kind(ty) && tname != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *tname = jl_atomic_load_relaxed(&cache->tname); + if (jl_is_kind(ty) && tname != (jl_genericmemory_t*)jl_an_empty_memory_any) { jl_value_t *name = jl_type_extract_name(a1); if (name) { if (ty != (jl_value_t*)jl_datatype_type) @@ -1235,8 +1236,8 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v } else { // couldn't figure out unique `name` initial point, so must scan all for matches - size_t i, l = jl_array_len(tname); - _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*)jl_array_ptr_data(tname); + size_t i, l = tname->length; + _Atomic(jl_typemap_t*) *data = (_Atomic(jl_typemap_t*)*) tname->data; JL_GC_PUSH1(&tname); for (i = 1; i < l; i += 2) { jl_typemap_t *ml_or_cache = jl_atomic_load_relaxed(&data[i]); @@ -1251,8 +1252,8 @@ jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_v JL_GC_POP(); } } - jl_array_t *name1 = jl_atomic_load_relaxed(&cache->name1); - if (name1 != (jl_array_t*)jl_an_empty_vec_any) { + jl_genericmemory_t *name1 = jl_atomic_load_relaxed(&cache->name1); + if (name1 != (jl_genericmemory_t*)jl_an_empty_memory_any) { while (1) { name1 = jl_atomic_load_relaxed(&cache->name1); // reload after tree descent (which may hit safepoints) jl_typemap_t *ml_or_cache = mtcache_hash_lookup( @@ -1297,23 +1298,23 @@ static jl_typemap_level_t *jl_new_typemap_level(void) jl_typemap_level_t *cache = (jl_typemap_level_t*)jl_gc_alloc(ct->ptls, sizeof(jl_typemap_level_t), jl_typemap_level_type); - jl_atomic_store_relaxed(&cache->arg1, (jl_array_t*)jl_an_empty_vec_any); - jl_atomic_store_relaxed(&cache->targ, (jl_array_t*)jl_an_empty_vec_any); - jl_atomic_store_relaxed(&cache->name1, (jl_array_t*)jl_an_empty_vec_any); - jl_atomic_store_relaxed(&cache->tname, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_relaxed(&cache->arg1, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&cache->targ, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&cache->name1, (jl_genericmemory_t*)jl_an_empty_memory_any); + jl_atomic_store_relaxed(&cache->tname, (jl_genericmemory_t*)jl_an_empty_memory_any); jl_atomic_store_relaxed(&cache->linear, (jl_typemap_entry_t*)jl_nothing); jl_atomic_store_relaxed(&cache->any, jl_nothing); return cache; } -static void jl_typemap_array_insert_( - jl_typemap_t *map, _Atomic(jl_array_t*) *pcache, jl_value_t *key, jl_typemap_entry_t *newrec, +static void jl_typemap_memory_insert_( + jl_typemap_t *map, _Atomic(jl_genericmemory_t*) *pcache, jl_value_t *key, jl_typemap_entry_t *newrec, jl_value_t *parent, int8_t tparam, int8_t offs, jl_value_t *doublesplit); static jl_value_t *jl_method_convert_list_to_cache( jl_typemap_t *map, jl_typemap_entry_t *ml, int8_t tparam, int8_t offs, int8_t doublesplit) { - jl_value_t *cache = doublesplit ? jl_an_empty_vec_any : (jl_value_t*)jl_new_typemap_level(); + jl_value_t *cache = doublesplit ? jl_an_empty_memory_any : (jl_value_t*)jl_new_typemap_level(); jl_typemap_entry_t *next = NULL; JL_GC_PUSH3(&cache, &next, &ml); while (ml != (void*)jl_nothing) { @@ -1336,7 +1337,7 @@ static jl_value_t *jl_method_convert_list_to_cache( assert(jl_is_type_type(key)); key = jl_tparam0(key); } - jl_typemap_array_insert_(map, (_Atomic(jl_array_t*)*)&cache, key, ml, NULL, 0, offs, NULL); + jl_typemap_memory_insert_(map, (_Atomic(jl_genericmemory_t*)*)&cache, key, ml, NULL, 0, offs, NULL); } else jl_typemap_level_insert_(map, (jl_typemap_level_t*)cache, ml, offs); @@ -1371,9 +1372,9 @@ static void jl_typemap_insert_generic( jl_typemap_entry_t *newrec, int8_t tparam, int8_t offs, jl_value_t *doublesplit) { jl_value_t *ml = jl_atomic_load_relaxed(pml); - if (jl_is_array(ml)) { + if (jl_is_genericmemory(ml)) { assert(doublesplit); - jl_typemap_array_insert_(map, (_Atomic(jl_array_t*)*)pml, doublesplit, newrec, parent, 0, offs, NULL); + jl_typemap_memory_insert_(map, (_Atomic(jl_genericmemory_t*)*)pml, doublesplit, newrec, parent, 0, offs, NULL); return; } if (jl_typeof(ml) == (jl_value_t*)jl_typemap_level_type) { @@ -1389,7 +1390,7 @@ static void jl_typemap_insert_generic( jl_atomic_store_release(pml, ml); jl_gc_wb(parent, ml); if (doublesplit) - jl_typemap_array_insert_(map, (_Atomic(jl_array_t*)*)pml, doublesplit, newrec, parent, 0, offs, NULL); + jl_typemap_memory_insert_(map, (_Atomic(jl_genericmemory_t*)*)pml, doublesplit, newrec, parent, 0, offs, NULL); else jl_typemap_level_insert_(map, (jl_typemap_level_t*)ml, newrec, offs); return; @@ -1399,16 +1400,16 @@ static void jl_typemap_insert_generic( parent, newrec); } -static void jl_typemap_array_insert_( - jl_typemap_t *map, _Atomic(jl_array_t*) *pcache, jl_value_t *key, jl_typemap_entry_t *newrec, +static void jl_typemap_memory_insert_( + jl_typemap_t *map, _Atomic(jl_genericmemory_t*) *pcache, jl_value_t *key, jl_typemap_entry_t *newrec, jl_value_t *parent, int8_t tparam, int8_t offs, jl_value_t *doublesplit) { - jl_array_t *cache = jl_atomic_load_relaxed(pcache); + jl_genericmemory_t *cache = jl_atomic_load_relaxed(pcache); _Atomic(jl_value_t*) *pml = mtcache_hash_lookup_bp(cache, key); if (pml == NULL) mtcache_hash_insert(pcache, parent, key, (jl_typemap_t*)newrec); else - jl_typemap_insert_generic(map, pml, (jl_value_t*)cache, newrec, tparam, offs + (doublesplit ? 0 : 1), doublesplit); + jl_typemap_insert_generic(map, pml, (jl_value_t*) cache, newrec, tparam, offs + (doublesplit ? 0 : 1), doublesplit); } static void jl_typemap_level_insert_( @@ -1451,13 +1452,13 @@ static void jl_typemap_level_insert_( jl_value_t *a0 = jl_tparam0(t1); if (is_cache_leaf(a0, 1)) { jl_typename_t *name = a0 == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)a0)->name; - jl_typemap_array_insert_(map, &cache->targ, (jl_value_t*)name, newrec, (jl_value_t*)cache, 1, offs, jl_is_datatype(name->wrapper) ? NULL : a0); + jl_typemap_memory_insert_(map, &cache->targ, (jl_value_t*)name, newrec, (jl_value_t*)cache, 1, offs, jl_is_datatype(name->wrapper) ? NULL : a0); return; } } if (is_cache_leaf(t1, 0)) { jl_typename_t *name = t1 == jl_bottom_type ? jl_typeofbottom_type->name : ((jl_datatype_t*)t1)->name; - jl_typemap_array_insert_(map, &cache->arg1, (jl_value_t*)name, newrec, (jl_value_t*)cache, 0, offs, jl_is_datatype(name->wrapper) ? NULL : t1); + jl_typemap_memory_insert_(map, &cache->arg1, (jl_value_t*)name, newrec, (jl_value_t*)cache, 0, offs, jl_is_datatype(name->wrapper) ? NULL : t1); return; } @@ -1467,12 +1468,12 @@ static void jl_typemap_level_insert_( if (jl_is_type_type(t1)) { a0 = jl_type_extract_name(jl_tparam0(t1)); jl_datatype_t *super = a0 ? (jl_datatype_t*)jl_unwrap_unionall(((jl_typename_t*)a0)->wrapper) : jl_any_type; - jl_typemap_array_insert_(map, &cache->tname, (jl_value_t*)super->name, newrec, (jl_value_t*)cache, 1, offs, NULL); + jl_typemap_memory_insert_(map, &cache->tname, (jl_value_t*)super->name, newrec, (jl_value_t*)cache, 1, offs, NULL); return; } a0 = jl_type_extract_name(t1); if (a0 && a0 != (jl_value_t*)jl_any_type->name) { - jl_typemap_array_insert_(map, &cache->name1, a0, newrec, (jl_value_t*)cache, 0, offs, NULL); + jl_typemap_memory_insert_(map, &cache->name1, a0, newrec, (jl_value_t*)cache, 0, offs, NULL); return; } } diff --git a/stdlib/Base64/src/buffer.jl b/stdlib/Base64/src/buffer.jl index 44a9c0931ac95..bc8c338bad19d 100644 --- a/stdlib/Base64/src/buffer.jl +++ b/stdlib/Base64/src/buffer.jl @@ -2,12 +2,12 @@ # Data buffer for pipes. mutable struct Buffer - data::Vector{UInt8} + data::Memory{UInt8} ptr::Ptr{UInt8} size::Int function Buffer(bufsize) - data = Vector{UInt8}(undef, bufsize) + data = Memory{UInt8}(undef, bufsize) return new(data, pointer(data), 0) end end @@ -18,7 +18,7 @@ Base.setindex!(buffer::Buffer, v::UInt8, i::Integer) = unsafe_store!(buffer.ptr, Base.firstindex(buffer::Buffer) = 1 Base.lastindex(buffer::Buffer) = buffer.size Base.pointer(buffer::Buffer) = buffer.ptr -capacity(buffer::Buffer) = Int(pointer(buffer.data, lastindex(buffer.data) + 1) - buffer.ptr) +capacity(buffer::Buffer) = Int(pointer(buffer.data, lastindex(buffer.data)) - buffer.ptr) + 1 function consumed!(buffer::Buffer, n::Integer) @assert n ≤ buffer.size diff --git a/stdlib/Profile/test/allocs.jl b/stdlib/Profile/test/allocs.jl index ae0cbab945f01..e77e5bc826612 100644 --- a/stdlib/Profile/test/allocs.jl +++ b/stdlib/Profile/test/allocs.jl @@ -143,12 +143,17 @@ end end @testset "alloc profiler catches allocs from buffer resize" begin + f(a) = for _ in 1:100; push!(a, 1); end + f(Int[]) + resize!(Int[], 1) a = Int[] - Allocs.@profile sample_rate=1 for _ in 1:100; push!(a, 1); end - + Allocs.clear() + Allocs.@profile sample_rate=1 f(a) + Allocs.@profile sample_rate=1 resize!(a, 1_000_000) # 4MB prof = Allocs.fetch() Allocs.clear() - @test length(prof.allocs) >= 1 - @test length([a for a in prof.allocs if a.type == Profile.Allocs.BufferType]) >= 1 + @test 3 <= length(prof.allocs) <= 10 + @test length([a for a in prof.allocs if a.type === Allocs.BufferType]) == 1 + @test length([a for a in prof.allocs if a.type === Memory{Int}]) >= 2 end diff --git a/stdlib/Serialization/src/Serialization.jl b/stdlib/Serialization/src/Serialization.jl index fbd02cc1a78dd..d23c1667452d3 100644 --- a/stdlib/Serialization/src/Serialization.jl +++ b/stdlib/Serialization/src/Serialization.jl @@ -288,6 +288,31 @@ function serialize(s::AbstractSerializer, a::SubArray{T,N,A}) where {T,N,A<:Arra serialize_any(s, b) end +serialize(s::AbstractSerializer, m::GenericMemory) = error("GenericMemory{:atomic} currently cannot be serialized") +function serialize(s::AbstractSerializer, m::Memory) + serialize_cycle_header(s, m) && return + serialize(s, length(m)) + elty = eltype(m) + if isbitstype(elty) + serialize_array_data(s.io, m) + else + sizehint!(s.table, div(length(m),4)) # prepare for lots of pointers + @inbounds for i in eachindex(m) + if isassigned(m, i) + serialize(s, m[i]) + else + writetag(s.io, UNDEFREF_TAG) + end + end + end +end + +function serialize(s::AbstractSerializer, x::MemoryRef) + serialize_type(s, typeof(x)) + serialize(s, getfield(x, :mem)) + serialize(s, Base.memoffset(x)) +end + function serialize(s::AbstractSerializer, ss::String) len = sizeof(ss) if len > 7 @@ -511,7 +536,7 @@ function serialize_typename(s::AbstractSerializer, t::Core.TypeName) serialize(s, primary.super) serialize(s, primary.parameters) serialize(s, primary.types) - serialize(s, isdefined(primary, :instance)) + serialize(s, Base.issingletontype(primary)) serialize(s, t.flags & 0x1 == 0x1) # .abstract serialize(s, t.flags & 0x2 == 0x2) # .mutable serialize(s, Int32(length(primary.types) - t.n_uninitialized)) @@ -1276,7 +1301,7 @@ function deserialize_array(s::AbstractSerializer) return A end -function deserialize_fillarray!(A::Array{T}, s::AbstractSerializer) where {T} +function deserialize_fillarray!(A::Union{Array{T},Memory{T}}, s::AbstractSerializer) where {T} for i = eachindex(A) tag = Int32(read(s.io, UInt8)::UInt8) if tag != UNDEFREF_TAG @@ -1286,6 +1311,45 @@ function deserialize_fillarray!(A::Array{T}, s::AbstractSerializer) where {T} return A end +function deserialize(s::AbstractSerializer, X::Type{Memory{T}} where T) + slot = pop!(s.pending_refs) # e.g. deserialize_cycle + n = deserialize(s)::Int + elty = eltype(X) + if isbitstype(elty) + A = X(undef, n) + if X === Memory{Bool} + i = 1 + while i <= n + b = read(s.io, UInt8)::UInt8 + v = (b >> 7) != 0 + count = b & 0x7f + nxt = i + count + while i < nxt + A[i] = v + i += 1 + end + end + else + A = read!(s.io, A)::X + end + s.table[slot] = A + return A + end + A = X(undef, n) + s.table[slot] = A + sizehint!(s.table, s.counter + div(n, 4)) + deserialize_fillarray!(A, s) + return A +end + +function deserialize(s::AbstractSerializer, X::Type{MemoryRef{:not_atomic,T}} where T) + x = Core.memoryref(deserialize(s))::X + i = deserialize(s)::Int + i == 0 || (x = Core.memoryref(x, i + 1, true)) + return x::X +end + + function deserialize_expr(s::AbstractSerializer, len) e = Expr(:temp) resolve_ref_immediately(s, e) @@ -1341,7 +1405,7 @@ function deserialize_typename(s::AbstractSerializer, number) tn.max_methods = maxm if has_instance ty = ty::DataType - if !Base.issingletontype(ty) + if !isdefined(ty, :instance) singleton = ccall(:jl_new_struct, Any, (Any, Any...), ty) # use setfield! directly to avoid `fieldtype` lowering expecting to see a Singleton object already on ty ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), ty, Base.fieldindex(DataType, :instance)-1, singleton) diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 29bc514bb1802..2847449f1cce4 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -1101,23 +1101,23 @@ end @testset "sizeof" begin let arrUInt8 = zeros(UInt8, 10) @test sizeof(arrUInt8) == 10 - @test Core.sizeof(arrUInt8) == 10 + @test Core.sizeof(arrUInt8) == 3 * sizeof(Int) end let arrUInt32 = zeros(UInt32, 10) @test sizeof(arrUInt32) == 40 - @test Core.sizeof(arrUInt32) == 40 + @test Core.sizeof(arrUInt32) == 3 * sizeof(Int) end let arrFloat64 = zeros(Float64, 10, 10) @test sizeof(arrFloat64) == 800 - @test Core.sizeof(arrFloat64) == 800 + @test Core.sizeof(arrFloat64) == 4 * sizeof(Int) end # Test union arrays (Issue #23321) let arrUnion = Union{Int64, Cvoid}[rand(Bool) ? k : nothing for k = 1:10] @test sizeof(arrUnion) == 80 - @test Core.sizeof(arrUnion) == 80 + @test Core.sizeof(arrUnion) == 3 * sizeof(Int) end # Test non-power of 2 types (Issue #35884) diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index b29904fb5eb6c..ac4c17e52ac04 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -508,9 +508,9 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test popfirst!(got) == " 32 Base.invokelatest(g, x)" end if Sys.WORD_SIZE == 64 - @test popfirst!(got) == " 48 []" - else @test popfirst!(got) == " 32 []" + else + @test popfirst!(got) == " 16 []" end @test popfirst!(got) == " - end" @test popfirst!(got) == " - f(1.23)" diff --git a/test/compiler/EscapeAnalysis/EscapeAnalysis.jl b/test/compiler/EscapeAnalysis/EscapeAnalysis.jl index 9dcfa591a54de..742ed692a8504 100644 --- a/test/compiler/EscapeAnalysis/EscapeAnalysis.jl +++ b/test/compiler/EscapeAnalysis/EscapeAnalysis.jl @@ -39,7 +39,6 @@ let utils_ex = quote Core.eval(@__MODULE__, utils_ex) end -using Core.Compiler: alloc_array_ndims using .EscapeAnalysis: EscapeInfo, IndexableElements, IndexableFields, array_resize_info, is_array_copy, normalize diff --git a/test/compiler/codegen.jl b/test/compiler/codegen.jl index bd0b5d8f93e7b..77a2b316c557d 100644 --- a/test/compiler/codegen.jl +++ b/test/compiler/codegen.jl @@ -128,11 +128,16 @@ if !is_debug_build && opt_level > 0 # String test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Core.SimpleVector})), [Iptr]) # Array - test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Vector{Int}})), [Iptr]) + test_loads_no_call(strip_debug_calls(get_llvm(sizeof, Tuple{Vector{Int}})), [Iptr]) + # As long as the eltype is known we don't need to load the elsize, but do need to check isvector + test_loads_no_call(strip_debug_calls(get_llvm(sizeof, Tuple{Array{Any}})), ["atomic $Iptr", "{} addrspace(10)* addrspace(10)*", "$Iptr addrspace(10)*", Iptr, Iptr, "{ i64, {} addrspace(10)** } addrspace(10)*", Iptr]) + # Memory + test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory{Int}})), [Iptr]) # As long as the eltype is known we don't need to load the elsize - test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Array{Any}})), [Iptr]) - # Check that we load the elsize - test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Vector})), [Iptr, "i16"]) + test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory{Any}})), [Iptr]) + # Check that we load the elsize from the typeof layout + test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory})), [Iptr, "atomic $Iptr", "i32*", "i32"]) + test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Memory})), [Iptr, "atomic $Iptr", "i32*", "i32"]) # Primitive Type size should be folded to a constant test_loads_no_call(strip_debug_calls(get_llvm(core_sizeof, Tuple{Ptr})), String[]) diff --git a/test/compiler/effects.jl b/test/compiler/effects.jl index 94eb46ee7810a..597232affe858 100644 --- a/test/compiler/effects.jl +++ b/test/compiler/effects.jl @@ -696,31 +696,33 @@ end # low-level constructor @noinline construct_array(@nospecialize(T), args...) = Array{T}(undef, args...) # should eliminate safe but dead allocations -let good_dims = @static Int === Int64 ? (1:10) : (1:8) - Ns = @static Int === Int64 ? (1:10) : (1:8) +let good_dims = [1, 2, 3, 4, 10] + Ns = [1, 2, 3, 4, 10] for dim = good_dims, N = Ns dims = ntuple(i->dim, N) @test @eval Base.infer_effects() do - $construct_array(Int, $(dims...)) + construct_array(Int, $(dims...)) end |> Core.Compiler.is_removable_if_unused @test @eval fully_eliminated() do - $construct_array(Int, $(dims...)) + construct_array(Int, $(dims...)) nothing end end end # should analyze throwness correctly let bad_dims = [-1, typemax(Int)] - for dim in bad_dims, N in 1:10 - dims = ntuple(i->dim, N) - @test @eval Base.infer_effects() do - $construct_array(Int, $(dims...)) - end |> !Core.Compiler.is_removable_if_unused - @test @eval !fully_eliminated() do - $construct_array(Int, $(dims...)) - nothing + for dim in bad_dims, N in [1, 2, 3, 4, 10] + for T in Any[Int, Union{Missing,Nothing}, Missing, Any] + dims = ntuple(i->dim, N) + @test @eval Base.infer_effects() do + construct_array($T, $(dims...)) + end |> !Core.Compiler.is_removable_if_unused + @test @eval !fully_eliminated() do + construct_array($T, $(dims...)) + nothing + end + @test_throws "invalid " @eval construct_array($T, $(dims...)) end - @test_throws "invalid Array" @eval $construct_array(Int, $(dims...)) end end @@ -764,12 +766,10 @@ for safesig = Any[ end end -# arrayref -# -------- - -for tt = Any[(Bool,Vector{Any},Int), - (Bool,Matrix{Any},Int,Int)] - @testset let effects = Base.infer_effects(Base.arrayref, tt) +# array getindex +for tt = Any[(Vector{Any},Int), + (Matrix{Any},Int,Int)] + @testset let effects = Base.infer_effects(Base.getindex, tt) @test Core.Compiler.is_consistent_if_inaccessiblememonly(effects) @test Core.Compiler.is_effect_free(effects) @test !Core.Compiler.is_nothrow(effects) @@ -777,12 +777,10 @@ for tt = Any[(Bool,Vector{Any},Int), end end -# arrayset -# -------- - -for tt = Any[(Bool,Vector{Any},Any,Int), - (Bool,Matrix{Any},Any,Int,Int)] - @testset let effects = Base.infer_effects(Base.arrayset, tt) +# array setindex! +for tt = Any[(Vector{Any},Any,Int), + (Matrix{Any},Any,Int,Int)] + @testset let effects = Base.infer_effects(Base.setindex!, tt) @test Core.Compiler.is_consistent_if_inaccessiblememonly(effects) @test Core.Compiler.is_effect_free_if_inaccessiblememonly(effects) @test !Core.Compiler.is_nothrow(effects) @@ -819,7 +817,7 @@ end # arraylen # -------- -let effects = Base.infer_effects(Base.arraylen, (Vector{Any},)) +let effects = Base.infer_effects(length, (Vector{Any},)) @test Core.Compiler.is_consistent_if_inaccessiblememonly(effects) @test Core.Compiler.is_effect_free(effects) @test Core.Compiler.is_nothrow(effects) diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index c7e0cdfb6f6b2..91cb479f1eda4 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -349,7 +349,7 @@ barTuple2() = fooTuple{tuple(:y)}() # issue #6050 @test Core.Compiler.getfield_tfunc(Core.Compiler.fallback_lattice, Dict{Int64,Tuple{UnitRange{Int64},UnitRange{Int64}}}, - Core.Compiler.Const(:vals)) == Array{Tuple{UnitRange{Int64},UnitRange{Int64}},1} + Core.Compiler.Const(:vals)) == Memory{Tuple{UnitRange{Int64},UnitRange{Int64}}} # assert robustness of `getfield_tfunc` struct GetfieldRobustness @@ -1173,8 +1173,6 @@ let isdefined_tfunc(@nospecialize xs...) = @test isdefined_tfunc(Core.SimpleVector, Const(1)) === Const(false) @test Const(false) ⊑ isdefined_tfunc(Const(:x), Symbol) @test Const(false) ⊑ isdefined_tfunc(Const(:x), Const(:y)) - @test isdefined_tfunc(Vector{Int}, Const(1)) == Const(false) - @test isdefined_tfunc(Vector{Any}, Const(1)) == Const(false) @test isdefined_tfunc(Module, Int) === Union{} @test isdefined_tfunc(Tuple{Any,Vararg{Any}}, Const(0)) === Const(false) @test isdefined_tfunc(Tuple{Any,Vararg{Any}}, Const(1)) === Const(true) @@ -1314,30 +1312,33 @@ test_const_return(()->sizeof(1), Tuple{}, sizeof(Int)) test_const_return(()->sizeof(DataType), Tuple{}, sizeof(DataType)) test_const_return(()->sizeof(1 < 2), Tuple{}, 1) test_const_return(()->fieldtype(Dict{Int64,Nothing}, :age), Tuple{}, UInt) -test_const_return(@eval(()->Core.sizeof($(Array{Int,0}(undef)))), Tuple{}, sizeof(Int)) -test_const_return(@eval(()->Core.sizeof($(Matrix{Float32}(undef, 2, 2)))), Tuple{}, 4 * 2 * 2) +test_const_return(@eval(()->Core.sizeof($(Array{Int,0}(undef)))), Tuple{}, 2 * sizeof(Int)) +test_const_return(@eval(()->Core.sizeof($(Matrix{Float32}(undef, 2, 2)))), Tuple{}, 4 * sizeof(Int)) +# TODO: do we want to implement these? +# test_const_return(@eval(()->sizeof($(Array{Int,0}(undef)))), Tuple{}, sizeof(Int)) +# test_const_return(@eval(()->sizeof($(Matrix{Float32}(undef, 2, 2)))), Tuple{}, 4 * 2 * 2) +# test_const_return(@eval(()->Core.sizeof($(Memory{Int}(undef, 0)))), Tuple{}, 0) # Make sure Core.sizeof with a ::DataType as inferred input type is inferred but not constant. function sizeof_typeref(typeref) return Core.sizeof(typeref[]) end @test @inferred(sizeof_typeref(Ref{DataType}(Int))) == sizeof(Int) -@test find_call(first(code_typed(sizeof_typeref, (Ref{DataType},))[1]), Core.sizeof, 2) +@test find_call(only(code_typed(sizeof_typeref, (Ref{DataType},)))[1], Core.sizeof, 2) # Constant `Vector` can be resized and shouldn't be optimized to a constant. const constvec = [1, 2, 3] @eval function sizeof_constvec() - return Core.sizeof($constvec) + return sizeof($constvec) end @test @inferred(sizeof_constvec()) == sizeof(Int) * 3 -@test find_call(first(code_typed(sizeof_constvec, ())[1]), Core.sizeof, 2) push!(constvec, 10) -@test @inferred(sizeof_constvec()) == sizeof(Int) * 4 +@test sizeof_constvec() == sizeof(Int) * 4 test_const_return(x->isdefined(x, :re), Tuple{ComplexF64}, true) isdefined_f3(x) = isdefined(x, 3) @test @inferred(isdefined_f3(())) == false -@test find_call(first(code_typed(isdefined_f3, Tuple{Tuple{Vararg{Int}}})[1]), isdefined, 3) +@test find_call(only(code_typed(isdefined_f3, Tuple{Tuple{Vararg{Int}}}))[1], isdefined, 3) let isa_tfunc(@nospecialize xs...) = Core.Compiler.isa_tfunc(Core.Compiler.fallback_lattice, xs...) @@ -1515,7 +1516,7 @@ let nfields_tfunc(@nospecialize xs...) = @test sizeof_nothrow(Type{Ptr}) @test sizeof_nothrow(Type{Union{Ptr{Int}, Int}}) @test !sizeof_nothrow(Const(Tuple)) - @test !sizeof_nothrow(Type{Vector{Int}}) + @test sizeof_nothrow(Type{Vector{Int}}) @test !sizeof_nothrow(Type{Union{Int, String}}) @test sizeof_nothrow(String) @test !sizeof_nothrow(Type{String}) diff --git a/test/compiler/inline.jl b/test/compiler/inline.jl index 07d4c7bf363a8..de838ed507d78 100644 --- a/test/compiler/inline.jl +++ b/test/compiler/inline.jl @@ -1566,11 +1566,11 @@ end $f(xs, x) end @test count(iscall((src, $f)), src.code) == 0 - @test count(src.code) do @nospecialize x - isa(x, Core.GotoNode) || - isa(x, Core.GotoIfNot) || - iscall((src, getfield))(x) - end == 0 # no loop should be involved for the common single arg case + end + let effects = Base.infer_effects((Vector{Any}, Any)) do xs, x + $f(xs, x) + end + @test Core.Compiler.Core.Compiler.is_terminates(effects) end let src = code_typed1((Vector{Any}, Any, Any)) do xs, x, y $f(xs, x, y) diff --git a/test/compiler/irpasses.jl b/test/compiler/irpasses.jl index 66d040fe8b893..d9e230a721d2c 100644 --- a/test/compiler/irpasses.jl +++ b/test/compiler/irpasses.jl @@ -1145,8 +1145,8 @@ end let # effect-freeness computation for array allocation # should eliminate dead allocations - good_dims = @static Int === Int64 ? (1:10) : (1:8) - Ns = @static Int === Int64 ? (1:10) : (1:8) + good_dims = [1, 2, 3, 4, 10] + Ns = [1, 2, 3, 4, 10] for dim = good_dims, N = Ns dims = ntuple(i->dim, N) @test @eval fully_eliminated() do @@ -1157,14 +1157,14 @@ let # effect-freeness computation for array allocation # shouldn't eliminate erroneous dead allocations bad_dims = [-1, typemax(Int)] - for dim in bad_dims, N in 1:10 + for dim in bad_dims, N in [1, 2, 3, 4, 10], T in Any[Int, Union{Missing,Nothing}, Nothing, Any] dims = ntuple(i->dim, N) @test @eval !fully_eliminated() do - Array{Int,$N}(undef, $(dims...)) + Array{$T,$N}(undef, $(dims...)) nothing end - @test_throws "invalid Array" @eval let - Array{Int,$N}(undef, $(dims...)) + @test_throws "invalid " @eval let + Array{$T,$N}(undef, $(dims...)) nothing end end diff --git a/test/compiler/ssair.jl b/test/compiler/ssair.jl index b654ad5923b01..17864987fc7d7 100644 --- a/test/compiler/ssair.jl +++ b/test/compiler/ssair.jl @@ -38,7 +38,7 @@ end # false, false, false, false # )) # -# NullLineInfo = Core.LineInfoNode(Main, Symbol(""), Symbol(""), Int32(0), Int32(0)) +# NullLineInfo = Core.LineInfoNode(Main, Symbol(""), Symbol(""), Int32(0), UInt32(0)) # Compiler.run_passes(ci, 1, [NullLineInfo]) # # XXX: missing @test #end diff --git a/test/core.jl b/test/core.jl index af9a896de59f8..39d9b3069c289 100644 --- a/test/core.jl +++ b/test/core.jl @@ -23,6 +23,8 @@ for (T, c) in ( (Core.TypeName, [:name, :module, :names, :wrapper, :mt, :hash, :n_uninitialized, :flags]), (DataType, [:name, :super, :parameters, :instance, :hash]), (TypeVar, [:name, :ub, :lb]), + (Core.Memory, [:length, :data]), + (Core.MemoryRef, [:mem, :ptr]), ) @test Set((fieldname(T, i) for i in 1:fieldcount(T) if isconst(T, i))) == Set(c) end @@ -38,6 +40,8 @@ for (T, c) in ( (Core.TypeMapLevel, [:arg1, :targ, :name1, :tname, :list, :any]), (Core.TypeName, [:cache, :linearcache]), (DataType, [:types, :layout]), + (Core.Memory, []), + (Core.MemoryRef, []), ) @test Set((fieldname(T, i) for i in 1:fieldcount(T) if Base.isfieldatomic(T, i))) == Set(c) end @@ -4334,6 +4338,7 @@ function f15180(x::T) where T end @test map(f15180(1), [1,2]) == [(Int,1),(Int,1)] +using Base: _growbeg!, _deletebeg!, _growend!, _deleteend! struct ValueWrapper vpadding::NTuple{2,VecElement{UInt}} value @@ -4342,43 +4347,44 @@ end Base.convert(::Type{ValueWrapper}, x) = ValueWrapper(x) for T in (Any, ValueWrapper) let ary = Vector{T}(undef, 10) - check_undef_and_fill(ary, rng) = for i in rng - @test !isassigned(ary, i) + check_undef_and_fill(ary, rng) = all(i -> begin + isassigned(ary, i) && return false ary[i] = (Float64(i), i) # some non-cached content - @test isassigned(ary, i) - end + isassigned(ary, i) || return false + return true + end, rng) # Check if the memory is initially zerod and fill it with value # to check if these values are not reused later. - check_undef_and_fill(ary, 1:10) + @test check_undef_and_fill(ary, 1:10) # Check if the memory grown at the end are zerod - ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), ary, 10) - check_undef_and_fill(ary, 11:20) + _growend!(ary, 10) + @test check_undef_and_fill(ary, 11:20) # Make sure the content of the memory deleted at the end are not reused - ccall(:jl_array_del_end, Cvoid, (Any, Csize_t), ary, 5) - ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), ary, 5) - check_undef_and_fill(ary, 16:20) + _deleteend!(ary, 5) + _growend!(ary, 5) + @test check_undef_and_fill(ary, 16:20) # Now check grow/del_end ary = Vector{T}(undef, 1010) - check_undef_and_fill(ary, 1:1010) + @test check_undef_and_fill(ary, 1:1010) # This del_beg should move the buffer - ccall(:jl_array_del_beg, Cvoid, (Any, Csize_t), ary, 1000) - ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), ary, 1000) - check_undef_and_fill(ary, 1:1000) + _deletebeg!(ary, 1000) + _growbeg!(ary, 1000) + @test check_undef_and_fill(ary, 1:1000) ary = Vector{T}(undef, 1010) - check_undef_and_fill(ary, 1:1010) + @test check_undef_and_fill(ary, 1:1010) # This del_beg should not move the buffer - ccall(:jl_array_del_beg, Cvoid, (Any, Csize_t), ary, 10) - ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), ary, 10) - check_undef_and_fill(ary, 1:10) + _deletebeg!(ary, 10) + _growbeg!(ary, 10) + @test check_undef_and_fill(ary, 1:10) ary = Vector{T}(undef, 1010) - check_undef_and_fill(ary, 1:1010) - ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), ary, 10) - check_undef_and_fill(ary, 1011:1020) - ccall(:jl_array_del_end, Cvoid, (Any, Csize_t), ary, 10) - ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), ary, 10) - check_undef_and_fill(ary, 1:10) + @test check_undef_and_fill(ary, 1:1010) + _growend!(ary, 10) + @test check_undef_and_fill(ary, 1011:1020) + _deleteend!(ary, 10) + _growbeg!(ary, 10) + @test check_undef_and_fill(ary, 1:10) # Make sure newly malloc'd buffers are filled with 0 # test this for a few different sizes since we need to make sure @@ -4391,33 +4397,30 @@ for T in (Any, ValueWrapper) GC.gc() GC.gc() GC.gc() - ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), ary, 4) - ccall(:jl_array_del_beg, Cvoid, (Any, Csize_t), ary, 4) - ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), ary, n) - ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), ary, 4) - check_undef_and_fill(ary, 1:(2n + 4)) + _growbeg!(ary, 4) + _deletebeg!(ary, 4) + _growend!(ary, n) + _growbeg!(ary, 4) + @test check_undef_and_fill(ary, 1:(2n + 4)) end ary = Vector{T}(undef, 100) - ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), ary, 10000) + _growend!(ary, 10000) ary[:] = 1:length(ary) - ccall(:jl_array_del_beg, Cvoid, (Any, Csize_t), ary, 10000) + _deletebeg!(ary, 10000) # grow on the back until a buffer reallocation happens cur_ptr = pointer(ary) while cur_ptr == pointer(ary) len = length(ary) - ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), ary, 10) - for i in (len + 1):(len + 10) - @test !isassigned(ary, i) - end + _growend!(ary, 10) + result = @test all(i -> !isassigned(ary, i), (len + 1):(len + 10)) + result isa Test.Pass || break end ary = Vector{T}(undef, 100) ary[:] = 1:length(ary) - ccall(:jl_array_grow_at, Cvoid, (Any, Csize_t, Csize_t), ary, 50, 10) - for i in 51:60 - @test !isassigned(ary, i) - end + Base._growat!(ary, 50, 10) + @test all(i -> !isassigned(ary, i), 51:60) end end @@ -4514,46 +4517,6 @@ let arrayset_unknown_dim(Int, 3) end -module TestSharedArrayResize -using Test -# Attempting to change the shape of a shared array should unshare it and -# not modify the original data -function test_shared_array_resize(::Type{T}) where T - len = 100 - a = Vector{T}(undef, len) - function test_unshare(f) - a′ = reshape(reshape(a, (len ÷ 2, 2)), len) - a[:] = 1:length(a) - # The operation should fail on the owner shared array - # and has no side effect. - @test_throws ErrorException f(a) - @test a == [1:len;] - @test a′ == [1:len;] - @test pointer(a) == pointer(a′) - # The operation should pass on the non-owner shared array - # and should unshare the arrays with no effect on the original one. - f(a′) - @test a == [1:len;] - @test pointer(a) != pointer(a′) - end - - test_unshare(a->ccall(:jl_array_del_end, Cvoid, (Any, Csize_t), a, 0)) - test_unshare(a->ccall(:jl_array_del_end, Cvoid, (Any, Csize_t), a, 1)) - test_unshare(a->ccall(:jl_array_del_beg, Cvoid, (Any, Csize_t), a, 0)) - test_unshare(a->ccall(:jl_array_del_beg, Cvoid, (Any, Csize_t), a, 1)) - test_unshare(a->deleteat!(a, 10)) - test_unshare(a->deleteat!(a, 90)) - test_unshare(a->ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), a, 0)) - test_unshare(a->ccall(:jl_array_grow_end, Cvoid, (Any, Csize_t), a, 1)) - test_unshare(a->ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), a, 0)) - test_unshare(a->ccall(:jl_array_grow_beg, Cvoid, (Any, Csize_t), a, 1)) - test_unshare(a->insert!(a, 10, 10)) - test_unshare(a->insert!(a, 90, 90)) -end -test_shared_array_resize(Int) -test_shared_array_resize(Any) -end - # Copy of `#undef` copyto!(Vector{Any}(undef, 10), Vector{Any}(undef, 10)) function test_copy_alias(::Type{T}) where T @@ -6337,7 +6300,7 @@ for U in unboxedunions resize!(A, len) @test length(A) === len @test A[1] === initvalue2(F2) - @test typeof(A[end]) === F + @test typeof(A[end]) === F2 # deleteat! F = Base.uniontypes(U)[2] @@ -6425,7 +6388,7 @@ for U in unboxedunions end end -@testset "jl_array_grow_at_end" begin +@testset "array _growatend!" begin # start w/ array, set & check elements, grow it, check that elements stayed correct, set & check elements A = Vector{Union{Missing, UInt8}}(undef, 2) @@ -6531,7 +6494,7 @@ end end # @testset -@testset "jl_array_grow_at_beg" begin +@testset "array _growatbeg!" begin # grow_at_beg 4 A = Vector{Union{Missing, UInt8}}(undef, 5) @@ -6655,7 +6618,7 @@ end end # @testset -@testset "jl_array_del_at_beg" begin +@testset "array _deleteatbeg!" begin A = Vector{Union{Missing, UInt8}}(undef, 5) Base.arrayset(true, A, 0x01, 1) @@ -6704,7 +6667,7 @@ Base._deleteat!(A, 1, 1) end # @testset -@testset "jl_array_del_at_end" begin +@testset "array _deleteatend!" begin A = Vector{Union{Missing, UInt8}}(undef, 5) Base.arrayset(true, A, 0x01, 1) @@ -7504,16 +7467,11 @@ function f34482() Base.not_int("ABC") 1 end -function g34482() - Core.Intrinsics.arraylen(1) - 1 -end function h34482() Core.Intrinsics.bitcast(1, 1) 1 end @test_throws ErrorException f34482() -@test_throws TypeError g34482() @test_throws TypeError h34482() struct NFANode34126 diff --git a/test/embedding/embedding.c b/test/embedding/embedding.c index c5b8845b7c823..746c59fc8ce1f 100644 --- a/test/embedding/embedding.c +++ b/test/embedding/embedding.c @@ -86,17 +86,17 @@ int main() // (aka, is gc-rooted until) the program reaches the corresponding JL_GC_POP() JL_GC_PUSH1(&x); - double* xData = jl_array_data(x); + double* xData = jl_array_data(x, double); size_t i; - for (i = 0; i < jl_array_len(x); i++) + for (i = 0; i < jl_array_nrows(x); i++) xData[i] = i; jl_function_t *func = jl_get_function(jl_base_module, "reverse!"); jl_call1(func, (jl_value_t*) x); printf("x = ["); - for (i = 0; i < jl_array_len(x); i++) + for (i = 0; i < jl_array_nrows(x); i++) printf("%e ", xData[i]); printf("]\n"); fflush(stdout); diff --git a/test/precompile.jl b/test/precompile.jl index 4841dfb07ddcd..1e49956b25b33 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -713,15 +713,15 @@ precompile_test_harness("code caching") do dir @test hasspec # Test that compilation adds to method roots with appropriate provenance m = which(setindex!, (Dict{M.X,Any}, Any, M.X)) - @test M.X ∈ m.roots + @test Memory{M.X} ∈ m.roots # Check that roots added outside of incremental builds get attributed to a moduleid of 0 Base.invokelatest() do Dict{M.X2,Any}()[M.X2()] = nothing end - @test M.X2 ∈ m.roots + @test Memory{M.X2} ∈ m.roots groups = group_roots(m) - @test M.X ∈ groups[Mid] # attributed to M - @test M.X2 ∈ groups[0] # activate module is not known + @test Memory{M.X} ∈ groups[Mid] # attributed to M + @test Memory{M.X2} ∈ groups[0] # activate module is not known @test !isempty(groups[Bid]) # Check that internal methods and their roots are accounted appropriately minternal = which(M.getelsize, (Vector,)) diff --git a/test/reflection.jl b/test/reflection.jl index a67407c2d0f48..30804a68d6219 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -614,11 +614,15 @@ end sizeof(Real)) @test sizeof(Union{ComplexF32,ComplexF64}) == 16 @test sizeof(Union{Int8,UInt8}) == 1 -@test_throws ErrorException sizeof(AbstractArray) +@test sizeof(MemoryRef{false,Int}) == 2 * sizeof(Int) +@test sizeof(Array{Int,0}) == 2 * sizeof(Int) +@test sizeof(Array{Int,1}) == 3 * sizeof(Int) +@test sizeof(Array{Int,2}) == 4 * sizeof(Int) +@test sizeof(Array{Int,20}) == 22 * sizeof(Int) @test_throws ErrorException sizeof(Tuple) @test_throws ErrorException sizeof(Tuple{Any,Any}) @test_throws ErrorException sizeof(String) -@test_throws ErrorException sizeof(Vector{Int}) +@test_throws ErrorException sizeof(Memory{false,Int}) @test_throws ErrorException sizeof(Symbol) @test_throws ErrorException sizeof(Core.SimpleVector) @test_throws ErrorException sizeof(Union{}) @@ -923,7 +927,7 @@ end @test nameof(Any) === :Any @test nameof(:) === :Colon @test nameof(Core.Intrinsics.mul_int) === :mul_int -@test nameof(Core.Intrinsics.arraylen) === :arraylen +@test nameof(Core.Intrinsics.cglobal) === :cglobal module TestMod33403 f(x) = 1 @@ -1039,7 +1043,7 @@ ambig_effects_test(a, b) = 1 @test Base.infer_effects(typeof, (Any,)) |> Core.Compiler.is_foldable_nothrow @test Base.infer_effects(===, (Any,Any)) |> Core.Compiler.is_foldable_nothrow @test (Base.infer_effects(setfield!, ()); true) # `builtin_effects` shouldn't throw on empty `argtypes` - @test (Base.infer_effects(Core.Intrinsics.arraylen, ()); true) # `intrinsic_effects` shouldn't throw on empty `argtypes` + @test (Base.infer_effects(Core.Intrinsics.mul_int, ()); true) # `intrinsic_effects` shouldn't throw on empty `argtypes` end @test Base._methods_by_ftype(Tuple{}, -1, Base.get_world_counter()) == Any[] diff --git a/test/show.jl b/test/show.jl index caeab7d17600c..d9908c6398fd0 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1963,12 +1963,12 @@ end end @testset "Intrinsic printing" begin - @test sprint(show, Core.Intrinsics.arraylen) == "Core.Intrinsics.arraylen" - @test repr(Core.Intrinsics.arraylen) == "Core.Intrinsics.arraylen" + @test sprint(show, Core.Intrinsics.cglobal) == "Core.Intrinsics.cglobal" + @test repr(Core.Intrinsics.cglobal) == "Core.Intrinsics.cglobal" let io = IOBuffer() - show(io, MIME"text/plain"(), Core.Intrinsics.arraylen) + show(io, MIME"text/plain"(), Core.Intrinsics.cglobal) str = String(take!(io)) - @test occursin("arraylen", str) + @test occursin("cglobal", str) @test occursin("(intrinsic function", str) end @test string(Core.Intrinsics.add_int) == "add_int" diff --git a/test/sorting.jl b/test/sorting.jl index 147a70a5db7d9..17888cfd0e4ee 100644 --- a/test/sorting.jl +++ b/test/sorting.jl @@ -972,8 +972,8 @@ end @testset "ScratchQuickSort allocations on non-concrete eltype" begin v = Vector{Union{Nothing, Bool}}(rand(Bool, 10000)) - @test 4 == @allocations sort(v) - @test 4 == @allocations sort(v; alg=Base.Sort.ScratchQuickSort()) + @test 10 > @allocations sort(v) + @test 10 > @allocations sort(v; alg=Base.Sort.ScratchQuickSort()) # it would be nice if these numbers were lower (1 or 2), but these # test that we don't have O(n) allocations due to type instability end @@ -981,15 +981,15 @@ end function test_allocs() v = rand(10) i = randperm(length(v)) - @test 1 == @allocations sort(v) + @test 2 >= @allocations sort(v) @test 0 == @allocations sortperm!(i, v) @test 0 == @allocations sort!(i) @test 0 == @allocations sortperm!(i, v, rev=true) - @test 1 == @allocations sortperm(v, rev=true) - @test 1 == @allocations sortperm(v, rev=false) + @test 2 >= @allocations sortperm(v, rev=true) + @test 2 >= @allocations sortperm(v, rev=false) @test 0 == @allocations sortperm!(i, v, order=Base.Reverse) - @test 1 == @allocations sortperm(v) - @test 1 == @allocations sortperm(i, by=sqrt) + @test 2 >= @allocations sortperm(v) + @test 2 >= @allocations sortperm(i, by=sqrt) @test 0 == @allocations sort!(v, lt=(a, b) -> hash(a) < hash(b)) sort!(Int[], rev=false) # compile @test 0 == @allocations sort!(i, rev=false)