From 898dc8f5f350da2d05c8b7cea10f1225839087ca Mon Sep 17 00:00:00 2001 From: Andy Ferris Date: Tue, 23 Jul 2019 22:18:13 +1000 Subject: [PATCH 01/21] WIP: make `reinterpret` work on structs --- base/reinterpretarray.jl | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index f34c295918f6a..44db6f554380c 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -765,3 +765,34 @@ end mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, ifirst::SCartesianIndex2, ilast::SCartesianIndex2) where {F,OP} = mapreduce_impl(f, op, A, ifirst, ilast, pairwise_blocksize(f, op)) + +@pure function struct_subpadding(::Type{Out}, ::Type{In}) where {Out, In} + # TODO I believe the semantic is we cannot read the padding bits of the input, + # and therefore we should relax this to padding(In) being a subset of padding(Out). + padding(Out) == padding(In) +end + +@inline function reinterpret(::Type{Out}, x) where {Out} + In = typeof(x) + if !isbitstype(Out) + error("reinterpret target type must be isbits") + end + if !isbitstype(In) + error("reinterpret source type must be isbits") + end + + if isprimitivetype(Out) && isprimitivetype(In) + return bitcast(Out, x) + elseif struct_subpadding(Out, In) + in = Ref{In}(x) + ptr_in = unsafe_convert(Ptr{In}, in) + out = Ref{Out}() + ptr_out = unsafe_convert(Ptr{Out}, out) + GC.@preserve in out begin + _memcpy!(ptr_out, ptr_in, sizeof(Out)) + end + return out[] + else + throw(PaddingError(Out, In)) + end +end From dc41f6e90acfd58679839898c3def505534b3865 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Fri, 17 Jun 2022 10:39:35 -0400 Subject: [PATCH 02/21] Reinterpret packed bits --- base/reinterpretarray.jl | 124 ++++++++++++++++++++++++++++++--------- 1 file changed, 97 insertions(+), 27 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 44db6f554380c..9a5483fc8f5fd 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -631,8 +631,8 @@ end # Padding struct Padding - offset::Int - size::Int + offset::Int # 0-indexed offset of the next valid byte; sizeof(T) indicates trailing padding + size::Int # bytes of padding before a valid byte end function intersect(p1::Padding, p2::Padding) start = max(p1.offset, p2.offset) @@ -676,20 +676,24 @@ function iterate(cp::CyclePadding, state::Tuple) end """ - Compute the location of padding in a type. + Compute the location of padding in a type. Recursive for nested types. """ -function padding(T) - padding = Padding[] - last_end::Int = 0 +function padding(T, baseoffset = 0) + pads = Padding[] + last_end::Int = baseoffset for i = 1:fieldcount(T) - offset = fieldoffset(T, i) + offset = baseoffset + fieldoffset(T, i) fT = fieldtype(T, i) + append!(pads, padding(fT, offset)) if offset != last_end - push!(padding, Padding(offset, offset-last_end)) + push!(pads, Padding(offset, offset-last_end)) end last_end = offset + sizeof(fT) end - padding + if 0 < last_end < sizeof(T) + push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end)) + end + return pads end function CyclePadding(T::DataType) @@ -767,32 +771,98 @@ mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, ifirst::SCartesianIn mapreduce_impl(f, op, A, ifirst, ilast, pairwise_blocksize(f, op)) @pure function struct_subpadding(::Type{Out}, ::Type{In}) where {Out, In} - # TODO I believe the semantic is we cannot read the padding bits of the input, - # and therefore we should relax this to padding(In) being a subset of padding(Out). padding(Out) == padding(In) end -@inline function reinterpret(::Type{Out}, x) where {Out} - In = typeof(x) - if !isbitstype(Out) - error("reinterpret target type must be isbits") - end - if !isbitstype(In) - error("reinterpret source type must be isbits") - end +@pure function packedsize(::Type{T}) where T + pads = padding(T) + return sizeof(T) - (isempty(pads) ? 0 : sum(p.size for p ∈ pads)) +end + +@pure ispacked(::Type{T}) where T = packedsize(T) == sizeof(T) - if isprimitivetype(Out) && isprimitivetype(In) - return bitcast(Out, x) - elseif struct_subpadding(Out, In) - in = Ref{In}(x) - ptr_in = unsafe_convert(Ptr{In}, in) - out = Ref{Out}() - ptr_out = unsafe_convert(Ptr{Out}, out) +@inline function reinterpret(::Type{Out}, x::In) where {Out, In} + isbitstype(Out) || error("reinterpret target type must be isbits") + isbitstype(In) || error("reinterpret source type must be isbits") + isprimitivetype(Out) && isprimitivetype(In) && return bitcast(Out, x) + inpackedsize = packedsize(In) + outpackedsize = packedsize(Out) + inpackedsize == outpackedsize || throw(PaddingError(Out, In)) + in = Ref{In}(x) + out = Ref{Out}() + if struct_subpadding(Out, In) + # if packed the same, just copy GC.@preserve in out begin + ptr_in = unsafe_convert(Ptr{In}, in) + ptr_out = unsafe_convert(Ptr{Out}, out) _memcpy!(ptr_out, ptr_in, sizeof(Out)) end return out[] else - throw(PaddingError(Out, In)) + # mismatched padding + function copytopacked(ptr_out, ptr_in) + writeoffset = 0 + for i ∈ 1:fieldcount(In) + readoffset = fieldoffset(In, i) + fT = fieldtype(In, i) + if ispacked(fT) + readsize = sizeof(fT) + _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, readsize) + writeoffset += readsize + else # nested padded type + fTpackedsize = packedsize(fT) + fpacked = Ref{NTuple{fTpackedsize, UInt8}}(reinterpret(NTuple{fTpackedsize, UInt8}, getfield(x, i))) + GC.@preserve fpacked begin + ptr_fpacked = unsafe_convert(Ptr{NTuple{fTpackedsize, UInt8}}, fpacked) + _memcpy!(ptr_out + writeoffset, ptr_fpacked, fTpackedsize) + end + writeoffset += fTpackedsize + end + end + end + function copyfrompacked(ptr_out, ptr_in) + readoffset = 0 + for i ∈ 1:fieldcount(Out) + writeoffset = fieldoffset(Out, i) + fT = fieldtype(Out, i) + if ispacked(fT) + writesize = sizeof(fT) + _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, writesize) + readoffset += writesize + else # nested padded type + fTpackedsize = packedsize(fT) + fpacked = Ref{NTuple{fTpackedsize, UInt8}}() + GC.@preserve fpacked begin + ptr_fpacked = unsafe_convert(Ptr{NTuple{fTpackedsize, UInt8}}, fpacked) + _memcpy!(ptr_fpacked, ptr_in + readoffset, fTpackedsize) + end + funpacked = Ref{fT}(reinterpret(fT, fpacked[])) + GC.@preserve funpacked begin + ptr_funpacked = unsafe_convert(Ptr{fT}, funpacked) + _memcpy!(ptr_out + writeoffset, ptr_funpacked, sizeof(fT)) + end + readoffset += fTpackedsize + end + end + end + + GC.@preserve in out begin + ptr_in = unsafe_convert(Ptr{In}, in) + ptr_out = unsafe_convert(Ptr{Out}, out) + + if fieldcount(In) > 0 && ispacked(Out) + copytopacked(ptr_out, ptr_in) + elseif fieldcount(Out) > 0 && ispacked(In) + copyfrompacked(ptr_out, ptr_in) + else + packed = Ref{NTuple{inpackedsize, UInt8}}() + GC.@preserve packed begin + ptr_packed = unsafe_convert(Ptr{NTuple{inpackedsize, UInt8}}, packed) + copytopacked(ptr_packed, ptr_in) + copyfrompacked(ptr_out, ptr_packed) + end + end + end + return out[] end end From d3ceb24eedc20c34adbe0ea5d164d332ba7fd7f4 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Sat, 13 Nov 2021 16:13:24 -0500 Subject: [PATCH 03/21] Fixed nested padding calc --- base/reinterpretarray.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 9a5483fc8f5fd..03e3881ae339e 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -678,7 +678,7 @@ end """ Compute the location of padding in a type. Recursive for nested types. """ -function padding(T, baseoffset = 0) +function padding(T, baseoffset = UInt(0)) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) @@ -690,8 +690,8 @@ function padding(T, baseoffset = 0) end last_end = offset + sizeof(fT) end - if 0 < last_end < sizeof(T) - push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end)) + if 0 < last_end - baseoffset < sizeof(T) + push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end - baseoffset)) end return pads end From 26a768a68cb8983da097e749bcd4b850decd7109 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Sat, 13 Nov 2021 16:26:43 -0500 Subject: [PATCH 04/21] Reorganize and remove allocations from nested --- base/reinterpretarray.jl | 88 +++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 51 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 03e3881ae339e..85839b122fb9b 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -691,7 +691,7 @@ function padding(T, baseoffset = UInt(0)) last_end = offset + sizeof(fT) end if 0 < last_end - baseoffset < sizeof(T) - push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end - baseoffset)) + push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end + baseoffset)) end return pads end @@ -781,6 +781,38 @@ end @pure ispacked(::Type{T}) where T = packedsize(T) == sizeof(T) +function _copytopacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} + writeoffset = 0 + for i ∈ 1:fieldcount(In) + readoffset = fieldoffset(In, i) + fT = fieldtype(In, i) + if ispacked(fT) + readsize = sizeof(fT) + _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, readsize) + writeoffset += readsize + else # nested padded type + _copytopacked!(ptr_out + writeoffset, Ptr{fT}(ptr_in + readoffset)) + writeoffset += packedsize(fT) + end + end +end + +function _copyfrompacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} + readoffset = 0 + for i ∈ 1:fieldcount(Out) + writeoffset = fieldoffset(Out, i) + fT = fieldtype(Out, i) + if ispacked(fT) + writesize = sizeof(fT) + _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, writesize) + readoffset += writesize + else # nested padded type + _copyfrompacked!(Ptr{fT}(ptr_out + writeoffset), ptr_in + readoffset) + readoffset += packedsize(fT) + end + end +end + @inline function reinterpret(::Type{Out}, x::In) where {Out, In} isbitstype(Out) || error("reinterpret target type must be isbits") isbitstype(In) || error("reinterpret source type must be isbits") @@ -800,66 +832,20 @@ end return out[] else # mismatched padding - function copytopacked(ptr_out, ptr_in) - writeoffset = 0 - for i ∈ 1:fieldcount(In) - readoffset = fieldoffset(In, i) - fT = fieldtype(In, i) - if ispacked(fT) - readsize = sizeof(fT) - _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, readsize) - writeoffset += readsize - else # nested padded type - fTpackedsize = packedsize(fT) - fpacked = Ref{NTuple{fTpackedsize, UInt8}}(reinterpret(NTuple{fTpackedsize, UInt8}, getfield(x, i))) - GC.@preserve fpacked begin - ptr_fpacked = unsafe_convert(Ptr{NTuple{fTpackedsize, UInt8}}, fpacked) - _memcpy!(ptr_out + writeoffset, ptr_fpacked, fTpackedsize) - end - writeoffset += fTpackedsize - end - end - end - function copyfrompacked(ptr_out, ptr_in) - readoffset = 0 - for i ∈ 1:fieldcount(Out) - writeoffset = fieldoffset(Out, i) - fT = fieldtype(Out, i) - if ispacked(fT) - writesize = sizeof(fT) - _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, writesize) - readoffset += writesize - else # nested padded type - fTpackedsize = packedsize(fT) - fpacked = Ref{NTuple{fTpackedsize, UInt8}}() - GC.@preserve fpacked begin - ptr_fpacked = unsafe_convert(Ptr{NTuple{fTpackedsize, UInt8}}, fpacked) - _memcpy!(ptr_fpacked, ptr_in + readoffset, fTpackedsize) - end - funpacked = Ref{fT}(reinterpret(fT, fpacked[])) - GC.@preserve funpacked begin - ptr_funpacked = unsafe_convert(Ptr{fT}, funpacked) - _memcpy!(ptr_out + writeoffset, ptr_funpacked, sizeof(fT)) - end - readoffset += fTpackedsize - end - end - end - GC.@preserve in out begin ptr_in = unsafe_convert(Ptr{In}, in) ptr_out = unsafe_convert(Ptr{Out}, out) if fieldcount(In) > 0 && ispacked(Out) - copytopacked(ptr_out, ptr_in) + _copytopacked!(ptr_out, ptr_in) elseif fieldcount(Out) > 0 && ispacked(In) - copyfrompacked(ptr_out, ptr_in) + _copyfrompacked!(ptr_out, ptr_in) else packed = Ref{NTuple{inpackedsize, UInt8}}() GC.@preserve packed begin ptr_packed = unsafe_convert(Ptr{NTuple{inpackedsize, UInt8}}, packed) - copytopacked(ptr_packed, ptr_in) - copyfrompacked(ptr_out, ptr_packed) + _copytopacked!(ptr_packed, ptr_in) + _copyfrompacked!(ptr_out, ptr_packed) end end end From 088b6dba9f072d68f503b511809bceada85ae58c Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Mon, 10 Oct 2022 14:52:52 -0400 Subject: [PATCH 05/21] Add `assume_effects` --- base/reinterpretarray.jl | 97 ++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 85839b122fb9b..bed7f6ce14920 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -678,7 +678,7 @@ end """ Compute the location of padding in a type. Recursive for nested types. """ -function padding(T, baseoffset = UInt(0)) +@assume_effects :foldable function padding(T, baseoffset = UInt(0)) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) @@ -727,59 +727,16 @@ using .Iterators: Stateful return true end -# Reductions with IndexSCartesian2 - -function _mapreduce(f::F, op::OP, style::IndexSCartesian2{K}, A::AbstractArrayOrBroadcasted) where {F,OP,K} - inds = eachindex(style, A) - n = size(inds)[2] - if n == 0 - return mapreduce_empty_iter(f, op, A, IteratorEltype(A)) - else - return mapreduce_impl(f, op, A, first(inds), last(inds)) - end -end - -@noinline function mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, - ifirst::SCI, ilast::SCI, blksize::Int) where {F,OP,SCI<:SCartesianIndex2{K}} where K - if ilast.j - ifirst.j < blksize - # sequential portion - @inbounds a1 = A[ifirst] - @inbounds a2 = A[SCI(2,ifirst.j)] - v = op(f(a1), f(a2)) - @simd for i = ifirst.i + 2 : K - @inbounds ai = A[SCI(i,ifirst.j)] - v = op(v, f(ai)) - end - # Remaining columns - for j = ifirst.j+1 : ilast.j - @simd for i = 1:K - @inbounds ai = A[SCI(i,j)] - v = op(v, f(ai)) - end - end - return v - else - # pairwise portion - jmid = ifirst.j + (ilast.j - ifirst.j) >> 1 - v1 = mapreduce_impl(f, op, A, ifirst, SCI(K,jmid), blksize) - v2 = mapreduce_impl(f, op, A, SCI(1,jmid+1), ilast, blksize) - return op(v1, v2) - end -end - -mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, ifirst::SCartesianIndex2, ilast::SCartesianIndex2) where {F,OP} = - mapreduce_impl(f, op, A, ifirst, ilast, pairwise_blocksize(f, op)) - -@pure function struct_subpadding(::Type{Out}, ::Type{In}) where {Out, In} +@assume_effects :foldable function struct_subpadding(::Type{Out}, ::Type{In}) where {Out, In} padding(Out) == padding(In) end -@pure function packedsize(::Type{T}) where T +@assume_effects :foldable function packedsize(::Type{T}) where T pads = padding(T) - return sizeof(T) - (isempty(pads) ? 0 : sum(p.size for p ∈ pads)) + return sizeof(T) - sum((p.size for p ∈ pads), init = 0) end -@pure ispacked(::Type{T}) where T = packedsize(T) == sizeof(T) +@assume_effects :foldable ispacked(::Type{T}) where T = packedsize(T) == sizeof(T) function _copytopacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} writeoffset = 0 @@ -852,3 +809,47 @@ end return out[] end end + + +# Reductions with IndexSCartesian2 + +function _mapreduce(f::F, op::OP, style::IndexSCartesian2{K}, A::AbstractArrayOrBroadcasted) where {F,OP,K} + inds = eachindex(style, A) + n = size(inds)[2] + if n == 0 + return mapreduce_empty_iter(f, op, A, IteratorEltype(A)) + else + return mapreduce_impl(f, op, A, first(inds), last(inds)) + end +end + +@noinline function mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, + ifirst::SCI, ilast::SCI, blksize::Int) where {F,OP,SCI<:SCartesianIndex2{K}} where K + if ilast.j - ifirst.j < blksize + # sequential portion + @inbounds a1 = A[ifirst] + @inbounds a2 = A[SCI(2,ifirst.j)] + v = op(f(a1), f(a2)) + @simd for i = ifirst.i + 2 : K + @inbounds ai = A[SCI(i,ifirst.j)] + v = op(v, f(ai)) + end + # Remaining columns + for j = ifirst.j+1 : ilast.j + @simd for i = 1:K + @inbounds ai = A[SCI(i,j)] + v = op(v, f(ai)) + end + end + return v + else + # pairwise portion + jmid = ifirst.j + (ilast.j - ifirst.j) >> 1 + v1 = mapreduce_impl(f, op, A, ifirst, SCI(K,jmid), blksize) + v2 = mapreduce_impl(f, op, A, SCI(1,jmid+1), ilast, blksize) + return op(v1, v2) + end +end + +mapreduce_impl(f::F, op::OP, A::AbstractArrayOrBroadcasted, ifirst::SCartesianIndex2, ilast::SCartesianIndex2) where {F,OP} = + mapreduce_impl(f, op, A, ifirst, ilast, pairwise_blocksize(f, op)) From 1016e8c14d17a2bd98f5efce33f54e9b40f6fffe Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Wed, 12 Oct 2022 15:50:30 -0400 Subject: [PATCH 06/21] Add tests, fix tests, change to ArgumentError --- base/reinterpretarray.jl | 2 +- test/numbers.jl | 6 ++---- test/reinterpretarray.jl | 26 ++++++++++++++++++++++++-- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index bed7f6ce14920..a973cfe5efd57 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -776,7 +776,7 @@ end isprimitivetype(Out) && isprimitivetype(In) && return bitcast(Out, x) inpackedsize = packedsize(In) outpackedsize = packedsize(Out) - inpackedsize == outpackedsize || throw(PaddingError(Out, In)) + inpackedsize == outpackedsize || throw(ArgumentError("Packed sizes of types $Out and $In do not match; got $outpackedsize and $inpackedsize, respectively.")) in = Ref{In}(x) out = Ref{Out}() if struct_subpadding(Out, In) diff --git a/test/numbers.jl b/test/numbers.jl index 926abf85b246d..b3dec8e2bf266 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2219,10 +2219,8 @@ end @test_throws ErrorException reinterpret(Int, 0x01) @testset "issue #12832" begin - @test_throws ErrorException reinterpret(Float64, Complex{Int64}(1)) - @test_throws ErrorException reinterpret(Float64, ComplexF32(1)) - @test_throws ErrorException reinterpret(ComplexF32, Float64(1)) - @test_throws ErrorException reinterpret(Int32, false) + @test_throws PaddingException reinterpret(Float64, Complex{Int64}(1)) + @test_throws PaddingException reinterpret(Int32, false) end # issue #41 ndigf(n) = Float64(log(Float32(n))) diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index e9803b9bd9ceb..eeff8025edc20 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -450,8 +450,8 @@ end SomeSingleton(x) = new() end - @test_throws ErrorException reinterpret(Int, nothing) - @test_throws ErrorException reinterpret(Missing, 3) + @test_throws ArgumentError reinterpret(Int, nothing) + @test_throws ArgumentError reinterpret(Missing, 3) @test_throws ErrorException reinterpret(Missing, NotASingleton()) @test_throws ErrorException reinterpret(NotASingleton, ()) @@ -513,3 +513,25 @@ end @test setindex!(x, SomeSingleton(:), 3, 5) == x2 @test_throws MethodError x[2,4] = nothing end + +# reinterpret of arbitrary bitstypes +@testset "Reinterpret arbitrary bitstypes" begin + struct Bytes15 + a::Int8 + b::Int16 + c::Int32 + d::Int64 + end + + @test reinterpret(Float64, ComplexF32(1, 1)) === 0.007812501848093234 + @test reinterpret(ComplexF32, 0.007812501848093234) === ComplexF32(1, 1) + @test reinterpret(Tuple{Float64, Float64}, ComplexF64(1, 1)) === (1.0, 1.0) + @test reinterpret(ComplexF64, (1.0, 1.0)) === ComplexF64(1, 1) + @test reinterpret(Tuple{Int8, Int16, Int32, Int64}, (Int64(1), Int32(2), Int16(3), Int8(4))) === (Int8(1), Int16(0), Int32(0), 288233674686595584) + @test reinterpret(Tuple{Int8, Int16, Tuple{Int32, Int64}}, (Int64(1), Int32(2), Int16(3), Int8(4))) === (Int8(1), Int16(0), (Int32(0), 288233674686595584)) + @test reinterpret(Tuple{Int64, Int32, Int16, Int8}, (Int8(1), Int16(0), (Int32(0), 288233674686595584))) === (Int64(1), Int32(2), Int16(3), Int8(4)) + @test reinterpret(Tuple{Int8, Int16, Int32, Int64}, Bytes15(Int8(1), Int16(2), Int32(3), Int64(4))) === (Int8(1), Int16(2), Int32(3), Int64(4)) + @test reinterpret(Bytes15, (Int8(1), Int16(2), Int32(3), Int64(4))) == Bytes15(Int8(1), Int16(2), Int32(3), Int64(4)) + + @test_throws ArgumentError reinterpret(Tuple{Int32, Int64}, (Int16(1), Int64(4))) +end From 26ccc199b56c1990ee6344c13408b89621d62fa0 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 10:27:40 -0400 Subject: [PATCH 07/21] Export PaddingException --- base/exports.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/base/exports.jl b/base/exports.jl index 266a4aa8038fb..e3127a60aa4cc 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -134,6 +134,7 @@ export InvalidStateException, KeyError, MissingException, + PaddingException, ProcessFailedException, TaskFailedException, SystemError, From fe923ec8b3fdcc5915ca2b440ee50da9a5853df0 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 14:29:37 -0400 Subject: [PATCH 08/21] Replace calls to `_memcpy!` with `memcpy` to reflect #49550 --- base/reinterpretarray.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 8dd24b0e7f797..f9793523e24da 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -770,7 +770,7 @@ function _copytopacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} fT = fieldtype(In, i) if ispacked(fT) readsize = sizeof(fT) - _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, readsize) + memcpy(ptr_out + writeoffset, ptr_in + readoffset, readsize) writeoffset += readsize else # nested padded type _copytopacked!(ptr_out + writeoffset, Ptr{fT}(ptr_in + readoffset)) @@ -786,7 +786,7 @@ function _copyfrompacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} fT = fieldtype(Out, i) if ispacked(fT) writesize = sizeof(fT) - _memcpy!(ptr_out + writeoffset, ptr_in + readoffset, writesize) + memcpy(ptr_out + writeoffset, ptr_in + readoffset, writesize) readoffset += writesize else # nested padded type _copyfrompacked!(Ptr{fT}(ptr_out + writeoffset), ptr_in + readoffset) @@ -809,7 +809,7 @@ end GC.@preserve in out begin ptr_in = unsafe_convert(Ptr{In}, in) ptr_out = unsafe_convert(Ptr{Out}, out) - _memcpy!(ptr_out, ptr_in, sizeof(Out)) + memcpy(ptr_out, ptr_in, sizeof(Out)) end return out[] else From c2a765a890450d00013528120a34f1c13fa81a3c Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 14:33:21 -0400 Subject: [PATCH 09/21] Return SimpleVector to ensure the function really is `:foldable` Co-authored-by: Jameson Nash --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index f9793523e24da..56727c01fdda9 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -713,7 +713,7 @@ end if 0 < last_end - baseoffset < sizeof(T) push!(pads, Padding(baseoffset + sizeof(T), sizeof(T) - last_end + baseoffset)) end - return pads + return Core.svec(pads...) end function CyclePadding(T::DataType) From 5fb9c1e0c0656f50f38f497ec6cbb23a9522a8f0 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 14:34:19 -0400 Subject: [PATCH 10/21] Refine docstring Co-authored-by: Jameson Nash --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 56727c01fdda9..560239b4f4fd0 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -696,7 +696,7 @@ function iterate(cp::CyclePadding, state::Tuple) end """ - Compute the location of padding in a type. Recursive for nested types. + Compute the location of padding in an isbits datatype. Recursive over the fields of that type. """ @assume_effects :foldable function padding(T, baseoffset = UInt(0)) pads = Padding[] From f7af68d05c2f0bef6010c9e9fd45c678a8eb7f8f Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 14:34:53 -0400 Subject: [PATCH 11/21] Narrow signature to proper types Co-authored-by: Jameson Nash --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 560239b4f4fd0..d77344074eadd 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -698,7 +698,7 @@ end """ Compute the location of padding in an isbits datatype. Recursive over the fields of that type. """ -@assume_effects :foldable function padding(T, baseoffset = UInt(0)) +@assume_effects :foldable function padding(T::DataType, baseoffset::Int = 0) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) From 122c82be2b1d5ad7470e91d8cde00157de533c73 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 14:35:50 -0400 Subject: [PATCH 12/21] Simplify packing test Co-authored-by: Jameson Nash --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index d77344074eadd..4624e3f880bb8 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -761,7 +761,7 @@ end return sizeof(T) - sum((p.size for p ∈ pads), init = 0) end -@assume_effects :foldable ispacked(::Type{T}) where T = packedsize(T) == sizeof(T) +@assume_effects :foldable ispacked(::Type{T}) where T = isempty(padding(T)) function _copytopacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} writeoffset = 0 From ecfb1387d7cb4759d5088456a8fed122310fed1b Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 15:55:22 -0400 Subject: [PATCH 13/21] Back to UInt - needed because fieldoffset returns UInt --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 4624e3f880bb8..415cad0796ae2 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -698,7 +698,7 @@ end """ Compute the location of padding in an isbits datatype. Recursive over the fields of that type. """ -@assume_effects :foldable function padding(T::DataType, baseoffset::Int = 0) +@assume_effects :foldable function padding(T::DataType, baseoffset::UInt = 0) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) From 706fccb6d76ad9b703f754df79cf274c2483d4dd Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 16:14:45 -0400 Subject: [PATCH 14/21] Back to UInt --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 415cad0796ae2..7e0dbd12a3d54 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -698,7 +698,7 @@ end """ Compute the location of padding in an isbits datatype. Recursive over the fields of that type. """ -@assume_effects :foldable function padding(T::DataType, baseoffset::UInt = 0) +@assume_effects :foldable function padding(T::DataType, baseoffset::UInt = 0x0) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) From 9dee0c64e731589a3cc2114062e8256efb53b409 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 16:40:03 -0400 Subject: [PATCH 15/21] UInt(0) specifically --- base/reinterpretarray.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 7e0dbd12a3d54..2530ac8bef150 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -698,7 +698,7 @@ end """ Compute the location of padding in an isbits datatype. Recursive over the fields of that type. """ -@assume_effects :foldable function padding(T::DataType, baseoffset::UInt = 0x0) +@assume_effects :foldable function padding(T::DataType, baseoffset::UInt = UInt(0)) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) From 8b35d55733b89b336f972b77e5c28e6a5b833714 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 22:33:43 -0400 Subject: [PATCH 16/21] Error, not Exception --- base/exports.jl | 2 +- test/numbers.jl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 07d8958a43757..6f80abf5afefc 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -134,7 +134,7 @@ export InvalidStateException, KeyError, MissingException, - PaddingException, + PaddingError, ProcessFailedException, TaskFailedException, SystemError, diff --git a/test/numbers.jl b/test/numbers.jl index 0d3617241b4dd..a4ea05adf5da1 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2219,8 +2219,8 @@ end @test_throws ErrorException reinterpret(Int, 0x01) @testset "issue #12832" begin - @test_throws PaddingException reinterpret(Float64, Complex{Int64}(1)) - @test_throws PaddingException reinterpret(Int32, false) + @test_throws PaddingError reinterpret(Float64, Complex{Int64}(1)) + @test_throws PaddingError reinterpret(Int32, false) end # issue #41 ndigf(n) = Float64(log(Float32(n))) From d74d1536a25e86fbe59051b18481df16c13576c4 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 22:41:06 -0400 Subject: [PATCH 17/21] More specific exception Co-authored-by: Sukera <11753998+Seelengrab@users.noreply.github.com> --- base/reinterpretarray.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 2530ac8bef150..161e84af63060 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -796,8 +796,8 @@ function _copyfrompacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} end @inline function reinterpret(::Type{Out}, x::In) where {Out, In} - isbitstype(Out) || error("reinterpret target type must be isbits") - isbitstype(In) || error("reinterpret source type must be isbits") + isbitstype(Out) || throw(ArgumentError("Target type for `reinterpret` must be isbits")) + isbitstype(In) || throw(ArgumentError("Source type for `reinterpret` must be isbits")) isprimitivetype(Out) && isprimitivetype(In) && return bitcast(Out, x) inpackedsize = packedsize(In) outpackedsize = packedsize(Out) From de7012713ed8b639c240193337efe518b24f43b2 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Thu, 22 Jun 2023 23:22:00 -0400 Subject: [PATCH 18/21] Okay actually ArgumentError --- base/reinterpretarray.jl | 13 +++++++++++-- test/numbers.jl | 4 ++-- test/reinterpretarray.jl | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index 161e84af63060..f51116a029a59 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -798,10 +798,19 @@ end @inline function reinterpret(::Type{Out}, x::In) where {Out, In} isbitstype(Out) || throw(ArgumentError("Target type for `reinterpret` must be isbits")) isbitstype(In) || throw(ArgumentError("Source type for `reinterpret` must be isbits")) - isprimitivetype(Out) && isprimitivetype(In) && return bitcast(Out, x) + if isprimitivetype(Out) && isprimitivetype(In) + outsize = sizeof(Out) + insize = sizeof(In) + outsize == insize || + throw(ArgumentError("Sizes of types $Out and $In do not match; got $outsize \ + and $insize, respectively.")) + return bitcast(Out, x) + end inpackedsize = packedsize(In) outpackedsize = packedsize(Out) - inpackedsize == outpackedsize || throw(ArgumentError("Packed sizes of types $Out and $In do not match; got $outpackedsize and $inpackedsize, respectively.")) + inpackedsize == outpackedsize || + throw(ArgumentError("Packed sizes of types $Out and $In do not match; got $outpackedsize \ + and $inpackedsize, respectively.")) in = Ref{In}(x) out = Ref{Out}() if struct_subpadding(Out, In) diff --git a/test/numbers.jl b/test/numbers.jl index a4ea05adf5da1..718690f2e067a 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2219,8 +2219,8 @@ end @test_throws ErrorException reinterpret(Int, 0x01) @testset "issue #12832" begin - @test_throws PaddingError reinterpret(Float64, Complex{Int64}(1)) - @test_throws PaddingError reinterpret(Int32, false) + @test_throws ArgumentError reinterpret(Float64, Complex{Int64}(1)) + @test_throws ArgumentError reinterpret(Int32, false) end # issue #41 ndigf(n) = Float64(log(Float32(n))) diff --git a/test/reinterpretarray.jl b/test/reinterpretarray.jl index 676df1be83d69..501e9f4a9b57f 100644 --- a/test/reinterpretarray.jl +++ b/test/reinterpretarray.jl @@ -452,8 +452,8 @@ end @test_throws ArgumentError reinterpret(Int, nothing) @test_throws ArgumentError reinterpret(Missing, 3) - @test_throws ErrorException reinterpret(Missing, NotASingleton()) - @test_throws ErrorException reinterpret(NotASingleton, ()) + @test_throws ArgumentError reinterpret(Missing, NotASingleton()) + @test_throws ArgumentError reinterpret(NotASingleton, ()) @test_throws ArgumentError reinterpret(NotASingleton, fill(nothing, ())) @test_throws ArgumentError reinterpret(reshape, NotASingleton, fill(missing, 3)) From 2cad1b9696e2e6f20d8cf3cf4c8a2abd79a1c224 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Fri, 23 Jun 2023 00:16:37 -0400 Subject: [PATCH 19/21] Fix tests --- test/core.jl | 2 +- test/numbers.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core.jl b/test/core.jl index f71baa843d25f..195cb5f71369f 100644 --- a/test/core.jl +++ b/test/core.jl @@ -1897,7 +1897,7 @@ function f4528(A, B) end end @test f4528(false, Int32(12)) === nothing -@test_throws ErrorException f4528(true, Int32(12)) +@test_throws ArgumentError f4528(true, Int32(12)) # issue #4518 f4518(x, y::Union{Int32,Int64}) = 0 diff --git a/test/numbers.jl b/test/numbers.jl index 718690f2e067a..d7fd6531b157d 100644 --- a/test/numbers.jl +++ b/test/numbers.jl @@ -2216,7 +2216,7 @@ end @test round(Int16, -32768.1) === Int16(-32768) end # issue #7508 -@test_throws ErrorException reinterpret(Int, 0x01) +@test_throws ArgumentError reinterpret(Int, 0x01) @testset "issue #12832" begin @test_throws ArgumentError reinterpret(Float64, Complex{Int64}(1)) From 4e4d13b5d020109d1535ead78068c02ee3c089f2 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Fri, 23 Jun 2023 10:23:19 -0400 Subject: [PATCH 20/21] Add docstring --- base/reinterpretarray.jl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index f51116a029a59..eafa19ea5e50f 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -795,6 +795,31 @@ function _copyfrompacked!(ptr_out::Ptr{Out}, ptr_in::Ptr{In}) where {Out, In} end end +""" + reinterpret(::Type{Out}, x::In) + +Reinterpret the valid non-padding bytes of an isbits value `x` as isbits type `Out`. + +Both types must have the same amount of non-padding bytes. This operation is guaranteed +to be reversible. + +```jldoctest +julia> reinterpret(NTuple{2, UInt8}, 0x1234) +(0x34, 0x12) + +julia> reinterpret(UInt16, (0x34, 0x12)) +0x1234 + +julia> reinterpret(Tuple{UInt16, UInt8}, (0x01, 0x0203)) +(0x0301, 0x02) +``` + +!!! warning + + Use caution if some combinations of bits in `Out` are not considered valid and would + otherwise be prevented by the type's constructors and methods. Unexpected behavior + may result without additional validation. +""" @inline function reinterpret(::Type{Out}, x::In) where {Out, In} isbitstype(Out) || throw(ArgumentError("Target type for `reinterpret` must be isbits")) isbitstype(In) || throw(ArgumentError("Source type for `reinterpret` must be isbits")) From e1d24134cd875f450495a0849714bff745ec6e60 Mon Sep 17 00:00:00 2001 From: Nicholas Bauer Date: Wed, 28 Jun 2023 19:27:04 -0400 Subject: [PATCH 21/21] Remove export, back to `Int` as better API Co-authored-by: Jameson Nash --- base/exports.jl | 1 - base/reinterpretarray.jl | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/base/exports.jl b/base/exports.jl index 6f80abf5afefc..10f43825e12df 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -134,7 +134,6 @@ export InvalidStateException, KeyError, MissingException, - PaddingError, ProcessFailedException, TaskFailedException, SystemError, diff --git a/base/reinterpretarray.jl b/base/reinterpretarray.jl index eafa19ea5e50f..d33c127b78c76 100644 --- a/base/reinterpretarray.jl +++ b/base/reinterpretarray.jl @@ -698,11 +698,11 @@ end """ Compute the location of padding in an isbits datatype. Recursive over the fields of that type. """ -@assume_effects :foldable function padding(T::DataType, baseoffset::UInt = UInt(0)) +@assume_effects :foldable function padding(T::DataType, baseoffset::Int = 0) pads = Padding[] last_end::Int = baseoffset for i = 1:fieldcount(T) - offset = baseoffset + fieldoffset(T, i) + offset = baseoffset + Int(fieldoffset(T, i)) fT = fieldtype(T, i) append!(pads, padding(fT, offset)) if offset != last_end