Readme example with Flux no longer works

[DrChainsaw]

I realize this package is half archived, but in case someone happens to see this and knows an easy fix.

Looks like it should "just" be a matter of implementing the Float32 conversion, although I don't know if it should be a noop or return the tracked float.

julia> versioninfo()
Julia Version 1.10.0-beta1
Commit 6616549950 (2023-07-25 17:43 UTC)
Platform Info:
  OS: Windows (x86_64-w64-mingw32)
  CPU: 12 × Intel(R) Core(TM) i7-5820K CPU @ 3.30GHz
  WORD_SIZE: 64
  LIBM: libopenlibm
  LLVM: libLLVM-15.0.7 (ORCJIT, haswell)
  Threads: 1 on 12 virtual cores

(jl_a6AlLv) pkg> status
Status `E:\Temp\systmp\jl_a6AlLv\Project.toml`
  [587475ba] Flux v0.14.1
  [9f7883ad] Tracker v0.2.26

julia> using Flux, Tracker

julia> model = Chain(Dense(2 => 1, tanh), Dense(1 => 1, bias=false));

julia> withgradient(model, rand(Float32, 2)) do m, x
                sum(abs2, m(x))
           end
ERROR: MethodError: no method matching Float32(::Tracker.TrackedReal{Float32})

Closest candidates are:
  (::Type{T})(::Real, ::RoundingMode) where T<:AbstractFloat
   @ Base rounding.jl:207
  (::Type{T})(::T) where T<:Number
   @ Core boot.jl:790
  Float32(::IrrationalConstants.Invsqrt2π)
   @ IrrationalConstants E:\Programs\julia\.julia\packages\IrrationalConstants\vp5v4\src\macro.jl:113
  ...

Stacktrace:
  [1] convert(::Type{Float32}, x::Tracker.TrackedReal{Float32})
    @ Base .\number.jl:7
  [2] setindex!
    @ Tracker .\array.jl:1022 [inlined]
  [3] setindex!
    @ Tracker .\multidimensional.jl:698 [inlined]
  [4] macro expansion
    @ Tracker .\broadcast.jl:1004 [inlined]
  [5] macro expansion
    @ Tracker .\simdloop.jl:77 [inlined]
  [6] copyto!
    @ Tracker .\broadcast.jl:1003 [inlined]
  [7] copyto!
    @ Tracker .\broadcast.jl:956 [inlined]
  [8] materialize!
    @ Tracker .\broadcast.jl:914 [inlined]
  [9] materialize!
    @ Tracker .\broadcast.jl:911 [inlined]
 [10] accum!(x::Matrix{Float32}, Δ::Matrix{Tracker.TrackedReal{Float32}})
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:45
 [11] back(x::Tracker.Tracked{Matrix{Float32}}, Δ::Matrix{Tracker.TrackedReal{Float32}}, once::Bool)
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:48
 [12] #678
    @ Base E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:38 [inlined]
 [13] #64
    @ Base .\tuple.jl:628 [inlined]
 [14] BottomRF
    @ Base .\reduce.jl:86 [inlined]
 [15] _foldl_impl(op::Base.BottomRF{Base.var"#64#65"{Tracker.var"#678#679"{Bool}}}, init::Nothing, itr::Base.Iterators.Zip{Tuple{Tuple{Tracker.Tracked{…}, Tracker.Tracked{…}}, Tuple{Matrix{…}, Vector{…}}}})
    @ Base .\reduce.jl:58
 [16] foldl_impl
    @ Base .\reduce.jl:48 [inlined]
 [17] mapfoldl_impl
    @ Base .\reduce.jl:44 [inlined]
 [18] mapfoldl
    @ Base .\reduce.jl:175 [inlined]
 [19] foldl
    @ Base .\reduce.jl:198 [inlined]
 [20] foreach
    @ Base .\tuple.jl:628 [inlined]
 [21] back_(c::Tracker.Call{Tracker.var"#631#632"{TrackedArray{…,Matrix{…}}, TrackedArray{…,Vector{…}}}, Tuple{Tracker.Tracked{Matrix{…}}, Tracker.Tracked{Vector{…}}}}, Δ::Vector{Tracker.TrackedReal{Float32}}, once::Bool)
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:38
 [22] back(x::Tracker.Tracked{Vector{Tracker.TrackedReal{Float32}}}, Δ::Vector{Tracker.TrackedReal{Float32}}, once::Bool)
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:58
 [23] #678
    @ Base E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:38 [inlined]
 [24] #64
    @ Base .\tuple.jl:628 [inlined]
 [25] BottomRF
    @ Base .\reduce.jl:86 [inlined]
 [26] _foldl_impl(op::Base.BottomRF{Base.var"#64#65"{Tracker.var"#678#679"{Bool}}}, init::Nothing, itr::Base.Iterators.Zip{Tuple{Tuple{Tracker.Tracked{…}, Nothing}, Tuple{Vector{…}, Tracker.TrackedReal{…}}}})        
    @ Base .\reduce.jl:58
 [27] foldl_impl
    @ Base .\reduce.jl:48 [inlined]
 [28] mapfoldl_impl(f::typeof(identity), op::Base.var"#64#65"{Tracker.var"#678#679"{Bool}}, nt::Nothing, itr::Base.Iterators.Zip{Tuple{Tuple{Tracker.Tracked{…}, Nothing}, Tuple{Vector{…}, Tracker.TrackedReal{…}}}})  
    @ Base .\reduce.jl:44
 [29] mapfoldl(f::Function, op::Function, itr::Base.Iterators.Zip{Tuple{Tuple{Tracker.Tracked{Vector{…}}, Nothing}, Tuple{Vector{Tracker.TrackedReal{…}}, Tracker.TrackedReal{Float32}}}}; init::Nothing)
    @ Base .\reduce.jl:175
 [30] mapfoldl
    @ Base .\reduce.jl:175 [inlined]
 [31] foldl
    @ Base .\reduce.jl:198 [inlined]
 [32] foreach(::Function, ::Tuple{Tracker.Tracked{Vector{Tracker.TrackedReal{Float32}}}, Nothing}, ::Tuple{Vector{Tracker.TrackedReal{Float32}}, Tracker.TrackedReal{Float32}})
    @ Base .\tuple.jl:628
 [33] back_(c::Tracker.Call{Tracker.var"#back#674"{2, Base.Broadcast.var"#12#14"{…}, Tuple{…}}, Tuple{Tracker.Tracked{…}, Nothing}}, Δ::Vector{Tracker.TrackedReal{Float32}}, once::Bool)
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:38
 [34] back(x::Tracker.Tracked{Vector{Tracker.TrackedReal{Float32}}}, Δ::Vector{Tracker.TrackedReal{Float32}}, once::Bool)
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:58
 [35] #678
    @ Base E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:38 [inlined]
 [36] #64
    @ Base .\tuple.jl:628 [inlined]
 [37] BottomRF
    @ Base .\reduce.jl:86 [inlined]
 [38] _foldl_impl(op::Base.BottomRF{Base.var"#64#65"{Tracker.var"#678#679"{Bool}}}, init::Nothing, itr::Base.Iterators.Zip{Tuple{Tuple{Tracker.Tracked{…}, Tracker.Tracked{…}}, Tuple{Matrix{…}, Vector{…}}}})
    @ Base .\reduce.jl:58 [inlined]
 [39] foldl_impl
    @ Tracker .\reduce.jl:48 [inlined]
 [40] mapfoldl_impl
    @ Tracker .\reduce.jl:44 [inlined]
 [41] mapfoldl
    @ Tracker .\reduce.jl:175 [inlined]
 [42] foldl
    @ Tracker .\reduce.jl:198 [inlined]
 [43] foreach
    @ Tracker .\tuple.jl:628 [inlined]
--- the last 11 lines are repeated 1 more time ---
 [55] back_(c::Tracker.Call{Tracker.var"#599#600"{TrackedArray{…,Vector{Tracker.TrackedReal{Float32}}}}, Tuple{Tracker.Tracked{Vector{Tracker.TrackedReal{Float32}}}}}, Δ::Tracker.TrackedReal{Float32}, once::Bool)    
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:38
 [56] back(x::Tracker.Tracked{Tracker.TrackedReal{Float32}}, Δ::Int64, once::Bool)
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:58
 [57] #back!#680
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:77 [inlined]
 [58] back!
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:74 [inlined]
 [59] #back!#13
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\lib\real.jl:18 [inlined]
 [60] back!(x::Tracker.TrackedReal{Tracker.TrackedReal{Float32}})
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\lib\real.jl:15
 [61] withgradient(::Function, ::Chain{Tuple{Dense{typeof(tanh), Matrix{Float32}, Vector{Float32}}, Dense{typeof(identity), Matrix{Float32}, Bool}}}, ::Vararg{Any})
    @ Tracker E:\Programs\julia\.julia\packages\Tracker\tdXqL\src\back.jl:4

[mcabbott]

This is a side-effect of FluxML/Flux.jl#2156, apparently there were zero tests. It never got an issue, nor an attempt to fix.

I think simplest would be a method _match_eltype(layer, ::Type{T}, x::TrackedArray) where {T}, using Requires/extensions either here or (better) in Flux. But maybe there's another way, without that hassle.

[mcabbott]

Not sure that diagnosis is correct.

Here's a related easier problem, where _match_eltype should do something, and tries:

julia> withgradient(model, rand(2)) do m, x  # now x is Vector{Float64}, _match_eltype tries to convert, and fails
         sum(abs2, m(x))
       end
ERROR: Not implemented: convert tracked Float64 to tracked Float32
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:35
  [2] convert(::Type{Tracker.TrackedReal{Float32}}, x::Tracker.TrackedReal{Float64})
    @ Tracker ~/.julia/packages/Tracker/tdXqL/src/lib/real.jl:41
  [3] setindex!(A::Vector{Tracker.TrackedReal{Float32}}, x::Tracker.TrackedReal{Float64}, i1::Int64)
    @ Base ./array.jl:1019
  [4] copyto_unaliased!(deststyle::IndexStyle, dest::AbstractArray, srcstyle::IndexStyle, src::AbstractArray)
    @ Base ./abstractarray.jl:1089 [inlined]
  [5] copyto!(dest::Vector{Tracker.TrackedReal{Float32}}, src::TrackedArray{…,Vector{Float64}})
    @ Base ./abstractarray.jl:1063
  [6] copyto_axcheck!(dest::Any, src::Any)
    @ Base ./abstractarray.jl:1172 [inlined]
  [7] AbstractArray{T, N}(A::AbstractArray{S, N}) where {T, N, S}
    @ Base ./array.jl:672 [inlined]
  [8] (AbstractArray{T})(A::AbstractArray{S, N}) where {T, S, N}
    @ Core ./boot.jl:501 [inlined]
  [9] convert
    @ Base ./abstractarray.jl:17 [inlined]
 [10] _match_eltype
    @ Flux ~/.julia/packages/Flux/r3RWW/src/layers/stateless.jl:77 [inlined]
 [11] _match_eltype
    @ Flux ~/.julia/packages/Flux/r3RWW/src/layers/stateless.jl:84 [inlined]
 [12] (::Dense{typeof(tanh), TrackedArray{…,Matrix{…}}, Vector{Tracker.TrackedReal{…}}})(x::TrackedArray{…,Vector{Float64}})

For the one above, notice [22] back(x::Tracker.Tracked{Vector{Tracker.TrackedReal{Float32}}}, Δ::Vector{Tracker.TrackedReal{Float32}}, once::Bool). Such an x is what you get when Array{TrackedReal} interacts badly with TrackedArray{Array{Float32}}. A possible source of Array{TrackedReal} is this:

julia> using Tracker, Flux

julia> x = param(rand32(2))
Tracked 2-element Vector{Float32}:
 0.066055834f0
 0.49695432f0

julia> m = Dense(param(randn32(3,2)))
Dense(2 => 3)       # 9 parameters

julia> m(x)
Tracked 3-element Vector{Tracker.TrackedReal{Float32}}:
 0.17689592f0
 0.32748032f0
 0.5916688f0

But fixing that (by adjusting _match_eltype) leads me to [11] back(x::Tracker.Tracked{Matrix{Float32}}, Δ::Matrix{Tracker.TrackedReal{Float32}}, once::Bool) which is still not good.