Add rectifiers

JuliaSIMD · Feb 22, 2022 · 15ab20f · 15ab20f · chriselrod · Feb 22, 2022
1 parent 3ff786a
commit 15ab20f
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "SLEEFPirates"
 uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa"
 authors = ["chriselrod <elrodc@gmail.com>"]
-version = "0.6.29"
+version = "0.6.30"
 
 [deps]
 IfElse = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173"

diff --git a/src/SLEEFPirates.jl b/src/SLEEFPirates.jl
@@ -12,7 +12,9 @@ using VectorizationBase: vzero, AbstractSIMD, _Vec, fma_fast, data, VecUnroll, N
 
 import IfElse: ifelse
 
-export Vec, sigmoid_fast, tanh_fast#, loggamma
+export Vec, sigmoid_fast, tanh_fast,
+  PReLu, gelu, softplus, silu, Elu
+  #, loggamma
 
 const FloatType64 = Union{Float64,AbstractSIMD{<:Any,Float64}}
 const FloatType32 = Union{Float32,AbstractSIMD{<:Any,Float32}}
@@ -118,6 +120,7 @@ include("log.jl")    # logarithmic functions
 include("trig.jl")   # trigonometric and inverse trigonometric functions
 include("hyp.jl")    # hyperbolic and inverse hyperbolic functions
 include("misc.jl")   # miscallenous math functions including pow and cbrt
+include("rectifier.jl")
 # if Int === Int64
 #     if isfile(joinpath(@__DIR__, "svmlwrap.jl"))
 #         include("svmlwrap.jl")

diff --git a/src/rectifier.jl b/src/rectifier.jl
@@ -0,0 +1,13 @@
+
+struct PReLu{T}; a::T; end
+@inline (p::PReLu)(x) = ifelse(x <= 0, x * p.a, x)
+
+@inline Φ(x::T) where {T} = @fastmath T(0.5) * (one(T) + VectorizationBase.verf(x * T(0.7071067811865476)))
+@inline gelu(x) = Base.FastMath.mul_fast(x, Φ(x))
+
+@inline softplus(x) = log1p(exp(x))
+@inline silu(x) = Base.FastMath.mul_fast(x, sigmoid_fast(x))
+
+struct Elu{T}; a::T; end
+@inline (e::Elu)(x) = ifelse(x <= 0, Base.FastMath.mul_fast(e.a, expm1_fast(x)), x)
+