Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Fix SoftReLU fused operator numerical stability #17849

Merged
merged 2 commits into from
Mar 18, 2020
Merged

Fix SoftReLU fused operator numerical stability #17849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
be1a6e2
fix numerically unstable fused softrelu op
RuRo Mar 16, 2020
31fae17
implement test for softrelu numerical stability
RuRo Mar 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/operator/fusion/fused_op-inl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,7 +550,10 @@ __device__ inline DType sigmoid(const DType val) {

template <typename DType>
__device__ inline DType softrelu(const DType val) {
return logf(1 + expf(val));
// Avoid overflow of exp for large inputs.
// The threshold 20 is chosen such that softrelu(a) = a
// for a > 20 using floating precision.
return val > 20 ? val : logf(1 + expf(val));
szha marked this conversation as resolved.
Show resolved Hide resolved
}

template <typename DType>
Expand Down
3 changes: 3 additions & 0 deletions tests/python/gpu/test_fusion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,9 @@ def announce_check(op_name):
for act_type in ['relu', 'sigmoid', 'tanh', 'softrelu', 'softsign']:
announce_check("Activation(act_type='{}')".format(act_type))
check_fused_symbol(mx.sym.Activation(a, act_type=act_type), a=arr)
if act_type == 'softrelu':
# Check that softrelu implementation doesn't overflow on large inputs
check_fused_symbol(mx.sym.Activation(a, act_type=act_type), a=1000 * arr)

# Cast requires dtype
for dtype in ['float16', 'float32', 'float64', 'int32']:
Expand Down