From 252e8d9741933b8ce4d2f2abb604673ce7d7590a Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Thu, 21 Nov 2024 19:38:25 -0600 Subject: [PATCH 01/16] Add NN activation functions in shared.h --- inst/include/rxode2_model_shared.h | 84 ++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/inst/include/rxode2_model_shared.h b/inst/include/rxode2_model_shared.h index f20f921a6..75a472fbf 100644 --- a/inst/include/rxode2_model_shared.h +++ b/inst/include/rxode2_model_shared.h @@ -18,6 +18,90 @@ static inline double _safe_log_(double a, rx_solve *rx) { return log(a); } } + +static inline double ReLU(double x) { + return (x > 0.0) ? x : 0.0; +} + +static inline double dReLU(double x) { + return (x > 0.0) ? 1.0 : 0.0; +} + +// +static inline double GELU(double x) { + return 0.5 * x * (1.0 + erf(x * M_SQRT1_2)); +} + +static inline double dGELU(double x) { + return 0.5 * (1.0 + erf(x * M_SQRT1_2)) + x * M_1_SQRT_2PI * exp(-0.5 * x * x); +} + +static inline double ELU(double x, double alpha) { + return (x > 0.0) ? x : (exp(x) - 1.0) * alpha; +} + +static inline double dELU(double x, double alpha) { + return (x > 0.0) ? 1.0 : exp(x)*alpha; +} + +static inline double dELUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : (exp(x) - 1.0); +} + +static inline double softplus(double x) { + return log(1.0 + exp(x)); +} + +static inline double dsoftplus(double x) { + return 1.0 / (1.0 + exp(-x)); +} + +static inline double SELU(double x) { +#define alpha 1.6732632423543772848170429916717 +#define scale 1.0507009873554804934193349852946 + return (x > 0.0) ? scale * x : scale * alpha * (exp(x) - 1.0); +#undef alpha +#undef scale +} + +static inline double dSELU(double x) { +#define alpha 1.6732632423543772848170429916717 +#define scale 1.0507009873554804934193349852946 + return (x > 0.0) ? scale : scale * alpha * exp(x); +#undef alpha +#undef scale +} + +static inline double lReLU(double x) { + return (x > 0.0) ? x : 0.01 * x; +} + +static inline double dlReLU(double x) { + return (x > 0.0) ? 1.0 : 0.01; +} + +static inline double PReLU(double x, double alpha) { + return (x >= 0.0) ? x : alpha * x; +} + +static inline double dPReLU(double x, double alpha) { + return (x >= 0.0) ? 1.0 : alpha; +} + +static inline double dPReLUa(double x, double alpha) { + return (x >= 0.0) ? 0.0 : x; +} + +static inline double Swish(double x) { + return x / (1.0 + exp(-x)); +} + +static inline double dSwish(double x) { + double ex = exp(x); + double den = 1.0 + ex; + return ex / (den * den) + x * ex / (den * den); +} + #define _safe_log(a) _safe_log_(a, _solveData) static inline double _div0_(double denom, rx_solve *rx) { if (rx->safeZero) { From 6fba2242698cd9a1f9461e71007d4fefe84cdc3e Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Thu, 21 Nov 2024 21:38:40 -0600 Subject: [PATCH 02/16] Add activation functions --- R/d.R | 175 +++++++++++++++++++++++++++-- R/symengine.R | 31 ++++- inst/include/rxode2_model_shared.h | 51 +++++++++ tests/testthat/test-dsl.R | 51 +++++++++ 4 files changed, 300 insertions(+), 8 deletions(-) diff --git a/R/d.R b/R/d.R index 9d18ea12d..8b62a6d1e 100644 --- a/R/d.R +++ b/R/d.R @@ -624,24 +624,185 @@ } ) +.rxD$ReLU <- list( + function(x) { + paste0("dReLU(", x, ")") + } +) + +.rxD$dReLU <- list( + function(x) { + paste0("0") + } +) + +.rxD$GELU <- list( + function(x) { + paste0("dGELU(", x, ")") + } +) + +.rxD$dGELU <- list( + function(x) { + paste0("d2GELU(", x, ")") + } +) + +.rxD$d2GELU <- list( + function(x) { + paste0("d3GELU(", x, ")") + } +) + +.rxD$d3GELU <- list( + function(x) { + paste0("d4GELU(", x, ")") + } +) + +.rxD$ELU <- list( + function(x, alpha) { + paste0("dELU(", x, ", ", alpha, ")") + }, + function(x, alpha) { + paste0("dELUa(", x, ", ", alpha, ")") + }) + +.rxD$dELU <- list( + function(x, alpha) { + paste0("d2ELU(", x, ", ", alpha, ")") + }, + function(x, alpha) { + paste0("d2aELU(", x, ", ", alpha, ")") + }) + +.rxD$dELUa <- list( + function(x, alpha) { + paste0("d2ELUa(", x, ", ", alpha, ")") + }, + function(x, alpha) { + paste0("0") + } +) +.rxD$d2ELUa <- list( + function(x, alpha) { + paste0("d2ELUa(", x, ", ", alpha, ")") + }, + function(x, alpha) { + paste0("0") + } +) + +.rxD$d2ELU <- list( + function(x, alpha) { + paste0("d2ELU(", x, ", ", alpha, ")") + }, + function(x, alpha) { + paste0("d2aELU(", x, ", ", alpha, ")") + }) + +.rxD$d2aELU <- list( + function(x, alpha) { + paste0("d2aELU(", x, ", ", alpha, ")") + }, + function(x, alpha) { + paste0("0") + }) + +.rxD$softplus <- list( + function(x) { + paste0("dsoftplus(", x, ")") + }) + +.rxD$dsoftplus <- list( + function(x) { + paste0("d2softplus(", x, ")") + }) + +.rxD$d2softplus <- list( + function(x) { + paste0("d3softplus(", x, ")") + }) + +.rxD$d3softplus <- list( + function(x) { + paste0("d4softplus(", x, ")") + }) + +.rxD$SELU <- list( + function(x) { + paste0("dSELU(", x, ")") + }) + + +.rxD$lReLU <- list( + function(x) { + paste0("dlReLU(", x, ")") + } +) + +.rxD$dlReLU <- list( + function(x) { + paste0("0") + } +) + +.rxD$PReLU <- list( + function(x, alpha) { + paste0("dPReLU(", x, ",", alpha, ")") + }, + function(x, alpha) { + paste0("dPReLUa(", x, ",", alpha, ")") + }) + +.rxD$dPReLU <- list( + function(x, alpha) { + paste0("0") + }, + function(x, alpha) { + paste0("dPReLUa1(", x, ",", alpha, ")") + }) + +.rxD$dPReLUa <- list( + function(x, alpha) { + paste0("dPReLUa1(", x, ",", alpha, ")") + }, + function(x, alpha) { + paste0("0") + }) + +.rxD$dPReLUa1 <- list( + function(x, alpha) { + paste0("0") + }, + function(x, alpha) { + paste0("0") + } +) + +.rxD$Swish <- list( + function(x) { + paste0("dSwish(", x, ")") + } +) #' This gives the derivative table for rxode2 #' -#' This will help allow registration of functions in `rxode2` -#' +#' This will help allow registration of functions in `rxode2` +#' #' @return Derivative table environment for rxode2 #' @details -#' +#' #' This environment is a derivative table; -#' +#' #' For example: -#' +#' #' Derivative(f(a,b,c), a) = fa() #' Derivative(f(a,b,c), b) = fb() #' Derivative(f(a,b,c), c) = fc() -#' +#' #' Then the derivative table for `f` would be: -#' +#' #' assign("f", list(fa(a,b,c), fb(a,b,c), fc(a,b,c)), rxode2parseD()) #' #' fa translates the arguments to the derivative with respect to a diff --git a/R/symengine.R b/R/symengine.R index 5d918851e..1bd0e3753 100644 --- a/R/symengine.R +++ b/R/symengine.R @@ -239,7 +239,36 @@ regIfOrElse <- rex::rex(or(regIf, regElse)) "llikXCauchyDscale"=4, "llikXNorm"=4, "llikXNormDmean"=4, - "llikXNormDsd"=4 + "llikXNormDsd"=4, + "ReLU"=1, + "dReLU"=1, + "GELU"=1, + "dGELU"=1, + "d2GELU"=1, + "d3GELU"=1, + "d4GELU"=1, + "ELU"=2, + "dELU"=2, + "d2ELU"=2, + "d2aELU"=2, + "dELUa"=2, + "d2ELUa"=2, + "softplus"=1, + "dsoftplus"=1, + "d2softplus"=1, + "d3softplus"=1, + "d4softplus"=1, + "SELU"=1, + "dSELU"=1, + "lReLU"=1, + "dlReLU"=1, + "PReLU"=2, + "dPReLU"=2, + "d2PReLU"=2, + "dPReLUa"=2, + "dPReLUa1"=2, + "Swish"=1, + "dSwish"=1 ) .rxOnly <- c( diff --git a/inst/include/rxode2_model_shared.h b/inst/include/rxode2_model_shared.h index 75a472fbf..a7d520858 100644 --- a/inst/include/rxode2_model_shared.h +++ b/inst/include/rxode2_model_shared.h @@ -36,17 +36,45 @@ static inline double dGELU(double x) { return 0.5 * (1.0 + erf(x * M_SQRT1_2)) + x * M_1_SQRT_2PI * exp(-0.5 * x * x); } +static inline double d2GELU(double x) { + return (2.0- x*x) * exp(-0.5* x * x)*M_1_SQRT_2PI; +} + +static inline double d3GELU(double x) { + return x * exp(-0.5 * x * x) * M_1_SQRT_2PI * (x * x - 4.0); +} + +static inline double d4GELU(double x) { + return exp(-0.5*x*x)*M_1_SQRT_2PI*(7.0*x*x - 4.0 - x*x*x*x); +} + static inline double ELU(double x, double alpha) { return (x > 0.0) ? x : (exp(x) - 1.0) * alpha; } +// derivative of ELU with respect to x static inline double dELU(double x, double alpha) { return (x > 0.0) ? 1.0 : exp(x)*alpha; } +// derivative of dELU with respect to x +static inline double d2ELU(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x)*alpha; +} + +// derivative of dELU with respect to alpha +static inline double d2aELU(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x); +} + +// derivative of ELU with respect to alpha static inline double dELUa(double x, double alpha) { return (x > 0.0) ? 0.0 : (exp(x) - 1.0); } +// derivative of dELAa with respect to x +static inline double d2ELUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x); +} static inline double softplus(double x) { return log(1.0 + exp(x)); @@ -56,6 +84,25 @@ static inline double dsoftplus(double x) { return 1.0 / (1.0 + exp(-x)); } +static inline double d2softplus(double x) { + double ex = exp(x); + return ex / ((1.0 + ex) * (1.0 + ex)); +} + +static inline double d3softplus(double x) { + double ex = exp(-x); + double ex1 = (1.0 + ex); + return 2.0*exp(-2.0*x)/(ex1*ex1*ex1) - 1.0*ex/(ex1*ex1); +} + +static inline double d4softplus(double x) { + double ex = exp(-x); + double ex1 = (1.0 + ex); + return 6.0*exp(-3.0*x)/(ex1*ex1*ex1*ex1) - + 6.0*exp(-2.0*x)/(ex1*ex1*ex1) + + 1.0*ex/(ex1*ex1); +} + static inline double SELU(double x) { #define alpha 1.6732632423543772848170429916717 #define scale 1.0507009873554804934193349852946 @@ -92,6 +139,10 @@ static inline double dPReLUa(double x, double alpha) { return (x >= 0.0) ? 0.0 : x; } +static inline double dPReLUa1(double x, double alpha) { + return (x >= 0.0) ? 0.0 : 1.0; +} + static inline double Swish(double x) { return x / (1.0 + exp(-x)); } diff --git a/tests/testthat/test-dsl.R b/tests/testthat/test-dsl.R index 8db1de780..cd0e3e14c 100644 --- a/tests/testthat/test-dsl.R +++ b/tests/testthat/test-dsl.R @@ -259,6 +259,57 @@ rxTest({ ) }) + test_that("NN Activation functions derivatives", { + + expect_equal(rxFromSE("Derivative(ReLU(x), x)"), "dReLU(x)") + expect_equal(rxFromSE("Derivative(dReLU(x), x)"), "0") + + expect_equal(rxFromSE("Derivative(GELU(x), x)"), "dGELU(x)") + expect_equal(rxFromSE("Derivative(dGELU(x), x)"), "d2GELU(x)") + expect_equal(rxFromSE("Derivative(d2GELU(x), x)"), "d3GELU(x)") + expect_equal(rxFromSE("Derivative(d3GELU(x), x)"), "d4GELU(x)") + + expect_equal(rxFromSE("Derivative(ELU(x, alpha), x)"), "dELU(x, alpha)") + expect_equal(rxFromSE("Derivative(ELU(x, alpha), alpha)"), "dELUa(x, alpha)") + + expect_equal(rxFromSE("Derivative(dELU(x, alpha), x)"), "d2ELU(x, alpha)") + expect_equal(rxFromSE("Derivative(dELU(x, alpha), alpha)"), "d2aELU(x, alpha)") + + expect_equal(rxFromSE("Derivative(dELUa(x, alpha), x)"), "d2ELUa(x, alpha)") + expect_equal(rxFromSE("Derivative(dELUa(x, alpha), alpha)"), "0") + + expect_equal(rxFromSE("Derivative(d2ELUa(x, alpha), x)"), "d2ELUa(x, alpha)") + expect_equal(rxFromSE("Derivative(d2ELUa(x, alpha), alpha)"), "0") + + expect_equal(rxFromSE("Derivative(d2aELU(x, alpha), x)"), "d2aELU(x, alpha)") + expect_equal(rxFromSE("Derivative(d2aELU(x, alpha), alpha)"), "0") + + expect_equal(rxFromSE("Derivative(softplus(x), x)"), "dsoftplus(x)") + expect_equal(rxFromSE("Derivative(dsoftplus(x), x)"), "d2softplus(x)") + expect_equal(rxFromSE("Derivative(d2softplus(x), x)"), "d3softplus(x)") + expect_equal(rxFromSE("Derivative(d3softplus(x), x)"), "d4softplus(x)") + + expect_equal(rxFromSE("Derivative(SELU(x), x)"), "dSELU(x)") + + expect_equal(rxFromSE("Derivative(lReLU(x), x)"), "dlReLU(x)") + expect_equal(rxFromSE("Derivative(dlReLU(x), x)"), "0") + + expect_equal(rxFromSE("Derivative(PReLU(x, alpha), x)"), "dPReLU(x,alpha)") + expect_equal(rxFromSE("Derivative(PReLU(x, alpha), alpha)"), "dPReLUa(x,alpha)") + + expect_equal(rxFromSE("Derivative(dPReLU(x, alpha), x)"), "0") + expect_equal(rxFromSE("Derivative(dPReLU(x, alpha), alpha)"), "dPReLUa1(x,alpha)") + + expect_equal(rxFromSE("Derivative(dPReLUa(x, alpha), x)"), "dPReLUa1(x,alpha)") + expect_equal(rxFromSE("Derivative(dPReLUa(x, alpha), alpha)"), "0") + + expect_equal(rxFromSE("Derivative(dPReLUa1(x, alpha), x)"), "0") + expect_equal(rxFromSE("Derivative(dPReLUa1(x, alpha), alpha)"), "0") + + expect_equal(rxFromSE("Derivative(Swish(x), x)"), "dSwish(x)") + + }) + test_that("logic tests", { expect_equal(rxFromSE("rxEq(a,b)"), "(a==b)") expect_equal(rxFromSE("rxNeq(a,b)"), "(a!=b)") From 4bc79f014c6571189feab09e07d4c37775e8d506 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 11:20:51 -0600 Subject: [PATCH 03/16] Add ReLU R function --- NAMESPACE | 2 + R/dfIni.R | 234 +++++++++++++++++------------ R/parseFuns.R | 21 ++- R/relu.R | 57 +++++++ inst/include/rxode2_model_shared.c | 59 ++++++++ inst/include/rxode2_model_shared.h | 134 ----------------- man/ReLU.Rd | 42 ++++++ man/dReLU.Rd | 37 +++++ src/init.c | 60 +++++++- src/utilc.c | 173 +++++++++++++++++++++ 10 files changed, 579 insertions(+), 240 deletions(-) create mode 100644 R/relu.R create mode 100644 man/ReLU.Rd create mode 100644 man/dReLU.Rd diff --git a/NAMESPACE b/NAMESPACE index 16a94c37a..ec8368b51 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -343,6 +343,7 @@ export(.udfExists) export(.udfMd5Info) export(.useUtf) export(.vecDf) +export(ReLU) export(RxODE) export(add.dosing) export(add.sampling) @@ -378,6 +379,7 @@ export(binomProbs) export(boxCox) export(boxCoxInv) export(cvPost) +export(dReLU) export(dfWishart) export(erf) export(et) diff --git a/R/dfIni.R b/R/dfIni.R index cc9cd6a3d..83680f278 100644 --- a/R/dfIni.R +++ b/R/dfIni.R @@ -3,103 +3,139 @@ "rxf", "rxgeom", "rxgamma", "rxbeta", "rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", "ribinom", "rinbinom", "rinbinomMu", "ricauchy", "richisq", "riexp", "rif", "rigeom", "rigamma", "ribeta", "ripois", -"rit_", "riunif", "riweibull", "phi", "gammap", "gammaq", "gammapInv", +"rit_", "riunif", "riweibull", "ReLU", "dReLU", "GELU", "dGELU", +"d2GELU", "d3GELU", "d4GELU", "softplus", "dsoftplus", "d2softplus", +"d3softplus", "d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", +"Swish", "dSwish", "PReLU", "dPReLU", "dPReLUa", "dPReLUa1", +"ELU", "dELU", "d2ELU", "d2aELU", "dELUa", "d2ELUa", "phi", "gammap", +"gammaq", "gammapInv", "gammapInva", "gammaqInv", "gammaqInva", +"uppergamma", "lowergamma", "gammapDer", "logit", "expit", "simeta", +"simeps", "llikNorm", "llikNormDmean", "llikNormDsd", "llikPois", +"llikPoisDlambda", "llikBinom", "llikBinomDprob", "llikNbinom", +"llikNbinomDprob", "llikNbinomMu", "llikNbinomMuDmu", "llikBeta", +"llikBetaDshape1", "llikBetaDshape2", "llikT", "llikTDdf", "llikTDmean", +"llikTDsd", "llikChisq", "llikChisqDdf", "llikExp", "llikExpDrate", +"llikF", "llikFDdf1", "llikFDdf2", "llikGeom", "llikGeomDp", +"llikUnif", "llikUnifDalpha", "llikUnifDbeta", "llikWeibull", +"llikWeibullDshape", "llikWeibullDscale", "llikGamma", "llikGammaDshape", +"llikGammaDrate", "llikCauchy", "llikCauchyDlocation", "llikCauchyDscale" +), fun = c("linCmtA", "linCmtB", "linCmtC", "rxnorm", "rxbinom", +"rxnbinom", "rxnbinomMu", "rxcauchy", "rxchisq", "rxexp", "rxf", +"rxgeom", "rxgamma", "rxbeta", "rxpois", "rxt_", "rxunif", "rxweibull", +"rinorm", "ribinom", "rinbinom", "rinbinomMu", "ricauchy", "richisq", +"riexp", "rif", "rigeom", "rigamma", "ribeta", "ripois", "rit_", +"riunif", "riweibull", "ReLU", "dReLU", "GELU", "dGELU", "d2GELU", +"d3GELU", "d4GELU", "softplus", "dsoftplus", "d2softplus", "d3softplus", +"d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", "Swish", "dSwish", +"PReLU", "dPReLU", "dPReLUa", "dPReLUa1", "ELU", "dELU", "d2ELU", +"d2aELU", "dELUa", "d2ELUa", "phi", "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", "gammaqInva", "uppergamma", "lowergamma", -"gammapDer", "logit", "expit", "simeta", "simeps", "llikNorm", -"llikNormDmean", "llikNormDsd", "llikPois", "llikPoisDlambda", -"llikBinom", "llikBinomDprob", "llikNbinom", "llikNbinomDprob", -"llikNbinomMu", "llikNbinomMuDmu", "llikBeta", "llikBetaDshape1", -"llikBetaDshape2", "llikT", "llikTDdf", "llikTDmean", "llikTDsd", -"llikChisq", "llikChisqDdf", "llikExp", "llikExpDrate", "llikF", -"llikFDdf1", "llikFDdf2", "llikGeom", "llikGeomDp", "llikUnif", -"llikUnifDalpha", "llikUnifDbeta", "llikWeibull", "llikWeibullDshape", -"llikWeibullDscale", "llikGamma", "llikGammaDshape", "llikGammaDrate", -"llikCauchy", "llikCauchyDlocation", "llikCauchyDscale"), fun = c("linCmtA", -"linCmtB", "linCmtC", "rxnorm", "rxbinom", "rxnbinom", "rxnbinomMu", -"rxcauchy", "rxchisq", "rxexp", "rxf", "rxgeom", "rxgamma", "rxbeta", -"rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", "ribinom", -"rinbinom", "rinbinomMu", "ricauchy", "richisq", "riexp", "rif", -"rigeom", "rigamma", "ribeta", "ripois", "rit_", "riunif", "riweibull", -"phi", "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", -"gammaqInva", "uppergamma", "lowergamma", "gammapDer", "logit", -"expit", "simeta", "simeps", "_llikNorm", "_llikNormDmean", "_llikNormDsd", -"_llikPois", "_llikPoisDlambda", "_llikBinom", "_llikBinomDprob", -"_llikNbinom", "_llikNbinomDprob", "_llikNbinomMu", "_llikNbinomMuDmu", -"_llikBeta", "_llikBetaDshape1", "_llikBetaDshape2", "_llikT", -"_llikTDdf", "_llikTDmean", "_llikTDsd", "_llikChisq", "_llikChisqDdf", -"_llikExp", "_llikExpDrate", "_llikF", "_llikFDdf1", "_llikFDdf2", -"_llikGeom", "_llikGeomDp", "_llikUnif", "_llikUnifDalpha", "_llikUnifDbeta", -"_llikWeibull", "_llikWeibullDshape", "_llikWeibullDscale", "_llikGamma", -"_llikGammaDshape", "_llikGammaDrate", "_llikCauchy", "_llikCauchyDlocation", -"_llikCauchyDscale"), type = c("linCmtA_p", "linCmtB_p", "linCmtA_p", -"rxode2i_fn2", "rxode2i_rxbinom", "rxode2i_rxbinom", "rxode2i_rxbinom", -"rxode2i_fn2", "rxode2i_fn", "rxode2i_fn", "rxode2i_fn2", "rxode2i_ifn", -"rxode2i_fn2", "rxode2i_fn2", "rxode2i_ifn", "rxode2i_fn", "rxode2i_fn2", -"rxode2i_fn2", "rxode2i2_fn2", "rxode2i2_ribinom", "rxode2i2_ribinom", -"rxode2i2_ribinom", "rxode2i2_fn2", "rxode2i2_fn", "rxode2i2_fn", -"rxode2i2_fn2", "rxode2i2_ifn", "rxode2i2_fn2", "rxode2i2_fn2", -"rxode2i2_ifn", "rxode2i2_fn", "rxode2i2_fn2", "rxode2i2_fn2", -"rxode2_fn", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", -"rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", -"rxode2_fn3", "rxode2_fn3", "_simfun", "_simfun", "rxode2_llikNormFun", -"rxode2_llikNormFun", "rxode2_llikNormFun", "rxode2_llikPoisFun", -"rxode2_llikPoisFun", "rxode2_llikBinomFun", "rxode2_llikBinomFun", -"rxode2_llikBinomFun", "rxode2_llikBinomFun", "rxode2_llikBinomFun", -"rxode2_llikBinomFun", "rxode2_llikBetaFun", "rxode2_llikBetaFun", -"rxode2_llikBetaFun", "rxode2_llikTFun", "rxode2_llikTFun", "rxode2_llikTFun", -"rxode2_llikTFun", "rxode2_llikChisqFun", "rxode2_llikChisqFun", -"rxode2_llikExpFun", "rxode2_llikExpFun", "rxode2_llikFFun", -"rxode2_llikFFun", "rxode2_llikFFun", "rxode2_llikGeomFun", "rxode2_llikGeomFun", -"rxode2_llikUnifFun", "rxode2_llikUnifFun", "rxode2_llikUnifFun", -"rxode2_llikWeibullFun", "rxode2_llikWeibullFun", "rxode2_llikWeibullFun", -"rxode2_llikGammaFun", "rxode2_llikGammaFun", "rxode2_llikGammaFun", -"rxode2_llikCauchyFun", "rxode2_llikCauchyFun", "rxode2_llikCauchyFun" -), package = c("rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll"), packageFun = c("linCmtA", -"linCmtB", "linCmtC", "rxnorm", "rxbinom", "rxnbinom", "rxnbinomMu", -"rxcauchy", "rxchisq", "rxexp", "rxf", "rxgeom", "rxgamma", "rxbeta", -"rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", "ribinom", -"rinbinom", "rinbinomMu", "ricauchy", "richisq", "riexp", "rif", -"rigeom", "rigamma", "ribeta", "ripois", "rit_", "riunif", "riweibull", -"phi", "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", -"gammaqInva", "uppergamma", "lowergamma", "gammapDer", "logit", -"expit", "simeta", "simeps", "rxLlikNorm", "rxLlikNormDmean", -"rxLlikNormDsd", "rxLlikPois", "rxLlikPoisDlambda", "rxLlikBinom", -"rxLlikBinomDprob", "rxLlikNbinom", "rxLlikNbinomDprob", "rxLlikNbinomMu", -"rxLlikNbinomMuDmu", "rxLlikBeta", "rxLlikBetaDshape1", "rxLlikBetaDshape2", -"rxLlikT", "rxLlikTDdf", "rxLlikTDmean", "rxLlikTDsd", "rxLlikChisq", -"rxLlikChisqDdf", "rxLlikExp", "rxLlikExpDrate", "rxLlikF", "rxLlikFDdf1", -"rxLlikFDdf2", "rxLlikGeom", "rxLlikGeomDp", "rxLlikUnif", "rxLlikUnifDalpha", -"rxLlikUnifDbeta", "rxLlikWeibull", "rxLlikWeibullDshape", "rxLlikWeibullDscale", -"rxLlikGamma", "rxLlikGammaDshape", "rxLlikGammaDrate", "rxLlikCauchy", -"rxLlikCauchyDlocation", "rxLlikCauchyDscale"), argMin = c(20L, -21L, 20L, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, -3L, 3L, 2L, 2L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, -4L, 2L, 2L, 2L, 2L, 3L, 3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, -3L, 3L, 3L, 3L, 3L, 3L), argMax = c(20L, 21L, 20L, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 2L, 2L, 2L, 2L, -2L, 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, 3L, 3L, 2L, 2L, 3L, 3L, -3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, 4L, 2L, 2L, 2L, 2L, 3L, -3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L -), threadSafe = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L)), row.names = c(NA, --86L), class = "data.frame") +"gammapDer", "logit", "expit", "simeta", "simeps", "_llikNorm", +"_llikNormDmean", "_llikNormDsd", "_llikPois", "_llikPoisDlambda", +"_llikBinom", "_llikBinomDprob", "_llikNbinom", "_llikNbinomDprob", +"_llikNbinomMu", "_llikNbinomMuDmu", "_llikBeta", "_llikBetaDshape1", +"_llikBetaDshape2", "_llikT", "_llikTDdf", "_llikTDmean", "_llikTDsd", +"_llikChisq", "_llikChisqDdf", "_llikExp", "_llikExpDrate", "_llikF", +"_llikFDdf1", "_llikFDdf2", "_llikGeom", "_llikGeomDp", "_llikUnif", +"_llikUnifDalpha", "_llikUnifDbeta", "_llikWeibull", "_llikWeibullDshape", +"_llikWeibullDscale", "_llikGamma", "_llikGammaDshape", "_llikGammaDrate", +"_llikCauchy", "_llikCauchyDlocation", "_llikCauchyDscale"), + type = c("linCmtA_p", "linCmtB_p", "linCmtA_p", "rxode2i_fn2", + "rxode2i_rxbinom", "rxode2i_rxbinom", "rxode2i_rxbinom", + "rxode2i_fn2", "rxode2i_fn", "rxode2i_fn", "rxode2i_fn2", + "rxode2i_ifn", "rxode2i_fn2", "rxode2i_fn2", "rxode2i_ifn", + "rxode2i_fn", "rxode2i_fn2", "rxode2i_fn2", "rxode2i2_fn2", + "rxode2i2_ribinom", "rxode2i2_ribinom", "rxode2i2_ribinom", + "rxode2i2_fn2", "rxode2i2_fn", "rxode2i2_fn", "rxode2i2_fn2", + "rxode2i2_ifn", "rxode2i2_fn2", "rxode2i2_fn2", "rxode2i2_ifn", + "rxode2i2_fn", "rxode2i2_fn2", "rxode2i2_fn2", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn3", "rxode2_fn3", "_simfun", + "_simfun", "rxode2_llikNormFun", "rxode2_llikNormFun", "rxode2_llikNormFun", + "rxode2_llikPoisFun", "rxode2_llikPoisFun", "rxode2_llikBinomFun", + "rxode2_llikBinomFun", "rxode2_llikBinomFun", "rxode2_llikBinomFun", + "rxode2_llikBinomFun", "rxode2_llikBinomFun", "rxode2_llikBetaFun", + "rxode2_llikBetaFun", "rxode2_llikBetaFun", "rxode2_llikTFun", + "rxode2_llikTFun", "rxode2_llikTFun", "rxode2_llikTFun", + "rxode2_llikChisqFun", "rxode2_llikChisqFun", "rxode2_llikExpFun", + "rxode2_llikExpFun", "rxode2_llikFFun", "rxode2_llikFFun", + "rxode2_llikFFun", "rxode2_llikGeomFun", "rxode2_llikGeomFun", + "rxode2_llikUnifFun", "rxode2_llikUnifFun", "rxode2_llikUnifFun", + "rxode2_llikWeibullFun", "rxode2_llikWeibullFun", "rxode2_llikWeibullFun", + "rxode2_llikGammaFun", "rxode2_llikGammaFun", "rxode2_llikGammaFun", + "rxode2_llikCauchyFun", "rxode2_llikCauchyFun", "rxode2_llikCauchyFun" + ), package = c("rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll"), packageFun = c("linCmtA", "linCmtB", + "linCmtC", "rxnorm", "rxbinom", "rxnbinom", "rxnbinomMu", + "rxcauchy", "rxchisq", "rxexp", "rxf", "rxgeom", "rxgamma", + "rxbeta", "rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", + "ribinom", "rinbinom", "rinbinomMu", "ricauchy", "richisq", + "riexp", "rif", "rigeom", "rigamma", "ribeta", "ripois", + "rit_", "riunif", "riweibull", "ReLU", "dReLU", "GELU", "dGELU", + "d2GELU", "d3GELU", "d4GELU", "softplus", "dsoftplus", "d2softplus", + "d3softplus", "d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", + "Swish", "dSwish", "PReLU", "dPReLU", "dPReLUa", "dPReLUa1", + "ELU", "dELU", "d2ELU", "d2aELU", "dELUa", "d2ELUa", "phi", + "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", + "gammaqInva", "uppergamma", "lowergamma", "gammapDer", "logit", + "expit", "simeta", "simeps", "rxLlikNorm", "rxLlikNormDmean", + "rxLlikNormDsd", "rxLlikPois", "rxLlikPoisDlambda", "rxLlikBinom", + "rxLlikBinomDprob", "rxLlikNbinom", "rxLlikNbinomDprob", + "rxLlikNbinomMu", "rxLlikNbinomMuDmu", "rxLlikBeta", "rxLlikBetaDshape1", + "rxLlikBetaDshape2", "rxLlikT", "rxLlikTDdf", "rxLlikTDmean", + "rxLlikTDsd", "rxLlikChisq", "rxLlikChisqDdf", "rxLlikExp", + "rxLlikExpDrate", "rxLlikF", "rxLlikFDdf1", "rxLlikFDdf2", + "rxLlikGeom", "rxLlikGeomDp", "rxLlikUnif", "rxLlikUnifDalpha", + "rxLlikUnifDbeta", "rxLlikWeibull", "rxLlikWeibullDshape", + "rxLlikWeibullDscale", "rxLlikGamma", "rxLlikGammaDshape", + "rxLlikGammaDrate", "rxLlikCauchy", "rxLlikCauchyDlocation", + "rxLlikCauchyDscale"), argMin = c(20L, 21L, 20L, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, NA, 2L, 2L, 2L, + 2L, 2L, 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, 3L, 3L, 2L, 2L, + 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, 4L, 2L, 2L, + 2L, 2L, 3L, 3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, + 3L, 3L, 3L, 3L), argMax = c(20L, 21L, 20L, NA, NA, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, + 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, NA, 2L, 2L, 2L, 2L, 2L, + 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, 3L, 3L, 2L, 2L, 3L, 3L, + 3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, 4L, 2L, 2L, 2L, 2L, + 3L, 3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, + 3L, 3L), threadSafe = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L + )), row.names = c(NA, -114L), class = "data.frame") diff --git a/R/parseFuns.R b/R/parseFuns.R index 397d90787..aded6523c 100644 --- a/R/parseFuns.R +++ b/R/parseFuns.R @@ -32,10 +32,14 @@ "llikXUnifDalpha", "llikXUnifDbeta", "llikXWeibull", "llikXWeibullDshape", "llikXWeibullDscale", "llikXGamma", "llikXGammaDshape", "llikXGammaDrate", "llikXCauchy", "llikXCauchyDlocation", "llikXCauchyDscale", "llikXNorm", -"llikXNormDmean", "llikXNormDsd", "linCmt", "rnorm", "rxnorm", -"rxbinom", "rbinom", "rxcauchy", "rcauchy", "rchisq", "rxchisq", -"rexp", "rxexp", "rbeta", "rxbeta", "rgeom", "rxgeom", "rxpois", -"rpois", "rxt", "rt") +"llikXNormDmean", "llikXNormDsd", "ReLU", "dReLU", "GELU", "dGELU", +"d2GELU", "d3GELU", "d4GELU", "ELU", "dELU", "d2ELU", "d2aELU", +"dELUa", "d2ELUa", "softplus", "dsoftplus", "d2softplus", "d3softplus", +"d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", "PReLU", "dPReLU", +"d2PReLU", "dPReLUa", "dPReLUa1", "Swish", "dSwish", "linCmt", +"rnorm", "rxnorm", "rxbinom", "rbinom", "rxcauchy", "rcauchy", +"rchisq", "rxchisq", "rexp", "rxexp", "rbeta", "rxbeta", "rgeom", +"rxgeom", "rxpois", "rpois", "rxt", "rt") .parseEnv$.parseNum <- c(lgamma = 1, abs = 1, acos = 1, acosh = 1, asin = 1, asinh = 1, atan = 1, atan2 = 2, atanh = 1, beta = 2, cos = 1, cosh = 1, erf = 1, erfc = 1, exp = 1, gamma = 1, linCmtA = 20, linCmtC = 20, @@ -70,5 +74,10 @@ llikXFDdf1 = 4, llikXFDdf2 = 4, llikXGeom = 3, llikXGeomDprob = 3, llikXUnif = 4, llikXUnifDalpha = 4, llikXUnifDbeta = 4, llikXWeibull = 4, llikXWeibullDshape = 4, llikXWeibullDscale = 4, llikXGamma = 4, llikXGammaDshape = 4, llikXGammaDrate = 4, llikXCauchy = 4, llikXCauchyDlocation = 4, -llikXCauchyDscale = 4, llikXNorm = 4, llikXNormDmean = 4, llikXNormDsd = 4 -) +llikXCauchyDscale = 4, llikXNorm = 4, llikXNormDmean = 4, llikXNormDsd = 4, +ReLU = 1, dReLU = 1, GELU = 1, dGELU = 1, d2GELU = 1, d3GELU = 1, +d4GELU = 1, ELU = 2, dELU = 2, d2ELU = 2, d2aELU = 2, dELUa = 2, +d2ELUa = 2, softplus = 1, dsoftplus = 1, d2softplus = 1, d3softplus = 1, +d4softplus = 1, SELU = 1, dSELU = 1, lReLU = 1, dlReLU = 1, PReLU = 2, +dPReLU = 2, d2PReLU = 2, dPReLUa = 2, dPReLUa1 = 2, Swish = 1, +dSwish = 1) diff --git a/R/relu.R b/R/relu.R new file mode 100644 index 000000000..e8a1d236c --- /dev/null +++ b/R/relu.R @@ -0,0 +1,57 @@ +#' Rectified Linear Unit (ReLU) Activation Function +#' +#' This function applies the Rectified Linear Unit (ReLU) activation function to the input numeric vector. +#' The ReLU function is defined as the positive part of its argument: \eqn{f(x) = max(0, x)}. +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' ReLU(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=ReLU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +ReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 2L) +} +#' Derivative of the Rectified Linear Unit (ReLU) Activation Function +#' +#' This function applies the derivative of the Rectified Linear Unit +#' (ReLU) activation function to the input numeric vector. +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' +#' @family Activation Functions +#' +#' @return A numeric vector where the derivative of the ReLU function +#' @export +#' @examples +#' +#' dReLU(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=dReLU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +dReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 7L) +} diff --git a/inst/include/rxode2_model_shared.c b/inst/include/rxode2_model_shared.c index 95f642cb1..7edcb5933 100644 --- a/inst/include/rxode2_model_shared.c +++ b/inst/include/rxode2_model_shared.c @@ -25,6 +25,37 @@ _rx_asgn _rxode2_rxAssignPtr =NULL; _rx_asgn _rxQr =NULL; rxode2_fn phi; +rxode2_fn ReLU; +rxode2_fn dReLU; +rxode2_fn GELU; +rxode2_fn dGELU; +rxode2_fn d2GELU; +rxode2_fn d3GELU; +rxode2_fn d4GELU; +rxode2_fn softplus; +rxode2_fn dsoftplus; +rxode2_fn d2softplus; +rxode2_fn d3softplus; +rxode2_fn d4softplus; +rxode2_fn SELU; +rxode2_fn dSELU; +rxode2_fn lReLU; +rxode2_fn dlReLU; +rxode2_fn Swish; +rxode2_fn dSwish; + +rxode2_fn2 PReLU; +rxode2_fn2 dPReLU; +rxode2_fn2 dPReLUa; +rxode2_fn2 dPReLUa1; + +rxode2_fn2 ELU; +rxode2_fn2 dELU; +rxode2_fn2 d2ELU; +rxode2_fn2 d2aELU; +rxode2_fn2 dELUa; +rxode2_fn2 d2ELUa; + rxode2_fn3 logit; rxode2_fn3 expit; rxode2_fn2 gammap; @@ -310,6 +341,34 @@ void _assignFuns0(void) { rit_ = (rxode2i2_fn)R_GetCCallable("rxode2","rit_"); riunif = (rxode2i2_fn2)R_GetCCallable("rxode2","riunif"); riweibull = (rxode2i2_fn2)R_GetCCallable("rxode2","riweibull"); + ReLU = (rxode2_fn)R_GetCCallable("rxode2","ReLU"); + dReLU = (rxode2_fn)R_GetCCallable("rxode2","dReLU"); + GELU = (rxode2_fn)R_GetCCallable("rxode2","GELU"); + dGELU = (rxode2_fn)R_GetCCallable("rxode2","dGELU"); + d2GELU = (rxode2_fn)R_GetCCallable("rxode2","d2GELU"); + d3GELU = (rxode2_fn)R_GetCCallable("rxode2","d3GELU"); + d4GELU = (rxode2_fn)R_GetCCallable("rxode2","d4GELU"); + softplus = (rxode2_fn)R_GetCCallable("rxode2","softplus"); + dsoftplus = (rxode2_fn)R_GetCCallable("rxode2","dsoftplus"); + d2softplus = (rxode2_fn)R_GetCCallable("rxode2","d2softplus"); + d3softplus = (rxode2_fn)R_GetCCallable("rxode2","d3softplus"); + d4softplus = (rxode2_fn)R_GetCCallable("rxode2","d4softplus"); + SELU = (rxode2_fn)R_GetCCallable("rxode2","SELU"); + dSELU = (rxode2_fn)R_GetCCallable("rxode2","dSELU"); + lReLU = (rxode2_fn)R_GetCCallable("rxode2","lReLU"); + dlReLU = (rxode2_fn)R_GetCCallable("rxode2","dlReLU"); + Swish = (rxode2_fn)R_GetCCallable("rxode2","Swish"); + dSwish = (rxode2_fn)R_GetCCallable("rxode2","dSwish"); + PReLU = (rxode2_fn2)R_GetCCallable("rxode2","PReLU"); + dPReLU = (rxode2_fn2)R_GetCCallable("rxode2","dPReLU"); + dPReLUa = (rxode2_fn2)R_GetCCallable("rxode2","dPReLUa"); + dPReLUa1 = (rxode2_fn2)R_GetCCallable("rxode2","dPReLUa1"); + ELU = (rxode2_fn2)R_GetCCallable("rxode2","ELU"); + dELU = (rxode2_fn2)R_GetCCallable("rxode2","dELU"); + d2ELU = (rxode2_fn2)R_GetCCallable("rxode2","d2ELU"); + d2aELU = (rxode2_fn2)R_GetCCallable("rxode2","d2aELU"); + dELUa = (rxode2_fn2)R_GetCCallable("rxode2","dELUa"); + d2ELUa = (rxode2_fn2)R_GetCCallable("rxode2","d2ELUa"); phi = (rxode2_fn)R_GetCCallable("rxode2","phi"); gammap = (rxode2_fn2) R_GetCCallable("rxode2","gammap"); diff --git a/inst/include/rxode2_model_shared.h b/inst/include/rxode2_model_shared.h index a7d520858..073000d55 100644 --- a/inst/include/rxode2_model_shared.h +++ b/inst/include/rxode2_model_shared.h @@ -19,140 +19,6 @@ static inline double _safe_log_(double a, rx_solve *rx) { } } -static inline double ReLU(double x) { - return (x > 0.0) ? x : 0.0; -} - -static inline double dReLU(double x) { - return (x > 0.0) ? 1.0 : 0.0; -} - -// -static inline double GELU(double x) { - return 0.5 * x * (1.0 + erf(x * M_SQRT1_2)); -} - -static inline double dGELU(double x) { - return 0.5 * (1.0 + erf(x * M_SQRT1_2)) + x * M_1_SQRT_2PI * exp(-0.5 * x * x); -} - -static inline double d2GELU(double x) { - return (2.0- x*x) * exp(-0.5* x * x)*M_1_SQRT_2PI; -} - -static inline double d3GELU(double x) { - return x * exp(-0.5 * x * x) * M_1_SQRT_2PI * (x * x - 4.0); -} - -static inline double d4GELU(double x) { - return exp(-0.5*x*x)*M_1_SQRT_2PI*(7.0*x*x - 4.0 - x*x*x*x); -} - -static inline double ELU(double x, double alpha) { - return (x > 0.0) ? x : (exp(x) - 1.0) * alpha; -} - -// derivative of ELU with respect to x -static inline double dELU(double x, double alpha) { - return (x > 0.0) ? 1.0 : exp(x)*alpha; -} - -// derivative of dELU with respect to x -static inline double d2ELU(double x, double alpha) { - return (x > 0.0) ? 0.0 : exp(x)*alpha; -} - -// derivative of dELU with respect to alpha -static inline double d2aELU(double x, double alpha) { - return (x > 0.0) ? 0.0 : exp(x); -} - -// derivative of ELU with respect to alpha -static inline double dELUa(double x, double alpha) { - return (x > 0.0) ? 0.0 : (exp(x) - 1.0); -} -// derivative of dELAa with respect to x -static inline double d2ELUa(double x, double alpha) { - return (x > 0.0) ? 0.0 : exp(x); -} - -static inline double softplus(double x) { - return log(1.0 + exp(x)); -} - -static inline double dsoftplus(double x) { - return 1.0 / (1.0 + exp(-x)); -} - -static inline double d2softplus(double x) { - double ex = exp(x); - return ex / ((1.0 + ex) * (1.0 + ex)); -} - -static inline double d3softplus(double x) { - double ex = exp(-x); - double ex1 = (1.0 + ex); - return 2.0*exp(-2.0*x)/(ex1*ex1*ex1) - 1.0*ex/(ex1*ex1); -} - -static inline double d4softplus(double x) { - double ex = exp(-x); - double ex1 = (1.0 + ex); - return 6.0*exp(-3.0*x)/(ex1*ex1*ex1*ex1) - - 6.0*exp(-2.0*x)/(ex1*ex1*ex1) + - 1.0*ex/(ex1*ex1); -} - -static inline double SELU(double x) { -#define alpha 1.6732632423543772848170429916717 -#define scale 1.0507009873554804934193349852946 - return (x > 0.0) ? scale * x : scale * alpha * (exp(x) - 1.0); -#undef alpha -#undef scale -} - -static inline double dSELU(double x) { -#define alpha 1.6732632423543772848170429916717 -#define scale 1.0507009873554804934193349852946 - return (x > 0.0) ? scale : scale * alpha * exp(x); -#undef alpha -#undef scale -} - -static inline double lReLU(double x) { - return (x > 0.0) ? x : 0.01 * x; -} - -static inline double dlReLU(double x) { - return (x > 0.0) ? 1.0 : 0.01; -} - -static inline double PReLU(double x, double alpha) { - return (x >= 0.0) ? x : alpha * x; -} - -static inline double dPReLU(double x, double alpha) { - return (x >= 0.0) ? 1.0 : alpha; -} - -static inline double dPReLUa(double x, double alpha) { - return (x >= 0.0) ? 0.0 : x; -} - -static inline double dPReLUa1(double x, double alpha) { - return (x >= 0.0) ? 0.0 : 1.0; -} - -static inline double Swish(double x) { - return x / (1.0 + exp(-x)); -} - -static inline double dSwish(double x) { - double ex = exp(x); - double den = 1.0 + ex; - return ex / (den * den) + x * ex / (den * den); -} - #define _safe_log(a) _safe_log_(a, _solveData) static inline double _div0_(double denom, rx_solve *rx) { if (rx->safeZero) { diff --git a/man/ReLU.Rd b/man/ReLU.Rd new file mode 100644 index 000000000..5cfd10bb3 --- /dev/null +++ b/man/ReLU.Rd @@ -0,0 +1,42 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/relu.R +\name{ReLU} +\alias{ReLU} +\title{Rectified Linear Unit (ReLU) Activation Function} +\usage{ +ReLU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +This function applies the Rectified Linear Unit (ReLU) activation function to the input numeric vector. +The ReLU function is defined as the positive part of its argument: \eqn{f(x) = max(0, x)}. +} +\examples{ + +ReLU(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r=ReLU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{dReLU}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dReLU.Rd b/man/dReLU.Rd new file mode 100644 index 000000000..7537e5a04 --- /dev/null +++ b/man/dReLU.Rd @@ -0,0 +1,37 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/relu.R +\name{dReLU} +\alias{dReLU} +\title{Derivative of the Rectified Linear Unit (ReLU) Activation Function} +\usage{ +dReLU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative of the ReLU function +} +\description{ +This function applies the derivative of the Rectified Linear Unit +(ReLU) activation function to the input numeric vector. +} +\examples{ + +dReLU(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r=dReLU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ReLU}()} +} +\concept{Activation Functions} diff --git a/src/init.c b/src/init.c index ac12333ff..3cd55d97d 100644 --- a/src/init.c +++ b/src/init.c @@ -112,6 +112,35 @@ SEXP _gammaqInva(SEXP, SEXP); double expit(double, double, double); double logit(double, double, double); +double ReLU(double x); +double dReLU(double x); +double GELU(double x); +double dGELU(double x); +double d2GELU(double x); +double d3GELU(double x); +double d4GELU(double x); +double ELU(double x, double alpha); +double dELU(double x, double alpha); +double d2ELU(double x, double alpha); +double d2aELU(double x, double alpha); +double dELUa(double x, double alpha); +double d2ELUa(double x, double alpha); +double softplus(double x); +double dsoftplus(double x); +double d2softplus(double x); +double d3softplus(double x); +double d4softplus(double x); +double SELU(double x); +double dSELU(double x); +double lReLU(double x); +double dlReLU(double x); +double PReLU(double x, double alpha); +double dPReLU(double x, double alpha); +double dPReLUa(double x, double alpha); +double dPReLUa1(double x, double alpha); +double Swish(double x); +double dSwish(double x); + SEXP _linCmtParse(SEXP vars, SEXP inStr, SEXP verbose); SEXP _rxode2_linCmtGen(SEXP linCmt, SEXP vars, SEXP linCmtSens, SEXP verbose); @@ -540,8 +569,10 @@ SEXP _rxode2_rxode2Ptr(void) { SEXP _rxode2_powerD(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); +SEXP _rxode2_activationF(SEXP xS, SEXP typeS); void R_init_rxode2(DllInfo *info){ R_CallMethodDef callMethods[] = { + {"_rxode2_activationF", (DL_FUNC) &_rxode2_activationF, 2}, {"_rxode2_itoletter", (DL_FUNC) &_rxode2_itoletter, 2}, {"_rxode2_itostr", (DL_FUNC) &_rxode2_itostr, 2}, {"_rxode2_powerD", (DL_FUNC) &_rxode2_powerD, 6}, @@ -716,7 +747,34 @@ void R_init_rxode2(DllInfo *info){ R_RegisterCCallable("rxode2", "getSilentErr", (DL_FUNC) &getSilentErr); R_RegisterCCallable("rxode2", "logit", (DL_FUNC) &logit); R_RegisterCCallable("rxode2", "expit", (DL_FUNC) &expit); - + R_RegisterCCallable("rxode2", "ReLU", (DL_FUNC) &ReLU); + R_RegisterCCallable("rxode2", "dReLU", (DL_FUNC) &dReLU); + R_RegisterCCallable("rxode2", "GELU", (DL_FUNC) &GELU); + R_RegisterCCallable("rxode2", "dGELU", (DL_FUNC) &dGELU); + R_RegisterCCallable("rxode2", "d2GELU", (DL_FUNC) &d2GELU); + R_RegisterCCallable("rxode2", "d3GELU", (DL_FUNC) &d3GELU); + R_RegisterCCallable("rxode2", "d4GELU", (DL_FUNC) &d4GELU); + R_RegisterCCallable("rxode2", "ELU", (DL_FUNC) &ELU); + R_RegisterCCallable("rxode2", "dELU", (DL_FUNC) &dELU); + R_RegisterCCallable("rxode2", "d2ELU", (DL_FUNC) &d2ELU); + R_RegisterCCallable("rxode2", "d2aELU", (DL_FUNC) &d2aELU); + R_RegisterCCallable("rxode2", "dELUa", (DL_FUNC) &dELUa); + R_RegisterCCallable("rxode2", "d2ELUa", (DL_FUNC) &d2ELUa); + R_RegisterCCallable("rxode2", "softplus", (DL_FUNC) &softplus); + R_RegisterCCallable("rxode2", "dsoftplus", (DL_FUNC) &dsoftplus); + R_RegisterCCallable("rxode2", "d2softplus", (DL_FUNC) &d2softplus); + R_RegisterCCallable("rxode2", "d3softplus", (DL_FUNC) &d3softplus); + R_RegisterCCallable("rxode2", "d4softplus", (DL_FUNC) &d4softplus); + R_RegisterCCallable("rxode2", "SELU", (DL_FUNC) &SELU); + R_RegisterCCallable("rxode2", "dSELU", (DL_FUNC) &dSELU); + R_RegisterCCallable("rxode2", "lReLU", (DL_FUNC) &lReLU); + R_RegisterCCallable("rxode2", "dlReLU", (DL_FUNC) &dlReLU); + R_RegisterCCallable("rxode2", "PReLU", (DL_FUNC) &PReLU); + R_RegisterCCallable("rxode2", "dPReLU", (DL_FUNC) &dPReLU); + R_RegisterCCallable("rxode2", "dPReLUa", (DL_FUNC) &dPReLUa); + R_RegisterCCallable("rxode2", "dPReLUa1", (DL_FUNC) &dPReLUa1); + R_RegisterCCallable("rxode2", "Swish", (DL_FUNC) &Swish); + R_RegisterCCallable("rxode2", "dSwish", (DL_FUNC) &dSwish); R_RegisterCCallable("rxode2", "powerDi", (DL_FUNC) &powerDi); R_RegisterCCallable("rxode2", "powerD", (DL_FUNC) &powerD); R_RegisterCCallable("rxode2", "powerDD", (DL_FUNC) &powerDD); diff --git a/src/utilc.c b/src/utilc.c index c9e2c7364..904ec7d62 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -596,6 +596,179 @@ double probitInv(double alpha, double low, double high) { return _powerDi(alpha, 1.0, 6, low, high); } +double ReLU(double x) { + return (x > 0.0) ? x : 0.0; +} + +double dReLU(double x) { + return (x > 0.0) ? 1.0 : 0.0; +} + +// +double GELU(double x) { + return 0.5 * x * (1.0 + erf(x * M_SQRT1_2)); +} + +double dGELU(double x) { + return 0.5 * (1.0 + erf(x * M_SQRT1_2)) + x * M_1_SQRT_2PI * exp(-0.5 * x * x); +} + +double d2GELU(double x) { + return (2.0- x*x) * exp(-0.5* x * x)*M_1_SQRT_2PI; +} + +double d3GELU(double x) { + return x * exp(-0.5 * x * x) * M_1_SQRT_2PI * (x * x - 4.0); +} + +double d4GELU(double x) { + return exp(-0.5*x*x)*M_1_SQRT_2PI*(7.0*x*x - 4.0 - x*x*x*x); +} + +double ELU(double x, double alpha) { + return (x > 0.0) ? x : (exp(x) - 1.0) * alpha; +} + +// derivative of ELU with respect to x +double dELU(double x, double alpha) { + return (x > 0.0) ? 1.0 : exp(x)*alpha; +} + +// derivative of dELU with respect to x +double d2ELU(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x)*alpha; +} + +// derivative of dELU with respect to alpha +double d2aELU(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x); +} + +// derivative of ELU with respect to alpha +double dELUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : (exp(x) - 1.0); +} +// derivative of dELAa with respect to x +double d2ELUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x); +} + +double softplus(double x) { + return log(1.0 + exp(x)); +} + +double dsoftplus(double x) { + return 1.0 / (1.0 + exp(-x)); +} + +double d2softplus(double x) { + double ex = exp(x); + return ex / ((1.0 + ex) * (1.0 + ex)); +} + +double d3softplus(double x) { + double ex = exp(-x); + double ex1 = (1.0 + ex); + return 2.0*exp(-2.0*x)/(ex1*ex1*ex1) - 1.0*ex/(ex1*ex1); +} + +double d4softplus(double x) { + double ex = exp(-x); + double ex1 = (1.0 + ex); + return 6.0*exp(-3.0*x)/(ex1*ex1*ex1*ex1) - + 6.0*exp(-2.0*x)/(ex1*ex1*ex1) + + 1.0*ex/(ex1*ex1); +} + +double SELU(double x) { +#define alpha 1.6732632423543772848170429916717 +#define scale 1.0507009873554804934193349852946 + return (x > 0.0) ? scale * x : scale * alpha * (exp(x) - 1.0); +#undef alpha +#undef scale +} + +double dSELU(double x) { +#define alpha 1.6732632423543772848170429916717 +#define scale 1.0507009873554804934193349852946 + return (x > 0.0) ? scale : scale * alpha * exp(x); +#undef alpha +#undef scale +} + +double lReLU(double x) { + return (x > 0.0) ? x : 0.01 * x; +} + +double dlReLU(double x) { + return (x > 0.0) ? 1.0 : 0.01; +} + +double PReLU(double x, double alpha) { + return (x >= 0.0) ? x : alpha * x; +} + +double dPReLU(double x, double alpha) { + return (x >= 0.0) ? 1.0 : alpha; +} + +double dPReLUa(double x, double alpha) { + return (x >= 0.0) ? 0.0 : x; +} + +double dPReLUa1(double x, double alpha) { + return (x >= 0.0) ? 0.0 : 1.0; +} + +double Swish(double x) { + return x / (1.0 + exp(-x)); +} + +double dSwish(double x) { + double ex = exp(x); + double den = 1.0 + ex; + return ex / (den * den) + x * ex / (den * den); +} + +SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { + int type = INTEGER(typeS)[0]; + int typex = TYPEOF(xS); + int lenx = Rf_length(xS); + SEXP ret = PROTECT(Rf_allocVector(REALSXP, lenx)); + for (int i = 0; i < lenx; ++i) { + double x = (typex == REALSXP) ? REAL(xS)[i] : (double)INTEGER(xS)[i]; + switch (type) { + case 1: + REAL(ret)[i] = GELU(x); + break; + case 2: + REAL(ret)[i] = ReLU(x); + break; + case 3: + REAL(ret)[i] = softplus(x); + break; + case 4: + REAL(ret)[i] = SELU(x); + break; + case 5: + REAL(ret)[i] = lReLU(x); + break; + case 6: + REAL(ret)[i] = Swish(x); + break; + case 7: + REAL(ret)[i] = dReLU(x); + break; + default: + REAL(ret)[i] = x; + break; + } + } + UNPROTECT(1); + return(ret); +} + + SEXP _rxode2_powerD(SEXP xS, SEXP lowS, SEXP highS, SEXP lambdaS, SEXP yjS, SEXP inverseS) { int typex = TYPEOF(xS); int typelow = TYPEOF(lowS); From 834bab9027d1206052d0d7290618eab18ade4b3f Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 11:42:30 -0600 Subject: [PATCH 04/16] Add GELU, lReLU, PReLU --- NAMESPACE | 5 ++++ R/gelu.R | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ R/lrelu.R | 40 ++++++++++++++++++++++++++++++ R/prelu.R | 0 man/GELU.Rd | 36 +++++++++++++++++++++++++++ man/ReLU.Rd | 6 ++++- man/dGELU.Rd | 50 +++++++++++++++++++++++++++++++++++++ man/dReLU.Rd | 6 ++++- man/dlReLU.Rd | 37 ++++++++++++++++++++++++++++ man/lReLU.Rd | 36 +++++++++++++++++++++++++++ man/reexports.Rd | 1 - src/utilc.c | 19 ++++++++++++-- 12 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 R/gelu.R create mode 100644 R/lrelu.R create mode 100644 R/prelu.R create mode 100644 man/GELU.Rd create mode 100644 man/dGELU.Rd create mode 100644 man/dlReLU.Rd create mode 100644 man/lReLU.Rd diff --git a/NAMESPACE b/NAMESPACE index ec8368b51..394479c97 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -379,8 +379,12 @@ export(binomProbs) export(boxCox) export(boxCoxInv) export(cvPost) +export(d2GELU) +export(d3GELU) +export(d4GELU) export(dReLU) export(dfWishart) +export(dlReLU) export(erf) export(et) export(etExpand) @@ -416,6 +420,7 @@ export(invWR1d) export(is.rxEt) export(is.rxSolve) export(is.rxStackData) +export(lReLU) export(label_both) export(label_context) export(label_value) diff --git a/R/gelu.R b/R/gelu.R new file mode 100644 index 000000000..661c43dbf --- /dev/null +++ b/R/gelu.R @@ -0,0 +1,64 @@ + +#' GELU activation function +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @examples +#' +#' GELU(c(-2, -1, 0, 1, 2)) +#' +#' # you can use rxode2 as well +#' r <- rxode2({r = GELU(x)}) +#' et <- et(c(-2, -1, 0, 1, 2)) +#' rxSolve(r, et) +#' +GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 1L) +} + + +#' Derivatives of GELU +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @examples +#' dGELU(c(-2, -1, 0, 1, 2)) +#' d2GELU(c(-2, -1, 0, 1, 2)) +#' d3GELU(c(-2, -1, 0, 1, 2)) +#' d4GELU(c(-2, -1, 0, 1, 2)) +#' # you can use rxode2 as well +#' r <- rxode2({ +#' r1 <- dGELU(x) +#' r2 <- d2GELU(x) +#' r3 <- d3GELU(x) +#' r4 <- d4GELU(x) +#' }) +#' et <- et(c(-2, -1, 0, 1, 2)) +#' rxSolve(r, et) +dGELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 9L) +} + +#' @rdname dGELU +#' @export +d2GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 10L) +} + +#' @rdname dGELU +#' @export +d3GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 11L) +} + +#' @rdname dGELU +#' @export +d4GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 12L) +} diff --git a/R/lrelu.R b/R/lrelu.R new file mode 100644 index 000000000..306584ab0 --- /dev/null +++ b/R/lrelu.R @@ -0,0 +1,40 @@ +#' Leaky ReLU activation function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @examples +#' +#' lReLU(c(-1, 0, 1)) +#' +#' # Can use in rxode2 as well +#' +#' r <- rxode2({r <- lReLU(time)}) +#' e <- et(c(-1, 0, 1)) +#' rxSolve(r, e) +lReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 5L) +} + +#' Derivative of Leaky ReLU activation function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @examples +#' +#' dlReLU(c(-1, 0, 1)) +#' +#' # Can use in rxode2 as well +#' +#' r <- rxode2({r <- dlReLU(time)}) +#' e <- et(c(-1, 0, 1)) +#' rxSolve(r, e) +#' +dlReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 8L) +} diff --git a/R/prelu.R b/R/prelu.R new file mode 100644 index 000000000..e69de29bb diff --git a/man/GELU.Rd b/man/GELU.Rd new file mode 100644 index 000000000..c0351f240 --- /dev/null +++ b/man/GELU.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gelu.R +\name{GELU} +\alias{GELU} +\title{GELU activation function} +\usage{ +GELU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +GELU activation function +} +\examples{ + +GELU(c(-2, -1, 0, 1, 2)) + +# you can use rxode2 as well +r <- rxode2({r = GELU(x)}) +et <- et(c(-2, -1, 0, 1, 2)) +rxSolve(r, et) + +} +\seealso{ +Other Activation Functions: +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()}, +\code{\link{lReLU}()} +} +\concept{Activation Functions} diff --git a/man/ReLU.Rd b/man/ReLU.Rd index 5cfd10bb3..9eb7a9fa8 100644 --- a/man/ReLU.Rd +++ b/man/ReLU.Rd @@ -34,7 +34,11 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: -\code{\link{dReLU}()} +\code{\link{GELU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()}, +\code{\link{lReLU}()} } \author{ Matthew Fidler diff --git a/man/dGELU.Rd b/man/dGELU.Rd new file mode 100644 index 000000000..dc1cfc694 --- /dev/null +++ b/man/dGELU.Rd @@ -0,0 +1,50 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gelu.R +\name{dGELU} +\alias{dGELU} +\alias{d2GELU} +\alias{d3GELU} +\alias{d4GELU} +\title{Derivatives of GELU} +\usage{ +dGELU(x) + +d2GELU(x) + +d3GELU(x) + +d4GELU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Derivatives of GELU +} +\examples{ +dGELU(c(-2, -1, 0, 1, 2)) +d2GELU(c(-2, -1, 0, 1, 2)) +d3GELU(c(-2, -1, 0, 1, 2)) +d4GELU(c(-2, -1, 0, 1, 2)) +# you can use rxode2 as well +r <- rxode2({ + r1 <- dGELU(x) + r2 <- d2GELU(x) + r3 <- d3GELU(x) + r4 <- d4GELU(x) +}) +et <- et(c(-2, -1, 0, 1, 2)) +rxSolve(r, et) +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()}, +\code{\link{lReLU}()} +} +\concept{Activation Functions} diff --git a/man/dReLU.Rd b/man/dReLU.Rd index 7537e5a04..6f86da41a 100644 --- a/man/dReLU.Rd +++ b/man/dReLU.Rd @@ -32,6 +32,10 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: -\code{\link{ReLU}()} +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dlReLU}()}, +\code{\link{lReLU}()} } \concept{Activation Functions} diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd new file mode 100644 index 000000000..749a1fcbe --- /dev/null +++ b/man/dlReLU.Rd @@ -0,0 +1,37 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/lrelu.R +\name{dlReLU} +\alias{dlReLU} +\title{Derivative of Leaky ReLU activation function} +\usage{ +dlReLU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Derivative of Leaky ReLU activation function +} +\examples{ + +dlReLU(c(-1, 0, 1)) + +# Can use in rxode2 as well + +r <- rxode2({r <- dlReLU(time)}) +e <- et(c(-1, 0, 1)) +rxSolve(r, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{lReLU}()} +} +\concept{Activation Functions} diff --git a/man/lReLU.Rd b/man/lReLU.Rd new file mode 100644 index 000000000..ef7f85013 --- /dev/null +++ b/man/lReLU.Rd @@ -0,0 +1,36 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/lrelu.R +\name{lReLU} +\alias{lReLU} +\title{Leaky ReLU activation function} +\usage{ +lReLU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Leaky ReLU activation function +} +\examples{ + +lReLU(c(-1, 0, 1)) + +# Can use in rxode2 as well + +r <- rxode2({r <- lReLU(time)}) +e <- et(c(-1, 0, 1)) +rxSolve(r, e) +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()} +} +\concept{Activation Functions} diff --git a/man/reexports.Rd b/man/reexports.Rd index 2857ff16d..8cba3d62e 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,4 +38,3 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} -\value{ Inherited from parent routine } diff --git a/src/utilc.c b/src/utilc.c index 904ec7d62..03cb187a5 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -697,11 +697,11 @@ double dSELU(double x) { } double lReLU(double x) { - return (x > 0.0) ? x : 0.01 * x; + return (x >= 0.0) ? x : 0.01 * x; } double dlReLU(double x) { - return (x > 0.0) ? 1.0 : 0.01; + return (x >= 0.0) ? 1.0 : 0.01; } double PReLU(double x, double alpha) { @@ -759,6 +759,21 @@ SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { case 7: REAL(ret)[i] = dReLU(x); break; + case 8: + REAL(ret)[i] = dlReLU(x); + break; + case 9: + REAL(ret)[i] = dGELU(x); + break; + case 10: + REAL(ret)[i] = d2GELU(x); + break; + case 11: + REAL(ret)[i] = d3GELU(x); + break; + case 12: + REAL(ret)[i] = d4GELU(x); + break; default: REAL(ret)[i] = x; break; From 0b6929b4d1501267e74fb4f1e7ab165b56e78498 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 14:34:14 -0600 Subject: [PATCH 05/16] Translate the d4GELU to a derivative --- R/symengine.R | 20 ++++++++++++++++++++ man/reexports.Rd | 1 + tests/testthat/test-dsl.R | 2 ++ 3 files changed, 23 insertions(+) diff --git a/R/symengine.R b/R/symengine.R index 1bd0e3753..448bc77e6 100644 --- a/R/symengine.R +++ b/R/symengine.R @@ -1197,6 +1197,23 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, } } +.rxToSEd4GELU <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "erf") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + return( + paste0("exp(-(", .x, ")^2/2)*(7*(", .x, ")^2 - 4 - (", .x, ")^4)/sqrt(2*pi)") + ) + } else { + stop("'d4GELU' can only take 1 argument", call. = FALSE) + } + +} + .rxToSETransit <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { if (length(x) == 4) { ## transit(n, mtt, bio) @@ -1256,6 +1273,7 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, return(.rxToSEMax(c(.ret, .xrest), min=min)) } + .rxToSECall <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { if (identical(x[[1]], quote(`(`))) { return(paste0("(", .rxToSE(x[[2]], envir = envir), ")")) @@ -1315,6 +1333,8 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, if (length(x) != 2) stop("abs only takes 1 argument", call.=FALSE) .r <- .rxToSE(x[[2]], envir = envir) return(paste0("(2.0*(", .r, ")*rxGt(", .r, ",0.0)-(", .r, "))")) + } else if (identical(x[[1]], quote(`d4GELU`))) { + return(.rxToSEd4GELU(x, envir = envir, progress = progress, isEnv=isEnv)) } else { if (length(x[[1]]) == 1) { .x1 <- as.character(x[[1]]) diff --git a/man/reexports.Rd b/man/reexports.Rd index 8cba3d62e..2857ff16d 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,3 +38,4 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} +\value{ Inherited from parent routine } diff --git a/tests/testthat/test-dsl.R b/tests/testthat/test-dsl.R index cd0e3e14c..239c5f330 100644 --- a/tests/testthat/test-dsl.R +++ b/tests/testthat/test-dsl.R @@ -269,6 +269,8 @@ rxTest({ expect_equal(rxFromSE("Derivative(d2GELU(x), x)"), "d3GELU(x)") expect_equal(rxFromSE("Derivative(d3GELU(x), x)"), "d4GELU(x)") + expect_equal(rxToSE("d4GELU(x)"), "exp(-(x)^2/2)*(7*(x)^2 - 4 - (x)^4)/sqrt(2*pi)") + expect_equal(rxFromSE("Derivative(ELU(x, alpha), x)"), "dELU(x, alpha)") expect_equal(rxFromSE("Derivative(ELU(x, alpha), alpha)"), "dELUa(x, alpha)") From 50368f742f91ae3fa789d1bd689bd4607e3fb201 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 15:36:56 -0600 Subject: [PATCH 06/16] d4softplus --- NAMESPACE | 2 ++ R/softplus.R | 75 +++++++++++++++++++++++++++++++++++++++ R/symengine.R | 22 +++++++++++- man/GELU.Rd | 4 ++- man/ReLU.Rd | 4 ++- man/dGELU.Rd | 4 ++- man/dReLU.Rd | 4 ++- man/dlReLU.Rd | 4 ++- man/dsoftplus.Rd | 61 +++++++++++++++++++++++++++++++ man/lReLU.Rd | 4 ++- man/softplus.Rd | 46 ++++++++++++++++++++++++ src/utilc.c | 14 +++++++- tests/testthat/test-dsl.R | 3 ++ 13 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 R/softplus.R create mode 100644 man/dsoftplus.Rd create mode 100644 man/softplus.Rd diff --git a/NAMESPACE b/NAMESPACE index 394479c97..d4c6f2f37 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -385,6 +385,7 @@ export(d4GELU) export(dReLU) export(dfWishart) export(dlReLU) +export(dsoftplus) export(erf) export(et) export(etExpand) @@ -644,6 +645,7 @@ export(scale_y_continuous) export(scale_y_date) export(scale_y_discrete) export(setRxThreads) +export(softplus) export(stat_amt) export(stat_cens) export(swapMatListWithCube) diff --git a/R/softplus.R b/R/softplus.R new file mode 100644 index 000000000..ed32d5386 --- /dev/null +++ b/R/softplus.R @@ -0,0 +1,75 @@ +#' Softplus Activation Function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @author Matthew L. Fidler +#' @examples +#' +#' softplus(c(-1, 0, 1, 2)) +#' +#' # You can use rxode2 too: +#' +#' r <- rxode2({ +#' s <- softplus(x) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(r, e) +#' +softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 3L) +} + +#' Default Softplus Activation Function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @author Matthew L. Fidler +#' @examples +#' +#' dsoftplus(c(-1, 0, 1, 2)) +#' d2softplus(c(-1, 0, 1, 2)) +#' d3softplus(c(-1, 0, 1, 2)) +#' d4softplus(c(-1, 0, 1, 2)) +#' +#' # You can use rxode2 too: +#' +#' r <- rxode2({ +#' s1 <- dsoftplus(time) +#' s2 <- d2softplus(time) +#' s3 <- d3softplus(time) +#' s4 <- d4softplus(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(r, e) +#' +dsoftplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 13L) +} + +#' @rdname dsoftplus +d2softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 14L) +} + +#' @rdname dsoftplus +d3softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 15L) +} + +#' @rdname dsoftplus +d4softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 16L) +} diff --git a/R/symengine.R b/R/symengine.R index 448bc77e6..e9a7392e2 100644 --- a/R/symengine.R +++ b/R/symengine.R @@ -1201,7 +1201,7 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, if (length(x) == 2) { if (isEnv) { .lastCall <- envir$..curCall - envir$..curCall <- c(envir$..curCall, "erf") + envir$..curCall <- c(envir$..curCall, "d4GELU") } .x <- .rxToSE(x[[2]], envir = envir) if (isEnv) envir$..curCall <- .lastCall @@ -1211,7 +1211,25 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, } else { stop("'d4GELU' can only take 1 argument", call. = FALSE) } +} +.rxToSEd4softplus <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "d4softplus") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + .ex1 <- paste0("(1.0 + exp(-(", .x, ")))") + return( + paste0("6.0*exp(-3.0*(", .x, "))/((", .ex1, + ")^4) - 6.0*exp(-2.0*(", .x, "))/((", .ex1, + ")^3) + exp(-(", .x, "))/((", .ex1, ")^2)") + ) + } else { + stop("'d4GELU' can only take 1 argument", call. = FALSE) + } } .rxToSETransit <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { @@ -1335,6 +1353,8 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, return(paste0("(2.0*(", .r, ")*rxGt(", .r, ",0.0)-(", .r, "))")) } else if (identical(x[[1]], quote(`d4GELU`))) { return(.rxToSEd4GELU(x, envir = envir, progress = progress, isEnv=isEnv)) + } else if (identical(x[[1]], quote(`d4softplus`))) { + return(.rxToSEd4softplus(x, envir = envir, progress = progress, isEnv=isEnv)) } else { if (length(x[[1]]) == 1) { .x1 <- as.character(x[[1]]) diff --git a/man/GELU.Rd b/man/GELU.Rd index c0351f240..8678c55c5 100644 --- a/man/GELU.Rd +++ b/man/GELU.Rd @@ -31,6 +31,8 @@ Other Activation Functions: \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dlReLU}()}, -\code{\link{lReLU}()} +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} } \concept{Activation Functions} diff --git a/man/ReLU.Rd b/man/ReLU.Rd index 9eb7a9fa8..d5c7ce88f 100644 --- a/man/ReLU.Rd +++ b/man/ReLU.Rd @@ -38,7 +38,9 @@ Other Activation Functions: \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dlReLU}()}, -\code{\link{lReLU}()} +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} } \author{ Matthew Fidler diff --git a/man/dGELU.Rd b/man/dGELU.Rd index dc1cfc694..184e8d3e5 100644 --- a/man/dGELU.Rd +++ b/man/dGELU.Rd @@ -45,6 +45,8 @@ Other Activation Functions: \code{\link{ReLU}()}, \code{\link{dReLU}()}, \code{\link{dlReLU}()}, -\code{\link{lReLU}()} +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} } \concept{Activation Functions} diff --git a/man/dReLU.Rd b/man/dReLU.Rd index 6f86da41a..2601f3311 100644 --- a/man/dReLU.Rd +++ b/man/dReLU.Rd @@ -36,6 +36,8 @@ Other Activation Functions: \code{\link{ReLU}()}, \code{\link{dGELU}()}, \code{\link{dlReLU}()}, -\code{\link{lReLU}()} +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} } \concept{Activation Functions} diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd index 749a1fcbe..53acf9d03 100644 --- a/man/dlReLU.Rd +++ b/man/dlReLU.Rd @@ -32,6 +32,8 @@ Other Activation Functions: \code{\link{ReLU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, -\code{\link{lReLU}()} +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} } \concept{Activation Functions} diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd new file mode 100644 index 000000000..2322344cc --- /dev/null +++ b/man/dsoftplus.Rd @@ -0,0 +1,61 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/softplus.R +\name{dsoftplus} +\alias{dsoftplus} +\alias{d2softplus} +\alias{d3softplus} +\alias{d4softplus} +\title{Default Softplus Activation Function} +\usage{ +dsoftplus(x) + +d2softplus(x) + +d3softplus(x) + +d4softplus(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Default Softplus Activation Function +} +\examples{ + +dsoftplus(c(-1, 0, 1, 2)) +d2softplus(c(-1, 0, 1, 2)) +d3softplus(c(-1, 0, 1, 2)) +d4softplus(c(-1, 0, 1, 2)) + +# You can use rxode2 too: + +r <- rxode2({ + s1 <- dsoftplus(time) + s2 <- d2softplus(time) + s3 <- d3softplus(time) + s4 <- d4softplus(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(r, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/man/lReLU.Rd b/man/lReLU.Rd index ef7f85013..6b940a513 100644 --- a/man/lReLU.Rd +++ b/man/lReLU.Rd @@ -31,6 +31,8 @@ Other Activation Functions: \code{\link{ReLU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, -\code{\link{dlReLU}()} +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{softplus}()} } \concept{Activation Functions} diff --git a/man/softplus.Rd b/man/softplus.Rd new file mode 100644 index 000000000..0f7e7efd1 --- /dev/null +++ b/man/softplus.Rd @@ -0,0 +1,46 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/softplus.R +\name{softplus} +\alias{softplus} +\title{Softplus Activation Function} +\usage{ +softplus(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Softplus Activation Function +} +\examples{ + +softplus(c(-1, 0, 1, 2)) + +# You can use rxode2 too: + +r <- rxode2({ + s <- softplus(x) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(r, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/src/utilc.c b/src/utilc.c index 03cb187a5..afa7464d7 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -677,7 +677,7 @@ double d4softplus(double x) { double ex1 = (1.0 + ex); return 6.0*exp(-3.0*x)/(ex1*ex1*ex1*ex1) - 6.0*exp(-2.0*x)/(ex1*ex1*ex1) + - 1.0*ex/(ex1*ex1); + ex/(ex1*ex1); } double SELU(double x) { @@ -774,6 +774,18 @@ SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { case 12: REAL(ret)[i] = d4GELU(x); break; + case 13: + REAL(ret)[i] = dsoftplus(x); + break; + case 14: + REAL(ret)[i] = d2softplus(x); + break; + case 15: + REAL(ret)[i] = d3softplus(x); + break; + case 16: + REAL(ret)[i] = d4softplus(x); + break; default: REAL(ret)[i] = x; break; diff --git a/tests/testthat/test-dsl.R b/tests/testthat/test-dsl.R index 239c5f330..a43d1e9a1 100644 --- a/tests/testthat/test-dsl.R +++ b/tests/testthat/test-dsl.R @@ -291,6 +291,9 @@ rxTest({ expect_equal(rxFromSE("Derivative(d2softplus(x), x)"), "d3softplus(x)") expect_equal(rxFromSE("Derivative(d3softplus(x), x)"), "d4softplus(x)") + expect_equal(rxToSE("d4softplus(x)"), + "6.0*exp(-3.0*(x))/(((1.0 + exp(-(x))))^4) - 6.0*exp(-2.0*(x))/(((1.0 + exp(-(x))))^3) + exp(-(x))/(((1.0 + exp(-(x))))^2)") + expect_equal(rxFromSE("Derivative(SELU(x), x)"), "dSELU(x)") expect_equal(rxFromSE("Derivative(lReLU(x), x)"), "dlReLU(x)") From 6fd2166b2b34600a966d82a50f39e207cfe62af2 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 15:52:17 -0600 Subject: [PATCH 07/16] Add SELU --- NAMESPACE | 2 ++ R/selu.R | 51 +++++++++++++++++++++++++++++++++++++++ R/symengine.R | 21 +++++++++++++++- man/GELU.Rd | 2 ++ man/ReLU.Rd | 2 ++ man/SELU.Rd | 51 +++++++++++++++++++++++++++++++++++++++ man/dGELU.Rd | 2 ++ man/dReLU.Rd | 2 ++ man/dSELU.Rd | 44 +++++++++++++++++++++++++++++++++ man/dlReLU.Rd | 2 ++ man/dsoftplus.Rd | 2 ++ man/lReLU.Rd | 2 ++ man/reexports.Rd | 1 - man/softplus.Rd | 2 ++ src/utilc.c | 3 +++ tests/testthat/test-dsl.R | 3 +++ 16 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 R/selu.R create mode 100644 man/SELU.Rd create mode 100644 man/dSELU.Rd diff --git a/NAMESPACE b/NAMESPACE index d4c6f2f37..e5f36a232 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -345,6 +345,7 @@ export(.useUtf) export(.vecDf) export(ReLU) export(RxODE) +export(SELU) export(add.dosing) export(add.sampling) export(aes) @@ -383,6 +384,7 @@ export(d2GELU) export(d3GELU) export(d4GELU) export(dReLU) +export(dSELU) export(dfWishart) export(dlReLU) export(dsoftplus) diff --git a/R/selu.R b/R/selu.R new file mode 100644 index 000000000..07a95d733 --- /dev/null +++ b/R/selu.R @@ -0,0 +1,51 @@ +#' Scaled Exponential Linear Unit (SELU) Activation Function +#' +#' The Scaled Exponential Linear Unit (SELU) activation function is a +#' piecewise linear function that allows for negative values to be +#' passed through the network. +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' SELU(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=SELU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +SELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 4L) +} + +#' Derivative of the Scaled Exponential Linear Unit (SELU) Activation Function +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative of the SELU function +#' has been applied to each element of `x`. +#' @author Matthew Fidler +#' @family Activation Functions +#' @export +#' @examples +#' dSELU(c(-1, 0, 1, 2)) +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=dSELU(time) +#' }) +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(x, e) +dSELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 17L) +} diff --git a/R/symengine.R b/R/symengine.R index e9a7392e2..44bfb9166 100644 --- a/R/symengine.R +++ b/R/symengine.R @@ -1228,8 +1228,25 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, ")^3) + exp(-(", .x, "))/((", .ex1, ")^2)") ) } else { - stop("'d4GELU' can only take 1 argument", call. = FALSE) + stop("'d4softplus' can only take 1 argument", call. = FALSE) + } +} + +.rxToSEdSELU <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "dSELU") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + return( + paste0("(rxGt(", .x, ", 0)*1.0507009873554804934193349852946 + 1.0507009873554804934193349852946*1.6732632423543772848170429916717*exp(", .x, ")*rxLeq(", .x, ", 0))") + ) + } else { + stop("'dSELU' can only take 1 argument", call. = FALSE) } + } .rxToSETransit <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { @@ -1355,6 +1372,8 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, return(.rxToSEd4GELU(x, envir = envir, progress = progress, isEnv=isEnv)) } else if (identical(x[[1]], quote(`d4softplus`))) { return(.rxToSEd4softplus(x, envir = envir, progress = progress, isEnv=isEnv)) + } else if (identical(x[[1]], quote(`dSELU`))) { + .rxToSEdSELU(x, envir=envir, progress=progress, isEnv=isEnv) } else { if (length(x[[1]]) == 1) { .x1 <- as.character(x[[1]]) diff --git a/man/GELU.Rd b/man/GELU.Rd index 8678c55c5..a49b02a02 100644 --- a/man/GELU.Rd +++ b/man/GELU.Rd @@ -28,8 +28,10 @@ rxSolve(r, et) \seealso{ Other Activation Functions: \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/ReLU.Rd b/man/ReLU.Rd index d5c7ce88f..4afde61ee 100644 --- a/man/ReLU.Rd +++ b/man/ReLU.Rd @@ -35,8 +35,10 @@ rxSolve(x, e) \seealso{ Other Activation Functions: \code{\link{GELU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/SELU.Rd b/man/SELU.Rd new file mode 100644 index 000000000..d8a48afb1 --- /dev/null +++ b/man/SELU.Rd @@ -0,0 +1,51 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/selu.R +\name{SELU} +\alias{SELU} +\title{Scaled Exponential Linear Unit (SELU) Activation Function} +\usage{ +SELU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +The Scaled Exponential Linear Unit (SELU) activation function is a +piecewise linear function that allows for negative values to be +passed through the network. +} +\examples{ + +SELU(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r=SELU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dGELU.Rd b/man/dGELU.Rd index 184e8d3e5..7a372358e 100644 --- a/man/dGELU.Rd +++ b/man/dGELU.Rd @@ -43,7 +43,9 @@ rxSolve(r, et) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/dReLU.Rd b/man/dReLU.Rd index 2601f3311..c906985da 100644 --- a/man/dReLU.Rd +++ b/man/dReLU.Rd @@ -34,7 +34,9 @@ rxSolve(x, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/dSELU.Rd b/man/dSELU.Rd new file mode 100644 index 000000000..af902d8c7 --- /dev/null +++ b/man/dSELU.Rd @@ -0,0 +1,44 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/selu.R +\name{dSELU} +\alias{dSELU} +\title{Derivative of the Scaled Exponential Linear Unit (SELU) Activation Function} +\usage{ +dSELU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative of the SELU function +has been applied to each element of \code{x}. +} +\description{ +Derivative of the Scaled Exponential Linear Unit (SELU) Activation Function +} +\examples{ +dSELU(c(-1, 0, 1, 2)) +# Can also be used in rxode2: +x <- rxode2({ + r=dSELU(time) +}) +e <- et(c(-1, 0, 1, 2)) +rxSolve(x, e) +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd index 53acf9d03..a760dd2a8 100644 --- a/man/dlReLU.Rd +++ b/man/dlReLU.Rd @@ -30,8 +30,10 @@ rxSolve(r, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, \code{\link{softplus}()} diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd index 2322344cc..faf731853 100644 --- a/man/dsoftplus.Rd +++ b/man/dsoftplus.Rd @@ -49,8 +49,10 @@ rxSolve(r, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{lReLU}()}, \code{\link{softplus}()} diff --git a/man/lReLU.Rd b/man/lReLU.Rd index 6b940a513..3814b387d 100644 --- a/man/lReLU.Rd +++ b/man/lReLU.Rd @@ -29,8 +29,10 @@ rxSolve(r, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{softplus}()} diff --git a/man/reexports.Rd b/man/reexports.Rd index 2857ff16d..8cba3d62e 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,4 +38,3 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} -\value{ Inherited from parent routine } diff --git a/man/softplus.Rd b/man/softplus.Rd index 0f7e7efd1..a171bf919 100644 --- a/man/softplus.Rd +++ b/man/softplus.Rd @@ -34,8 +34,10 @@ rxSolve(r, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{SELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSELU}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()} diff --git a/src/utilc.c b/src/utilc.c index afa7464d7..7a8c81a6f 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -786,6 +786,9 @@ SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { case 16: REAL(ret)[i] = d4softplus(x); break; + case 17: + REAL(ret)[i] = dSELU(x); + break; default: REAL(ret)[i] = x; break; diff --git a/tests/testthat/test-dsl.R b/tests/testthat/test-dsl.R index a43d1e9a1..484bfe82e 100644 --- a/tests/testthat/test-dsl.R +++ b/tests/testthat/test-dsl.R @@ -296,6 +296,9 @@ rxTest({ expect_equal(rxFromSE("Derivative(SELU(x), x)"), "dSELU(x)") + expect_equal(rxToSE("dSELU(x)"), + "(rxGt(x, 0)*1.0507009873554804934193349852946 + 1.0507009873554804934193349852946*1.6732632423543772848170429916717*exp(x)*rxLeq(x, 0))") + expect_equal(rxFromSE("Derivative(lReLU(x), x)"), "dlReLU(x)") expect_equal(rxFromSE("Derivative(dlReLU(x), x)"), "0") From 5f20ac94e0ebe418dd48e2affd2a31a94d6044c9 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 16:10:21 -0600 Subject: [PATCH 08/16] Swish --- NAMESPACE | 2 ++ R/swish.R | 58 ++++++++++++++++++++++++++++++++++++++++++++++++ man/GELU.Rd | 2 ++ man/ReLU.Rd | 2 ++ man/SELU.Rd | 2 ++ man/Swish.Rd | 54 ++++++++++++++++++++++++++++++++++++++++++++ man/dGELU.Rd | 2 ++ man/dReLU.Rd | 2 ++ man/dSELU.Rd | 2 ++ man/dSwish.Rd | 47 +++++++++++++++++++++++++++++++++++++++ man/dlReLU.Rd | 2 ++ man/dsoftplus.Rd | 2 ++ man/lReLU.Rd | 2 ++ man/reexports.Rd | 1 + man/softplus.Rd | 2 ++ src/utilc.c | 3 +++ 16 files changed, 185 insertions(+) create mode 100644 R/swish.R create mode 100644 man/Swish.Rd create mode 100644 man/dSwish.Rd diff --git a/NAMESPACE b/NAMESPACE index e5f36a232..4de755d14 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -346,6 +346,7 @@ export(.vecDf) export(ReLU) export(RxODE) export(SELU) +export(Swish) export(add.dosing) export(add.sampling) export(aes) @@ -385,6 +386,7 @@ export(d3GELU) export(d4GELU) export(dReLU) export(dSELU) +export(dSwish) export(dfWishart) export(dlReLU) export(dsoftplus) diff --git a/R/swish.R b/R/swish.R new file mode 100644 index 000000000..737ac4a1e --- /dev/null +++ b/R/swish.R @@ -0,0 +1,58 @@ +#' Switch Activation Function +#' +#' The switch activation function is defined as: +#' +#' \deqn{f(x) = x \cdot \text{sigmoid}(x)} +#' +#' @family Activation Functions +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' +#' @author Matthew Fidler +#' +#' @export +#' +#' @examples +#' +#' Swish(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r<- Swish(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +Swish <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 6L) +} + +#' Derivative of the Swish Activation Function +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative of the SELU function +#' has been applied to each element of `x`. +#' @author Matthew Fidler +#' @family Activation Functions +#' @export +#' @examples +#' dSwish(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r <- dSwish(time) +#' }) +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(x, e) +dSwish <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 18L) +} diff --git a/man/GELU.Rd b/man/GELU.Rd index a49b02a02..6159d4c45 100644 --- a/man/GELU.Rd +++ b/man/GELU.Rd @@ -29,9 +29,11 @@ rxSolve(r, et) Other Activation Functions: \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/ReLU.Rd b/man/ReLU.Rd index 4afde61ee..ba3b4ad7f 100644 --- a/man/ReLU.Rd +++ b/man/ReLU.Rd @@ -36,9 +36,11 @@ rxSolve(x, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/SELU.Rd b/man/SELU.Rd index d8a48afb1..9e81f45ec 100644 --- a/man/SELU.Rd +++ b/man/SELU.Rd @@ -37,9 +37,11 @@ rxSolve(x, e) Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/Swish.Rd b/man/Swish.Rd new file mode 100644 index 000000000..4ebce998b --- /dev/null +++ b/man/Swish.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/swish.R +\name{Swish} +\alias{Swish} +\title{Switch Activation Function} +\usage{ +Swish(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +The switch activation function is defined as: +} +\details{ +\deqn{f(x) = x \cdot \text{sigmoid}(x)} +} +\examples{ + +Swish(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r<- Swish(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dGELU.Rd b/man/dGELU.Rd index 7a372358e..4b98e80e9 100644 --- a/man/dGELU.Rd +++ b/man/dGELU.Rd @@ -44,8 +44,10 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/dReLU.Rd b/man/dReLU.Rd index c906985da..ef7856c9c 100644 --- a/man/dReLU.Rd +++ b/man/dReLU.Rd @@ -35,8 +35,10 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/dSELU.Rd b/man/dSELU.Rd index af902d8c7..698e25962 100644 --- a/man/dSELU.Rd +++ b/man/dSELU.Rd @@ -31,8 +31,10 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, diff --git a/man/dSwish.Rd b/man/dSwish.Rd new file mode 100644 index 000000000..ed1e3abf0 --- /dev/null +++ b/man/dSwish.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/swish.R +\name{dSwish} +\alias{dSwish} +\title{Derivative of the Swish Activation Function} +\usage{ +dSwish(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative of the SELU function +has been applied to each element of \code{x}. +} +\description{ +Derivative of the Swish Activation Function +} +\examples{ +dSwish(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r <- dSwish(time) +}) +e <- et(c(-1, 0, 1, 2)) +rxSolve(x, e) +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd index a760dd2a8..858a1ba92 100644 --- a/man/dlReLU.Rd +++ b/man/dlReLU.Rd @@ -31,9 +31,11 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()}, \code{\link{softplus}()} diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd index faf731853..56c4d8357 100644 --- a/man/dsoftplus.Rd +++ b/man/dsoftplus.Rd @@ -50,9 +50,11 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{lReLU}()}, \code{\link{softplus}()} diff --git a/man/lReLU.Rd b/man/lReLU.Rd index 3814b387d..c489b861e 100644 --- a/man/lReLU.Rd +++ b/man/lReLU.Rd @@ -30,9 +30,11 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{softplus}()} diff --git a/man/reexports.Rd b/man/reexports.Rd index 8cba3d62e..2857ff16d 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,3 +38,4 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} +\value{ Inherited from parent routine } diff --git a/man/softplus.Rd b/man/softplus.Rd index a171bf919..a1d5e102b 100644 --- a/man/softplus.Rd +++ b/man/softplus.Rd @@ -35,9 +35,11 @@ Other Activation Functions: \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{Swish}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, +\code{\link{dSwish}()}, \code{\link{dlReLU}()}, \code{\link{dsoftplus}()}, \code{\link{lReLU}()} diff --git a/src/utilc.c b/src/utilc.c index 7a8c81a6f..65d3959a8 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -789,6 +789,9 @@ SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { case 17: REAL(ret)[i] = dSELU(x); break; + case 18: + REAL(ret)[i] = dSwish(x); + break; default: REAL(ret)[i] = x; break; From ef244dfabdb70170f277e0aa044bd6febb8df191 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 16:25:34 -0600 Subject: [PATCH 09/16] Add dSwish --- R/symengine.R | 22 +++++++++++++++++++++- src/utilc.c | 6 +++--- tests/testthat/test-dsl.R | 3 +++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/R/symengine.R b/R/symengine.R index 44bfb9166..07bf6babc 100644 --- a/R/symengine.R +++ b/R/symengine.R @@ -718,7 +718,6 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, } } - .rxToSECurlyBrace <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { .x2 <- x[-1] if (progress) { @@ -1249,6 +1248,25 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, } +.rxToSEdSwish <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "dSwish") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + # x*exp(-x)/(1.0 + exp(-x))^2 + (1.0 + exp(-x))^(-1); + return( + paste0("((", .x, ")*exp(-(", .x, "))/(1.0 + exp(-(", .x, + ")))^2 + 1.0/(1.0 + exp(-(", .x, ")))") + ) + } else { + stop("'dSwish' can only take 1 argument", call. = FALSE) + } + +} + .rxToSETransit <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { if (length(x) == 4) { ## transit(n, mtt, bio) @@ -1374,6 +1392,8 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, return(.rxToSEd4softplus(x, envir = envir, progress = progress, isEnv=isEnv)) } else if (identical(x[[1]], quote(`dSELU`))) { .rxToSEdSELU(x, envir=envir, progress=progress, isEnv=isEnv) + } else if (identical(x[[1]], quote(`dSwish`))) { + .rxToSEdSwish(x, envir=envir, progress=progress, isEnv=isEnv) } else { if (length(x[[1]]) == 1) { .x1 <- as.character(x[[1]]) diff --git a/src/utilc.c b/src/utilc.c index 65d3959a8..800a8aa2e 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -725,9 +725,9 @@ double Swish(double x) { } double dSwish(double x) { - double ex = exp(x); - double den = 1.0 + ex; - return ex / (den * den) + x * ex / (den * den); + double ex = exp(-x); + double den = (1.0 + ex); + return x*ex/(den*den) + 1.0/den; } SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { diff --git a/tests/testthat/test-dsl.R b/tests/testthat/test-dsl.R index 484bfe82e..4ad417168 100644 --- a/tests/testthat/test-dsl.R +++ b/tests/testthat/test-dsl.R @@ -316,6 +316,9 @@ rxTest({ expect_equal(rxFromSE("Derivative(Swish(x), x)"), "dSwish(x)") + expect_equal(rxToSE("dSwish(x)"), + "((x)*exp(-(x))/(1.0 + exp(-(x)))^2 + 1.0/(1.0 + exp(-(x)))") + }) test_that("logic tests", { From ee15cce825d06ef944e18169fdfbb8a5e7612191 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 16:47:47 -0600 Subject: [PATCH 10/16] Add ELU --- NAMESPACE | 18 ++++++--- R/elu.R | 100 +++++++++++++++++++++++++++++++++++++++++++++++ R/selu.R | 4 -- man/ELU.Rd | 53 +++++++++++++++++++++++++ man/GELU.Rd | 2 + man/ReLU.Rd | 2 + man/SELU.Rd | 6 +-- man/Swish.Rd | 2 + man/dELU.Rd | 73 ++++++++++++++++++++++++++++++++++ man/dGELU.Rd | 2 + man/dReLU.Rd | 2 + man/dSELU.Rd | 2 + man/dSwish.Rd | 2 + man/dlReLU.Rd | 2 + man/dsoftplus.Rd | 2 + man/lReLU.Rd | 2 + man/reexports.Rd | 1 - man/rxSolve.Rd | 12 +++--- man/softplus.Rd | 2 + src/init.c | 2 + src/utilc.c | 51 +++++++++++++++++++++++- 21 files changed, 321 insertions(+), 21 deletions(-) create mode 100644 R/elu.R create mode 100644 man/ELU.Rd create mode 100644 man/dELU.Rd diff --git a/NAMESPACE b/NAMESPACE index 4de755d14..0c8999c05 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -95,12 +95,6 @@ S3method(modelExtract,rxode2) S3method(plot,rxSolve) S3method(plot,rxSolveConfint1) S3method(plot,rxSolveConfint2) -S3method(predict,"function") -S3method(predict,rxEt) -S3method(predict,rxParams) -S3method(predict,rxSolve) -S3method(predict,rxUi) -S3method(predict,rxode2) S3method(print,boundInits) S3method(print,boundParams) S3method(print,rxC) @@ -343,6 +337,7 @@ export(.udfExists) export(.udfMd5Info) export(.useUtf) export(.vecDf) +export(ELU) export(ReLU) export(RxODE) export(SELU) @@ -381,9 +376,14 @@ export(binomProbs) export(boxCox) export(boxCoxInv) export(cvPost) +export(d2ELU) +export(d2ELUa) export(d2GELU) +export(d2aELU) export(d3GELU) export(d4GELU) +export(dELU) +export(dELUa) export(dReLU) export(dSELU) export(dSwish) @@ -463,6 +463,12 @@ export(model) export(modelExtract) export(odeMethodToInt) export(phi) +export(predict.function) +export(predict.rxEt) +export(predict.rxParams) +export(predict.rxSolve) +export(predict.rxUi) +export(predict.rxode2) export(probit) export(probitInv) export(probitNormInfo) diff --git a/R/elu.R b/R/elu.R new file mode 100644 index 000000000..76b3287fa --- /dev/null +++ b/R/elu.R @@ -0,0 +1,100 @@ +#' Exponential Linear Unit (ELU) Activation Function +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' ELU(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=SELU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +ELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF`, .df$x, .df$alpha, 1L) +} +#' Derivatives of the Exponential Linear Unit (ELU) Activation Function +#' +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative(s) of the ELU function has been applied +#' to each element of `x`. +#' @export +#' @author Matthew L. Fidler +#' @family Activation Functions +#' @examples +#' dELU(c(-1, 0, 1, 2), 2) +#' d2ELU(c(-1, 0, 1, 2), 2) +#' d2aELU(c(-1, 0, 1, 2), 2) +#' dELUa(c(-1, 0, 1, 2), 2) +#' d2ELUa(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' r <- rxode2({ +#' r1=dELU(time, 2) +#' r2=d2ELU(time, 2) +#' r2a=d2aELU(time, 2) +#' ra=dELUa(time, 2) +#' r2a=d2ELUa(time, 2) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(r, e) +dELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF`, .df$x, .df$alpha, 2L) +} + +#' @rdname dELU +#' @export +d2ELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF`, .df$x, .df$alpha, 3L) +} + +#' @rdname dELU +#' @export +d2aELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF`, .df$x, .df$alpha, 4L) +} + +#' @rdname dELU +#' @export +dELUa <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF`, .df$x, .df$alpha, 5L) +} + +#' @rdname dELU +#' @export +d2ELUa <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF`, .df$x, .df$alpha, 6L) +} diff --git a/R/selu.R b/R/selu.R index 07a95d733..804cc0f08 100644 --- a/R/selu.R +++ b/R/selu.R @@ -1,9 +1,5 @@ #' Scaled Exponential Linear Unit (SELU) Activation Function #' -#' The Scaled Exponential Linear Unit (SELU) activation function is a -#' piecewise linear function that allows for negative values to be -#' passed through the network. -#' #' @family Activation Functions #' @param x A numeric vector. All elements must be finite and #' non-missing. diff --git a/man/ELU.Rd b/man/ELU.Rd new file mode 100644 index 000000000..5011270c4 --- /dev/null +++ b/man/ELU.Rd @@ -0,0 +1,53 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/elu.R +\name{ELU} +\alias{ELU} +\title{Exponential Linear Unit (ELU) Activation Function} +\usage{ +ELU(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +Exponential Linear Unit (ELU) Activation Function +} +\examples{ + +ELU(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +x <- rxode2({ + r=SELU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/GELU.Rd b/man/GELU.Rd index 6159d4c45..c6862f890 100644 --- a/man/GELU.Rd +++ b/man/GELU.Rd @@ -27,9 +27,11 @@ rxSolve(r, et) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/ReLU.Rd b/man/ReLU.Rd index ba3b4ad7f..743bc1305 100644 --- a/man/ReLU.Rd +++ b/man/ReLU.Rd @@ -34,9 +34,11 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/SELU.Rd b/man/SELU.Rd index 9e81f45ec..f30635183 100644 --- a/man/SELU.Rd +++ b/man/SELU.Rd @@ -15,9 +15,7 @@ A numeric vector where the ReLU function has been applied to each element of \code{x}. } \description{ -The Scaled Exponential Linear Unit (SELU) activation function is a -piecewise linear function that allows for negative values to be -passed through the network. +Scaled Exponential Linear Unit (SELU) Activation Function } \examples{ @@ -35,9 +33,11 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/Swish.Rd b/man/Swish.Rd index 4ebce998b..871ea0a2f 100644 --- a/man/Swish.Rd +++ b/man/Swish.Rd @@ -36,9 +36,11 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/dELU.Rd b/man/dELU.Rd new file mode 100644 index 000000000..c663dc733 --- /dev/null +++ b/man/dELU.Rd @@ -0,0 +1,73 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/elu.R +\name{dELU} +\alias{dELU} +\alias{d2ELU} +\alias{d2aELU} +\alias{dELUa} +\alias{d2ELUa} +\title{Derivatives of the Exponential Linear Unit (ELU) Activation Function} +\usage{ +dELU(x, alpha = 1) + +d2ELU(x, alpha = 1) + +d2aELU(x, alpha = 1) + +dELUa(x, alpha = 1) + +d2ELUa(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative(s) of the ELU function has been applied +to each element of \code{x}. +} +\description{ +Derivatives of the Exponential Linear Unit (ELU) Activation Function +} +\examples{ +dELU(c(-1, 0, 1, 2), 2) +d2ELU(c(-1, 0, 1, 2), 2) +d2aELU(c(-1, 0, 1, 2), 2) +dELUa(c(-1, 0, 1, 2), 2) +d2ELUa(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +r <- rxode2({ + r1=dELU(time, 2) + r2=d2ELU(time, 2) + r2a=d2aELU(time, 2) + ra=dELUa(time, 2) + r2a=d2ELUa(time, 2) +}) + +e <- et(c(-1, 0, 1, 2)) +rxSolve(r, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/man/dGELU.Rd b/man/dGELU.Rd index 4b98e80e9..457c352ab 100644 --- a/man/dGELU.Rd +++ b/man/dGELU.Rd @@ -41,10 +41,12 @@ rxSolve(r, et) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/dReLU.Rd b/man/dReLU.Rd index ef7856c9c..a6601988b 100644 --- a/man/dReLU.Rd +++ b/man/dReLU.Rd @@ -32,10 +32,12 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/dSELU.Rd b/man/dSELU.Rd index 698e25962..ed61625c0 100644 --- a/man/dSELU.Rd +++ b/man/dSELU.Rd @@ -28,10 +28,12 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSwish}()}, diff --git a/man/dSwish.Rd b/man/dSwish.Rd index ed1e3abf0..c084da69e 100644 --- a/man/dSwish.Rd +++ b/man/dSwish.Rd @@ -29,10 +29,12 @@ rxSolve(x, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd index 858a1ba92..761b37336 100644 --- a/man/dlReLU.Rd +++ b/man/dlReLU.Rd @@ -28,10 +28,12 @@ rxSolve(r, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd index 56c4d8357..4cdec27c8 100644 --- a/man/dsoftplus.Rd +++ b/man/dsoftplus.Rd @@ -47,10 +47,12 @@ rxSolve(r, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/lReLU.Rd b/man/lReLU.Rd index c489b861e..d88d20b04 100644 --- a/man/lReLU.Rd +++ b/man/lReLU.Rd @@ -27,10 +27,12 @@ rxSolve(r, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/man/reexports.Rd b/man/reexports.Rd index 2857ff16d..8cba3d62e 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,4 +38,3 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} -\value{ Inherited from parent routine } diff --git a/man/rxSolve.Rd b/man/rxSolve.Rd index 1d04267e5..c38c34723 100644 --- a/man/rxSolve.Rd +++ b/man/rxSolve.Rd @@ -209,17 +209,17 @@ rxSolve( \method{update}{rxSolve}(object, ...) -\method{predict}{rxode2}(object, ...) +predict.rxode2(object, ...) -\method{predict}{`function`}(object, ...) +predict.function(object, ...) -\method{predict}{rxUi}(object, ...) +predict.rxUi(object, ...) -\method{predict}{rxSolve}(object, ...) +predict.rxSolve(object, ...) -\method{predict}{rxEt}(object, ...) +predict.rxEt(object, ...) -\method{predict}{rxParams}(object, ...) +predict.rxParams(object, ...) \method{simulate}{rxode2}(object, nsim = 1L, seed = NULL, ...) diff --git a/man/softplus.Rd b/man/softplus.Rd index a1d5e102b..006c265d3 100644 --- a/man/softplus.Rd +++ b/man/softplus.Rd @@ -32,10 +32,12 @@ rxSolve(r, e) } \seealso{ Other Activation Functions: +\code{\link{ELU}()}, \code{\link{GELU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, +\code{\link{dELU}()}, \code{\link{dGELU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, diff --git a/src/init.c b/src/init.c index 3cd55d97d..08f6d0ce7 100644 --- a/src/init.c +++ b/src/init.c @@ -570,8 +570,10 @@ SEXP _rxode2_rxode2Ptr(void) { SEXP _rxode2_powerD(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); SEXP _rxode2_activationF(SEXP xS, SEXP typeS); +SEXP _rxode2_activationF2(SEXP xS, SEXP aS, SEXP typeS); void R_init_rxode2(DllInfo *info){ R_CallMethodDef callMethods[] = { + {"_rxode2_activationF2", (DL_FUNC) &_rxode2_activationF2, 3}, {"_rxode2_activationF", (DL_FUNC) &_rxode2_activationF, 2}, {"_rxode2_itoletter", (DL_FUNC) &_rxode2_itoletter, 2}, {"_rxode2_itostr", (DL_FUNC) &_rxode2_itostr, 2}, diff --git a/src/utilc.c b/src/utilc.c index 800a8aa2e..a49c568d7 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -730,6 +730,55 @@ double dSwish(double x) { return x*ex/(den*den) + 1.0/den; } +SEXP _rxode2_activationF2(SEXP xS, SEXP aS, SEXP typeS) { + int type = INTEGER(typeS)[0]; + int typex = TYPEOF(xS); + int typea = TYPEOF(aS); + int lenx = Rf_length(xS); + SEXP ret = PROTECT(Rf_allocVector(REALSXP, lenx)); + for (int i = 0; i < lenx; ++i) { + double x = (typex == REALSXP) ? REAL(xS)[i] : (double)INTEGER(xS)[i]; + double a = (typea == REALSXP) ? REAL(aS)[i] : (double)INTEGER(aS)[i]; + switch (type) { + case 1: + REAL(ret)[i] = ELU(x, a); + break; + case 2: + REAL(ret)[i] = dELU(x, a); + break; + case 3: + REAL(ret)[i] = d2ELU(x, a); + break; + case 4: + REAL(ret)[i] = d2aELU(x, a); + break; + case 5: + REAL(ret)[i] = dELUa(x, a); + break; + case 6: + REAL(ret)[i] = d2ELUa(x, a); + break; + case 7: + REAL(ret)[i] = PReLU(x, a); + break; + case 8: + REAL(ret)[i] = dPReLU(x, a); + break; + case 9: + REAL(ret)[i] = dPReLUa(x, a); + break; + case 10: + REAL(ret)[i] = dPReLUa1(x, a); + break; + default: + REAL(ret)[i] = NA_REAL; + break; + } + } + UNPROTECT(1); + return(ret); +} + SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { int type = INTEGER(typeS)[0]; int typex = TYPEOF(xS); @@ -793,7 +842,7 @@ SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { REAL(ret)[i] = dSwish(x); break; default: - REAL(ret)[i] = x; + REAL(ret)[i] = NA_REAL; break; } } From 2e1eb65f516b2b3915884e67e802c6057a93c445 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 16:55:25 -0600 Subject: [PATCH 11/16] PReLU --- NAMESPACE | 4 +++ R/elu.R | 12 +++---- R/prelu.R | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ man/ELU.Rd | 2 ++ man/GELU.Rd | 2 ++ man/PReLU.Rd | 58 ++++++++++++++++++++++++++++++++++ man/ReLU.Rd | 2 ++ man/SELU.Rd | 2 ++ man/Swish.Rd | 2 ++ man/dELU.Rd | 2 ++ man/dGELU.Rd | 2 ++ man/dPReLU.Rd | 66 +++++++++++++++++++++++++++++++++++++++ man/dReLU.Rd | 2 ++ man/dSELU.Rd | 2 ++ man/dSwish.Rd | 2 ++ man/dlReLU.Rd | 2 ++ man/dsoftplus.Rd | 2 ++ man/lReLU.Rd | 2 ++ man/reexports.Rd | 1 + man/softplus.Rd | 2 ++ 20 files changed, 244 insertions(+), 6 deletions(-) create mode 100644 man/PReLU.Rd create mode 100644 man/dPReLU.Rd diff --git a/NAMESPACE b/NAMESPACE index 0c8999c05..ee4e49853 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -338,6 +338,7 @@ export(.udfMd5Info) export(.useUtf) export(.vecDf) export(ELU) +export(PReLU) export(ReLU) export(RxODE) export(SELU) @@ -384,6 +385,9 @@ export(d3GELU) export(d4GELU) export(dELU) export(dELUa) +export(dPReLU) +export(dPReLUa) +export(dPReLUa1) export(dReLU) export(dSELU) export(dSwish) diff --git a/R/elu.R b/R/elu.R index 76b3287fa..f335da1f4 100644 --- a/R/elu.R +++ b/R/elu.R @@ -24,7 +24,7 @@ ELU <- function(x, alpha=1) { checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) .df <- data.frame(x=x, alpha=alpha) - .Call(`_rxode2_activationF`, .df$x, .df$alpha, 1L) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 1L) } #' Derivatives of the Exponential Linear Unit (ELU) Activation Function #' @@ -60,7 +60,7 @@ dELU <- function(x, alpha=1) { checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) .df <- data.frame(x=x, alpha=alpha) - .Call(`_rxode2_activationF`, .df$x, .df$alpha, 2L) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 2L) } #' @rdname dELU @@ -69,7 +69,7 @@ d2ELU <- function(x, alpha=1) { checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) .df <- data.frame(x=x, alpha=alpha) - .Call(`_rxode2_activationF`, .df$x, .df$alpha, 3L) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 3L) } #' @rdname dELU @@ -78,7 +78,7 @@ d2aELU <- function(x, alpha=1) { checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) .df <- data.frame(x=x, alpha=alpha) - .Call(`_rxode2_activationF`, .df$x, .df$alpha, 4L) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 4L) } #' @rdname dELU @@ -87,7 +87,7 @@ dELUa <- function(x, alpha=1) { checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) .df <- data.frame(x=x, alpha=alpha) - .Call(`_rxode2_activationF`, .df$x, .df$alpha, 5L) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 5L) } #' @rdname dELU @@ -96,5 +96,5 @@ d2ELUa <- function(x, alpha=1) { checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) .df <- data.frame(x=x, alpha=alpha) - .Call(`_rxode2_activationF`, .df$x, .df$alpha, 6L) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 6L) } diff --git a/R/prelu.R b/R/prelu.R index e69de29bb..5708d8efb 100644 --- a/R/prelu.R +++ b/R/prelu.R @@ -0,0 +1,81 @@ +#' Parametric ReLU Activation Function +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' PReLU(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=PReLU(time, 2) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +PReLU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 7L) +} +#' Derivatives Parametric ReLU Activation Function +#' +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative(s) of the ELU function has been applied +#' to each element of `x`. +#' @export +#' @author Matthew L. Fidler +#' @family Activation Functions +#' @examples +#' +#' dPReLU(c(-1, 0, 1, 2), 2) +#' dPReLUa(c(-1, 0, 1, 2), 2) +#' dPReLUa1(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' r <- rxode2({ +#' r1=dPReLU(time, 2) +#' r2a=dPReLUa(time, 2) +#' ra=dPReLUa1(time, 2) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(r, e) +dPReLU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 8L) +} + +#' @rdname dPReLU +#' @export +dPReLUa <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 9L) +} + +#' @rdname dPReLU +#' @export +dPReLUa1 <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 10L) +} diff --git a/man/ELU.Rd b/man/ELU.Rd index 5011270c4..cefc84179 100644 --- a/man/ELU.Rd +++ b/man/ELU.Rd @@ -34,11 +34,13 @@ rxSolve(x, e) \seealso{ Other Activation Functions: \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/GELU.Rd b/man/GELU.Rd index c6862f890..1f1f8fb2f 100644 --- a/man/GELU.Rd +++ b/man/GELU.Rd @@ -28,11 +28,13 @@ rxSolve(r, et) \seealso{ Other Activation Functions: \code{\link{ELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/PReLU.Rd b/man/PReLU.Rd new file mode 100644 index 000000000..ea74e0def --- /dev/null +++ b/man/PReLU.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prelu.R +\name{PReLU} +\alias{PReLU} +\title{Parametric ReLU Activation Function} +\usage{ +PReLU(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +Parametric ReLU Activation Function +} +\examples{ + +PReLU(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +x <- rxode2({ + r=PReLU(time, 2) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/ReLU.Rd b/man/ReLU.Rd index 743bc1305..698f7d6c5 100644 --- a/man/ReLU.Rd +++ b/man/ReLU.Rd @@ -36,10 +36,12 @@ rxSolve(x, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/SELU.Rd b/man/SELU.Rd index f30635183..31bdef1be 100644 --- a/man/SELU.Rd +++ b/man/SELU.Rd @@ -35,10 +35,12 @@ rxSolve(x, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/Swish.Rd b/man/Swish.Rd index 871ea0a2f..a12b2336b 100644 --- a/man/Swish.Rd +++ b/man/Swish.Rd @@ -38,10 +38,12 @@ rxSolve(x, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/dELU.Rd b/man/dELU.Rd index c663dc733..f4c4d65ce 100644 --- a/man/dELU.Rd +++ b/man/dELU.Rd @@ -55,10 +55,12 @@ rxSolve(r, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/dGELU.Rd b/man/dGELU.Rd index 457c352ab..0826f4974 100644 --- a/man/dGELU.Rd +++ b/man/dGELU.Rd @@ -43,10 +43,12 @@ rxSolve(r, et) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/dPReLU.Rd b/man/dPReLU.Rd new file mode 100644 index 000000000..463283315 --- /dev/null +++ b/man/dPReLU.Rd @@ -0,0 +1,66 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prelu.R +\name{dPReLU} +\alias{dPReLU} +\alias{dPReLUa} +\alias{dPReLUa1} +\title{Derivatives Parametric ReLU Activation Function} +\usage{ +dPReLU(x, alpha = 1) + +dPReLUa(x, alpha = 1) + +dPReLUa1(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative(s) of the ELU function has been applied +to each element of \code{x}. +} +\description{ +Derivatives Parametric ReLU Activation Function +} +\examples{ + +dPReLU(c(-1, 0, 1, 2), 2) +dPReLUa(c(-1, 0, 1, 2), 2) +dPReLUa1(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +r <- rxode2({ + r1=dPReLU(time, 2) + r2a=dPReLUa(time, 2) + ra=dPReLUa1(time, 2) +}) + +e <- et(c(-1, 0, 1, 2)) +rxSolve(r, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/man/dReLU.Rd b/man/dReLU.Rd index a6601988b..1345fa681 100644 --- a/man/dReLU.Rd +++ b/man/dReLU.Rd @@ -34,11 +34,13 @@ rxSolve(x, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, \code{\link{dlReLU}()}, diff --git a/man/dSELU.Rd b/man/dSELU.Rd index ed61625c0..f1e3005fa 100644 --- a/man/dSELU.Rd +++ b/man/dSELU.Rd @@ -30,11 +30,13 @@ rxSolve(x, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSwish}()}, \code{\link{dlReLU}()}, diff --git a/man/dSwish.Rd b/man/dSwish.Rd index c084da69e..db244e29e 100644 --- a/man/dSwish.Rd +++ b/man/dSwish.Rd @@ -31,11 +31,13 @@ rxSolve(x, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dlReLU}()}, diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd index 761b37336..cd1f2e188 100644 --- a/man/dlReLU.Rd +++ b/man/dlReLU.Rd @@ -30,11 +30,13 @@ rxSolve(r, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd index 4cdec27c8..460143388 100644 --- a/man/dsoftplus.Rd +++ b/man/dsoftplus.Rd @@ -49,11 +49,13 @@ rxSolve(r, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/lReLU.Rd b/man/lReLU.Rd index d88d20b04..6ea8fc124 100644 --- a/man/lReLU.Rd +++ b/man/lReLU.Rd @@ -29,11 +29,13 @@ rxSolve(r, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, diff --git a/man/reexports.Rd b/man/reexports.Rd index 8cba3d62e..2857ff16d 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,3 +38,4 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} +\value{ Inherited from parent routine } diff --git a/man/softplus.Rd b/man/softplus.Rd index 006c265d3..1059978bc 100644 --- a/man/softplus.Rd +++ b/man/softplus.Rd @@ -34,11 +34,13 @@ rxSolve(r, e) Other Activation Functions: \code{\link{ELU}()}, \code{\link{GELU}()}, +\code{\link{PReLU}()}, \code{\link{ReLU}()}, \code{\link{SELU}()}, \code{\link{Swish}()}, \code{\link{dELU}()}, \code{\link{dGELU}()}, +\code{\link{dPReLU}()}, \code{\link{dReLU}()}, \code{\link{dSELU}()}, \code{\link{dSwish}()}, From 567a3f1c7496c90c81aeb4da4939719419934d59 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 18:04:48 -0600 Subject: [PATCH 12/16] Start activation tests --- R/prelu.R | 1 + src/utilc.c | 13 ++-- tests/testthat/test-activation.R | 108 +++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 tests/testthat/test-activation.R diff --git a/R/prelu.R b/R/prelu.R index 5708d8efb..a0aaef006 100644 --- a/R/prelu.R +++ b/R/prelu.R @@ -46,6 +46,7 @@ PReLU <- function(x, alpha=1) { #' dPReLUa(c(-1, 0, 1, 2), 2) #' dPReLUa1(c(-1, 0, 1, 2), 2) #' +#' #' # Can also be used in rxode2: #' r <- rxode2({ #' r1=dPReLU(time, 2) diff --git a/src/utilc.c b/src/utilc.c index a49c568d7..1c0eabfc2 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -618,7 +618,8 @@ double d2GELU(double x) { } double d3GELU(double x) { - return x * exp(-0.5 * x * x) * M_1_SQRT_2PI * (x * x - 4.0); + double x2 = x*x; + return -8.0*x*exp(-x2*0.5)*0.5*M_1_SQRT_2PI + 4.0*x2*x*exp(-x2*0.5)*0.25*M_1_SQRT_2PI; } double d4GELU(double x) { @@ -701,23 +702,23 @@ double lReLU(double x) { } double dlReLU(double x) { - return (x >= 0.0) ? 1.0 : 0.01; + return (x > 0.0) ? 1.0 : 0.01; } double PReLU(double x, double alpha) { - return (x >= 0.0) ? x : alpha * x; + return (x > 0.0) ? x : alpha * x; } double dPReLU(double x, double alpha) { - return (x >= 0.0) ? 1.0 : alpha; + return (x > 0.0) ? 1.0 : alpha; } double dPReLUa(double x, double alpha) { - return (x >= 0.0) ? 0.0 : x; + return (x > 0.0) ? 0.0 : x; } double dPReLUa1(double x, double alpha) { - return (x >= 0.0) ? 0.0 : 1.0; + return (x > 0.0) ? 0.0 : 1.0; } double Swish(double x) { diff --git a/tests/testthat/test-activation.R b/tests/testthat/test-activation.R new file mode 100644 index 000000000..481cc98f8 --- /dev/null +++ b/tests/testthat/test-activation.R @@ -0,0 +1,108 @@ +test_that("activation functions", { + + r <- rxode2({ + r <- ReLU(time) + dr <- dReLU(time) + # + l <- lReLU(time) + dl <- dlReLU(time) + # + p <- PReLU(time, alpha) + dp <- dPReLU(time, alpha) + dpa <- dPReLUa(time, alpha) + dpa1 <- dPReLUa1(time, alpha) + # + g <- GELU(time) + dg <- dGELU(time) + d2g <- d2GELU(time) + d3g <- d3GELU(time) + d4g <- d4GELU(time) + # + s <- Swish(time) + ds <- dSwish(time) + # + sp <- softplus(time) + dsp <- dsoftplus(time) + # + el <- ELU(time, alpha) + del <- dELU(time, alpha) + d2el <- d2ELU(time, alpha) + d2ael <- d2aELU(time, alpha) + dael <- d2aELU(time, alpha) + d2ael <- d2ELUa(time, alpha) + }) + + e <- et(seq(-10, 10, length.out = 41)) + + s <- rxSolve(r, e, c(alpha=2)) + + expect_equal(ReLU(e$time), s$r) + expect_equal(ReLU(e$time), vapply(e$time, + function(x) { + max(0, x) + }, double(1))) + expect_equal(dReLU(e$time), s$dr) + expect_equal(dReLU(e$time), vapply(e$time, + function(x) { + (x > 0)*1 + }, double(1))) + expect_equal(lReLU(e$time), s$l) + expect_equal(lReLU(e$time), vapply(e$time, + function(x) { + (x > 0)*x + (x <= 0)*0.01*x + }, double(1))) + expect_equal(dlReLU(e$time), s$dl) + expect_equal(dlReLU(e$time), vapply(e$time, + function(x) { + (x > 0) + (x <= 0)*0.01 + }, double(1))) + + expect_equal(PReLU(e$time, 2), s$p) + expect_equal(PReLU(e$time, 2), vapply(e$time, + function(x) { + (x > 0)*x + (x <= 0)*2*x + }, double(1))) + + expect_equal(dPReLU(e$time, 2), s$dp) + expect_equal(dPReLU(e$time, 2), vapply(e$time, + function(x) { + (x > 0) + (x <= 0)*2 + }, double(1))) + + expect_equal(GELU(e$time), s$g) + expect_equal(GELU(e$time), vapply(e$time, + function(x) { + phi(x)*x + }, double(1))) + + expect_equal(dGELU(e$time), s$dg) + expect_equal(dGELU(e$time), vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + 1.0*x*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2/sqrt(pi) + 0.5*(1 + erf(x*M_SQRT1_2)) + }, double(1))) + expect_equal(d2GELU(e$time), s$d2g) + expect_equal(d2GELU(e$time), + vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + 2.0*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2/sqrt(pi) - 2.0*x^2*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^3/sqrt(pi) + }, double(1))) + + expect_equal(d3GELU(e$time), s$d3g) + expect_equal(d3GELU(e$time), + vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + M_1_SQRT_2PI <- 1/sqrt(2*pi) + -8.0*x*exp(-x^2*0.5)*0.5*M_1_SQRT_2PI + 4.0*x^3*exp(-x^2*0.5)*0.25*M_1_SQRT_2PI + }, double(1))) + expect_equal(d4GELU(e$time), s$d4g) + expect_equal(d4GELU(e$time), + vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + -8.0*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^3/sqrt(pi) + 28.0*x^2*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^5/sqrt(pi) - 8.0*x^4*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^7/sqrt(pi) + }, double(1))) + +}) From 3f6768e129a9f17042627fa484d891c2960e6843 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 18:19:49 -0600 Subject: [PATCH 13/16] Add more activation tests --- tests/testthat/test-activation.R | 61 +++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-activation.R b/tests/testthat/test-activation.R index 481cc98f8..4e8be6da0 100644 --- a/tests/testthat/test-activation.R +++ b/tests/testthat/test-activation.R @@ -27,9 +27,10 @@ test_that("activation functions", { el <- ELU(time, alpha) del <- dELU(time, alpha) d2el <- d2ELU(time, alpha) + d2a <- dELUa(time, alpha) d2ael <- d2aELU(time, alpha) dael <- d2aELU(time, alpha) - d2ael <- d2ELUa(time, alpha) + d2aela <- d2ELUa(time, alpha) }) e <- et(seq(-10, 10, length.out = 41)) @@ -105,4 +106,62 @@ test_that("activation functions", { -8.0*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^3/sqrt(pi) + 28.0*x^2*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^5/sqrt(pi) - 8.0*x^4*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^7/sqrt(pi) }, double(1))) + expect_equal(Swish(e$time), s$s) + expect_equal(Swish(e$time), vapply(e$time, + function(x) { + x/(1 + exp(-x)) + }, double(1))) + + expect_equal(dSwish(e$time), s$ds) + expect_equal(dSwish(e$time), vapply(e$time, + function(x) { + x*exp(-x)/(1 + exp(-x))^2 + (1 + exp(-x))^(-1) + }, double(1))) + + expect_equal(softplus(e$time), s$sp) + expect_equal(softplus(e$time), vapply(e$time, + function(x) { + log(1 + exp(x)) + }, double(1))) + + expect_equal(dsoftplus(e$time), s$dsp) + expect_equal(dsoftplus(e$time), vapply(e$time, + function(x) { + exp(x)/(1 + exp(x)) + }, double(1))) + + expect_equal(ELU(e$time, 2), s$el) + expect_equal(ELU(e$time, 2), vapply(e$time, + function(x) { + (x > 0)*x + (x <= 0)*2*(exp(x) - 1) + }, double(1))) + + # dELU/dx + expect_equal(dELU(e$time, 2), s$del) + expect_equal(dELU(e$time, 2), vapply(e$time, + function(x) { + (x > 0) + (x <= 0)*2*exp(x) + }, double(1))) + + # d2ELU/d2x + expect_equal(d2ELU(e$time, 2), s$d2el) + expect_equal(d2ELU(e$time, 2), vapply(e$time, + function(x) { + (x <= 0)*2*exp(x) + }, double(1))) + + # dELU/dalpha + expect_equal(dELUa(e$time, 2), s$d2a) + expect_equal(dELUa(e$time, 2), vapply(e$time, + function(x) { + (x <= 0)*(exp(x) - 1) + }, double(1))) + + # d2ELU/dalphad2x + expect_equal(d2ELUa(e$time, 2), s$d2ael) + expect_equal(d2ELUa(e$time, 2), vapply(e$time, + function(x) { + (x <= 0)*exp(x) + }, double(1))) + }) From ba237a624d52310fc75d171f8b0adc4999b5b97e Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 21:44:14 -0600 Subject: [PATCH 14/16] ::doc() --- NAMESPACE | 12 ++++++------ R/elu.R | 2 ++ man/ELU.Rd | 3 +++ man/dPReLU.Rd | 1 + man/reexports.Rd | 1 - man/rxSolve.Rd | 12 ++++++------ 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index ee4e49853..bbe44c5e6 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -95,6 +95,12 @@ S3method(modelExtract,rxode2) S3method(plot,rxSolve) S3method(plot,rxSolveConfint1) S3method(plot,rxSolveConfint2) +S3method(predict,"function") +S3method(predict,rxEt) +S3method(predict,rxParams) +S3method(predict,rxSolve) +S3method(predict,rxUi) +S3method(predict,rxode2) S3method(print,boundInits) S3method(print,boundParams) S3method(print,rxC) @@ -467,12 +473,6 @@ export(model) export(modelExtract) export(odeMethodToInt) export(phi) -export(predict.function) -export(predict.rxEt) -export(predict.rxParams) -export(predict.rxSolve) -export(predict.rxUi) -export(predict.rxode2) export(probit) export(probitInv) export(probitNormInfo) diff --git a/R/elu.R b/R/elu.R index f335da1f4..02bdb5c10 100644 --- a/R/elu.R +++ b/R/elu.R @@ -3,6 +3,8 @@ #' @family Activation Functions #' @param x A numeric vector. All elements must be finite and #' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. #' @return A numeric vector where the ReLU function has been applied #' to each element of `x`. #' @author Matthew Fidler diff --git a/man/ELU.Rd b/man/ELU.Rd index cefc84179..12b2ce0ab 100644 --- a/man/ELU.Rd +++ b/man/ELU.Rd @@ -9,6 +9,9 @@ ELU(x, alpha = 1) \arguments{ \item{x}{A numeric vector. All elements must be finite and non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} } \value{ A numeric vector where the ReLU function has been applied diff --git a/man/dPReLU.Rd b/man/dPReLU.Rd index 463283315..3b2f4295f 100644 --- a/man/dPReLU.Rd +++ b/man/dPReLU.Rd @@ -32,6 +32,7 @@ dPReLU(c(-1, 0, 1, 2), 2) dPReLUa(c(-1, 0, 1, 2), 2) dPReLUa1(c(-1, 0, 1, 2), 2) + # Can also be used in rxode2: r <- rxode2({ r1=dPReLU(time, 2) diff --git a/man/reexports.Rd b/man/reexports.Rd index 2857ff16d..8cba3d62e 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,4 +38,3 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} -\value{ Inherited from parent routine } diff --git a/man/rxSolve.Rd b/man/rxSolve.Rd index c38c34723..1d04267e5 100644 --- a/man/rxSolve.Rd +++ b/man/rxSolve.Rd @@ -209,17 +209,17 @@ rxSolve( \method{update}{rxSolve}(object, ...) -predict.rxode2(object, ...) +\method{predict}{rxode2}(object, ...) -predict.function(object, ...) +\method{predict}{`function`}(object, ...) -predict.rxUi(object, ...) +\method{predict}{rxUi}(object, ...) -predict.rxSolve(object, ...) +\method{predict}{rxSolve}(object, ...) -predict.rxEt(object, ...) +\method{predict}{rxEt}(object, ...) -predict.rxParams(object, ...) +\method{predict}{rxParams}(object, ...) \method{simulate}{rxode2}(object, nsim = 1L, seed = NULL, ...) From afd11908eac842419e47937868204c5658e6f152 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 22:17:47 -0600 Subject: [PATCH 15/16] Update pkgdown --- _pkgdown.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/_pkgdown.yml b/_pkgdown.yml index 2feb7bf3c..5053e5034 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -236,6 +236,24 @@ reference: - rxode2parseGetTranslation - summary.rxode2 - rxUiDeparse +- title: Neural Network Activation functions + contents: + - ELU + - GELU + - PReLU + - ReLU + - SELU + - Swish + - dELU + - dGELU + - dPReLU + - dReLU + - dSELU + - dSwish + - dlReLU + - dsoftplus + - lReLU + - softplus - title: Rxode2 data sets contents: - rxResidualError From c39e03bdb8bfae8a20d3f2759b5b640394d2dd91 Mon Sep 17 00:00:00 2001 From: Matthew Fidler Date: Sun, 24 Nov 2024 22:24:56 -0600 Subject: [PATCH 16/16] Fix examples --- NAMESPACE | 2 ++ R/gelu.R | 14 +++++++++----- R/softplus.R | 8 +------- man/GELU.Rd | 4 +++- man/dGELU.Rd | 8 ++++---- man/dsoftplus.Rd | 6 ------ man/reexports.Rd | 1 + man/softplus.Rd | 2 +- 8 files changed, 21 insertions(+), 24 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index bbe44c5e6..acec56761 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -344,6 +344,7 @@ export(.udfMd5Info) export(.useUtf) export(.vecDf) export(ELU) +export(GELU) export(PReLU) export(ReLU) export(RxODE) @@ -391,6 +392,7 @@ export(d3GELU) export(d4GELU) export(dELU) export(dELUa) +export(dGELU) export(dPReLU) export(dPReLUa) export(dPReLUa1) diff --git a/R/gelu.R b/R/gelu.R index 661c43dbf..e04a7e6f7 100644 --- a/R/gelu.R +++ b/R/gelu.R @@ -3,12 +3,15 @@ #' @param x numeric vector #' @return numeric vector #' @family Activation Functions +#' @export #' @examples #' #' GELU(c(-2, -1, 0, 1, 2)) #' #' # you can use rxode2 as well -#' r <- rxode2({r = GELU(x)}) +#' r <- rxode2({ +#' r = GELU(time) +#' }) #' et <- et(c(-2, -1, 0, 1, 2)) #' rxSolve(r, et) #' @@ -23,6 +26,7 @@ GELU <- function(x) { #' @param x numeric vector #' @return numeric vector #' @family Activation Functions +#' @export #' @examples #' dGELU(c(-2, -1, 0, 1, 2)) #' d2GELU(c(-2, -1, 0, 1, 2)) @@ -30,10 +34,10 @@ GELU <- function(x) { #' d4GELU(c(-2, -1, 0, 1, 2)) #' # you can use rxode2 as well #' r <- rxode2({ -#' r1 <- dGELU(x) -#' r2 <- d2GELU(x) -#' r3 <- d3GELU(x) -#' r4 <- d4GELU(x) +#' r1 <- dGELU(time) +#' r2 <- d2GELU(time) +#' r3 <- d3GELU(time) +#' r4 <- d4GELU(time) #' }) #' et <- et(c(-2, -1, 0, 1, 2)) #' rxSolve(r, et) diff --git a/R/softplus.R b/R/softplus.R index ed32d5386..0a069706c 100644 --- a/R/softplus.R +++ b/R/softplus.R @@ -12,7 +12,7 @@ #' # You can use rxode2 too: #' #' r <- rxode2({ -#' s <- softplus(x) +#' s <- softplus(time) #' }) #' #' e <- et(c(-1, 0, 1, 2)) @@ -34,17 +34,11 @@ softplus <- function(x) { #' @examples #' #' dsoftplus(c(-1, 0, 1, 2)) -#' d2softplus(c(-1, 0, 1, 2)) -#' d3softplus(c(-1, 0, 1, 2)) -#' d4softplus(c(-1, 0, 1, 2)) #' #' # You can use rxode2 too: #' #' r <- rxode2({ #' s1 <- dsoftplus(time) -#' s2 <- d2softplus(time) -#' s3 <- d3softplus(time) -#' s4 <- d4softplus(time) #' }) #' #' e <- et(c(-1, 0, 1, 2)) diff --git a/man/GELU.Rd b/man/GELU.Rd index 1f1f8fb2f..19b552b49 100644 --- a/man/GELU.Rd +++ b/man/GELU.Rd @@ -20,7 +20,9 @@ GELU activation function GELU(c(-2, -1, 0, 1, 2)) # you can use rxode2 as well -r <- rxode2({r = GELU(x)}) +r <- rxode2({ + r = GELU(time) +}) et <- et(c(-2, -1, 0, 1, 2)) rxSolve(r, et) diff --git a/man/dGELU.Rd b/man/dGELU.Rd index 0826f4974..e551f8be3 100644 --- a/man/dGELU.Rd +++ b/man/dGELU.Rd @@ -31,10 +31,10 @@ d3GELU(c(-2, -1, 0, 1, 2)) d4GELU(c(-2, -1, 0, 1, 2)) # you can use rxode2 as well r <- rxode2({ - r1 <- dGELU(x) - r2 <- d2GELU(x) - r3 <- d3GELU(x) - r4 <- d4GELU(x) + r1 <- dGELU(time) + r2 <- d2GELU(time) + r3 <- d3GELU(time) + r4 <- d4GELU(time) }) et <- et(c(-2, -1, 0, 1, 2)) rxSolve(r, et) diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd index 460143388..019db5954 100644 --- a/man/dsoftplus.Rd +++ b/man/dsoftplus.Rd @@ -27,17 +27,11 @@ Default Softplus Activation Function \examples{ dsoftplus(c(-1, 0, 1, 2)) -d2softplus(c(-1, 0, 1, 2)) -d3softplus(c(-1, 0, 1, 2)) -d4softplus(c(-1, 0, 1, 2)) # You can use rxode2 too: r <- rxode2({ s1 <- dsoftplus(time) - s2 <- d2softplus(time) - s3 <- d3softplus(time) - s4 <- d4softplus(time) }) e <- et(c(-1, 0, 1, 2)) diff --git a/man/reexports.Rd b/man/reexports.Rd index 8cba3d62e..2857ff16d 100644 --- a/man/reexports.Rd +++ b/man/reexports.Rd @@ -38,3 +38,4 @@ below to see their documentation. \item{magrittr}{\code{\link[magrittr:pipe]{\%>\%}}} }} +\value{ Inherited from parent routine } diff --git a/man/softplus.Rd b/man/softplus.Rd index 1059978bc..631e058b0 100644 --- a/man/softplus.Rd +++ b/man/softplus.Rd @@ -22,7 +22,7 @@ softplus(c(-1, 0, 1, 2)) # You can use rxode2 too: r <- rxode2({ - s <- softplus(x) + s <- softplus(time) }) e <- et(c(-1, 0, 1, 2))