diff --git a/NAMESPACE b/NAMESPACE index 16a94c37a..acec56761 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -343,7 +343,13 @@ export(.udfExists) export(.udfMd5Info) export(.useUtf) export(.vecDf) +export(ELU) +export(GELU) +export(PReLU) +export(ReLU) export(RxODE) +export(SELU) +export(Swish) export(add.dosing) export(add.sampling) export(aes) @@ -378,7 +384,24 @@ export(binomProbs) export(boxCox) export(boxCoxInv) export(cvPost) +export(d2ELU) +export(d2ELUa) +export(d2GELU) +export(d2aELU) +export(d3GELU) +export(d4GELU) +export(dELU) +export(dELUa) +export(dGELU) +export(dPReLU) +export(dPReLUa) +export(dPReLUa1) +export(dReLU) +export(dSELU) +export(dSwish) export(dfWishart) +export(dlReLU) +export(dsoftplus) export(erf) export(et) export(etExpand) @@ -414,6 +437,7 @@ export(invWR1d) export(is.rxEt) export(is.rxSolve) export(is.rxStackData) +export(lReLU) export(label_both) export(label_context) export(label_value) @@ -637,6 +661,7 @@ export(scale_y_continuous) export(scale_y_date) export(scale_y_discrete) export(setRxThreads) +export(softplus) export(stat_amt) export(stat_cens) export(swapMatListWithCube) diff --git a/R/dfIni.R b/R/dfIni.R index cc9cd6a3d..83680f278 100644 --- a/R/dfIni.R +++ b/R/dfIni.R @@ -3,103 +3,139 @@ "rxf", "rxgeom", "rxgamma", "rxbeta", "rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", "ribinom", "rinbinom", "rinbinomMu", "ricauchy", "richisq", "riexp", "rif", "rigeom", "rigamma", "ribeta", "ripois", -"rit_", "riunif", "riweibull", "phi", "gammap", "gammaq", "gammapInv", +"rit_", "riunif", "riweibull", "ReLU", "dReLU", "GELU", "dGELU", +"d2GELU", "d3GELU", "d4GELU", "softplus", "dsoftplus", "d2softplus", +"d3softplus", "d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", +"Swish", "dSwish", "PReLU", "dPReLU", "dPReLUa", "dPReLUa1", +"ELU", "dELU", "d2ELU", "d2aELU", "dELUa", "d2ELUa", "phi", "gammap", +"gammaq", "gammapInv", "gammapInva", "gammaqInv", "gammaqInva", +"uppergamma", "lowergamma", "gammapDer", "logit", "expit", "simeta", +"simeps", "llikNorm", "llikNormDmean", "llikNormDsd", "llikPois", +"llikPoisDlambda", "llikBinom", "llikBinomDprob", "llikNbinom", +"llikNbinomDprob", "llikNbinomMu", "llikNbinomMuDmu", "llikBeta", +"llikBetaDshape1", "llikBetaDshape2", "llikT", "llikTDdf", "llikTDmean", +"llikTDsd", "llikChisq", "llikChisqDdf", "llikExp", "llikExpDrate", +"llikF", "llikFDdf1", "llikFDdf2", "llikGeom", "llikGeomDp", +"llikUnif", "llikUnifDalpha", "llikUnifDbeta", "llikWeibull", +"llikWeibullDshape", "llikWeibullDscale", "llikGamma", "llikGammaDshape", +"llikGammaDrate", "llikCauchy", "llikCauchyDlocation", "llikCauchyDscale" +), fun = c("linCmtA", "linCmtB", "linCmtC", "rxnorm", "rxbinom", +"rxnbinom", "rxnbinomMu", "rxcauchy", "rxchisq", "rxexp", "rxf", +"rxgeom", "rxgamma", "rxbeta", "rxpois", "rxt_", "rxunif", "rxweibull", +"rinorm", "ribinom", "rinbinom", "rinbinomMu", "ricauchy", "richisq", +"riexp", "rif", "rigeom", "rigamma", "ribeta", "ripois", "rit_", +"riunif", "riweibull", "ReLU", "dReLU", "GELU", "dGELU", "d2GELU", +"d3GELU", "d4GELU", "softplus", "dsoftplus", "d2softplus", "d3softplus", +"d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", "Swish", "dSwish", +"PReLU", "dPReLU", "dPReLUa", "dPReLUa1", "ELU", "dELU", "d2ELU", +"d2aELU", "dELUa", "d2ELUa", "phi", "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", "gammaqInva", "uppergamma", "lowergamma", -"gammapDer", "logit", "expit", "simeta", "simeps", "llikNorm", -"llikNormDmean", "llikNormDsd", "llikPois", "llikPoisDlambda", -"llikBinom", "llikBinomDprob", "llikNbinom", "llikNbinomDprob", -"llikNbinomMu", "llikNbinomMuDmu", "llikBeta", "llikBetaDshape1", -"llikBetaDshape2", "llikT", "llikTDdf", "llikTDmean", "llikTDsd", -"llikChisq", "llikChisqDdf", "llikExp", "llikExpDrate", "llikF", -"llikFDdf1", "llikFDdf2", "llikGeom", "llikGeomDp", "llikUnif", -"llikUnifDalpha", "llikUnifDbeta", "llikWeibull", "llikWeibullDshape", -"llikWeibullDscale", "llikGamma", "llikGammaDshape", "llikGammaDrate", -"llikCauchy", "llikCauchyDlocation", "llikCauchyDscale"), fun = c("linCmtA", -"linCmtB", "linCmtC", "rxnorm", "rxbinom", "rxnbinom", "rxnbinomMu", -"rxcauchy", "rxchisq", "rxexp", "rxf", "rxgeom", "rxgamma", "rxbeta", -"rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", "ribinom", -"rinbinom", "rinbinomMu", "ricauchy", "richisq", "riexp", "rif", -"rigeom", "rigamma", "ribeta", "ripois", "rit_", "riunif", "riweibull", -"phi", "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", -"gammaqInva", "uppergamma", "lowergamma", "gammapDer", "logit", -"expit", "simeta", "simeps", "_llikNorm", "_llikNormDmean", "_llikNormDsd", -"_llikPois", "_llikPoisDlambda", "_llikBinom", "_llikBinomDprob", -"_llikNbinom", "_llikNbinomDprob", "_llikNbinomMu", "_llikNbinomMuDmu", -"_llikBeta", "_llikBetaDshape1", "_llikBetaDshape2", "_llikT", -"_llikTDdf", "_llikTDmean", "_llikTDsd", "_llikChisq", "_llikChisqDdf", -"_llikExp", "_llikExpDrate", "_llikF", "_llikFDdf1", "_llikFDdf2", -"_llikGeom", "_llikGeomDp", "_llikUnif", "_llikUnifDalpha", "_llikUnifDbeta", -"_llikWeibull", "_llikWeibullDshape", "_llikWeibullDscale", "_llikGamma", -"_llikGammaDshape", "_llikGammaDrate", "_llikCauchy", "_llikCauchyDlocation", -"_llikCauchyDscale"), type = c("linCmtA_p", "linCmtB_p", "linCmtA_p", -"rxode2i_fn2", "rxode2i_rxbinom", "rxode2i_rxbinom", "rxode2i_rxbinom", -"rxode2i_fn2", "rxode2i_fn", "rxode2i_fn", "rxode2i_fn2", "rxode2i_ifn", -"rxode2i_fn2", "rxode2i_fn2", "rxode2i_ifn", "rxode2i_fn", "rxode2i_fn2", -"rxode2i_fn2", "rxode2i2_fn2", "rxode2i2_ribinom", "rxode2i2_ribinom", -"rxode2i2_ribinom", "rxode2i2_fn2", "rxode2i2_fn", "rxode2i2_fn", -"rxode2i2_fn2", "rxode2i2_ifn", "rxode2i2_fn2", "rxode2i2_fn2", -"rxode2i2_ifn", "rxode2i2_fn", "rxode2i2_fn2", "rxode2i2_fn2", -"rxode2_fn", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", -"rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", -"rxode2_fn3", "rxode2_fn3", "_simfun", "_simfun", "rxode2_llikNormFun", -"rxode2_llikNormFun", "rxode2_llikNormFun", "rxode2_llikPoisFun", -"rxode2_llikPoisFun", "rxode2_llikBinomFun", "rxode2_llikBinomFun", -"rxode2_llikBinomFun", "rxode2_llikBinomFun", "rxode2_llikBinomFun", -"rxode2_llikBinomFun", "rxode2_llikBetaFun", "rxode2_llikBetaFun", -"rxode2_llikBetaFun", "rxode2_llikTFun", "rxode2_llikTFun", "rxode2_llikTFun", -"rxode2_llikTFun", "rxode2_llikChisqFun", "rxode2_llikChisqFun", -"rxode2_llikExpFun", "rxode2_llikExpFun", "rxode2_llikFFun", -"rxode2_llikFFun", "rxode2_llikFFun", "rxode2_llikGeomFun", "rxode2_llikGeomFun", -"rxode2_llikUnifFun", "rxode2_llikUnifFun", "rxode2_llikUnifFun", -"rxode2_llikWeibullFun", "rxode2_llikWeibullFun", "rxode2_llikWeibullFun", -"rxode2_llikGammaFun", "rxode2_llikGammaFun", "rxode2_llikGammaFun", -"rxode2_llikCauchyFun", "rxode2_llikCauchyFun", "rxode2_llikCauchyFun" -), package = c("rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", -"rxode2ll", "rxode2ll", "rxode2ll"), packageFun = c("linCmtA", -"linCmtB", "linCmtC", "rxnorm", "rxbinom", "rxnbinom", "rxnbinomMu", -"rxcauchy", "rxchisq", "rxexp", "rxf", "rxgeom", "rxgamma", "rxbeta", -"rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", "ribinom", -"rinbinom", "rinbinomMu", "ricauchy", "richisq", "riexp", "rif", -"rigeom", "rigamma", "ribeta", "ripois", "rit_", "riunif", "riweibull", -"phi", "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", -"gammaqInva", "uppergamma", "lowergamma", "gammapDer", "logit", -"expit", "simeta", "simeps", "rxLlikNorm", "rxLlikNormDmean", -"rxLlikNormDsd", "rxLlikPois", "rxLlikPoisDlambda", "rxLlikBinom", -"rxLlikBinomDprob", "rxLlikNbinom", "rxLlikNbinomDprob", "rxLlikNbinomMu", -"rxLlikNbinomMuDmu", "rxLlikBeta", "rxLlikBetaDshape1", "rxLlikBetaDshape2", -"rxLlikT", "rxLlikTDdf", "rxLlikTDmean", "rxLlikTDsd", "rxLlikChisq", -"rxLlikChisqDdf", "rxLlikExp", "rxLlikExpDrate", "rxLlikF", "rxLlikFDdf1", -"rxLlikFDdf2", "rxLlikGeom", "rxLlikGeomDp", "rxLlikUnif", "rxLlikUnifDalpha", -"rxLlikUnifDbeta", "rxLlikWeibull", "rxLlikWeibullDshape", "rxLlikWeibullDscale", -"rxLlikGamma", "rxLlikGammaDshape", "rxLlikGammaDrate", "rxLlikCauchy", -"rxLlikCauchyDlocation", "rxLlikCauchyDscale"), argMin = c(20L, -21L, 20L, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, -3L, 3L, 2L, 2L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, -4L, 2L, 2L, 2L, 2L, 3L, 3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, -3L, 3L, 3L, 3L, 3L, 3L), argMax = c(20L, 21L, 20L, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, -NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 2L, 2L, 2L, 2L, -2L, 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, 3L, 3L, 2L, 2L, 3L, 3L, -3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, 4L, 2L, 2L, 2L, 2L, 3L, -3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L -), threadSafe = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, -1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L)), row.names = c(NA, --86L), class = "data.frame") +"gammapDer", "logit", "expit", "simeta", "simeps", "_llikNorm", +"_llikNormDmean", "_llikNormDsd", "_llikPois", "_llikPoisDlambda", +"_llikBinom", "_llikBinomDprob", "_llikNbinom", "_llikNbinomDprob", +"_llikNbinomMu", "_llikNbinomMuDmu", "_llikBeta", "_llikBetaDshape1", +"_llikBetaDshape2", "_llikT", "_llikTDdf", "_llikTDmean", "_llikTDsd", +"_llikChisq", "_llikChisqDdf", "_llikExp", "_llikExpDrate", "_llikF", +"_llikFDdf1", "_llikFDdf2", "_llikGeom", "_llikGeomDp", "_llikUnif", +"_llikUnifDalpha", "_llikUnifDbeta", "_llikWeibull", "_llikWeibullDshape", +"_llikWeibullDscale", "_llikGamma", "_llikGammaDshape", "_llikGammaDrate", +"_llikCauchy", "_llikCauchyDlocation", "_llikCauchyDscale"), + type = c("linCmtA_p", "linCmtB_p", "linCmtA_p", "rxode2i_fn2", + "rxode2i_rxbinom", "rxode2i_rxbinom", "rxode2i_rxbinom", + "rxode2i_fn2", "rxode2i_fn", "rxode2i_fn", "rxode2i_fn2", + "rxode2i_ifn", "rxode2i_fn2", "rxode2i_fn2", "rxode2i_ifn", + "rxode2i_fn", "rxode2i_fn2", "rxode2i_fn2", "rxode2i2_fn2", + "rxode2i2_ribinom", "rxode2i2_ribinom", "rxode2i2_ribinom", + "rxode2i2_fn2", "rxode2i2_fn", "rxode2i2_fn", "rxode2i2_fn2", + "rxode2i2_ifn", "rxode2i2_fn2", "rxode2i2_fn2", "rxode2i2_ifn", + "rxode2i2_fn", "rxode2i2_fn2", "rxode2i2_fn2", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", "rxode2_fn", + "rxode2_fn", "rxode2_fn", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", "rxode2_fn2", + "rxode2_fn2", "rxode2_fn2", "rxode2_fn3", "rxode2_fn3", "_simfun", + "_simfun", "rxode2_llikNormFun", "rxode2_llikNormFun", "rxode2_llikNormFun", + "rxode2_llikPoisFun", "rxode2_llikPoisFun", "rxode2_llikBinomFun", + "rxode2_llikBinomFun", "rxode2_llikBinomFun", "rxode2_llikBinomFun", + "rxode2_llikBinomFun", "rxode2_llikBinomFun", "rxode2_llikBetaFun", + "rxode2_llikBetaFun", "rxode2_llikBetaFun", "rxode2_llikTFun", + "rxode2_llikTFun", "rxode2_llikTFun", "rxode2_llikTFun", + "rxode2_llikChisqFun", "rxode2_llikChisqFun", "rxode2_llikExpFun", + "rxode2_llikExpFun", "rxode2_llikFFun", "rxode2_llikFFun", + "rxode2_llikFFun", "rxode2_llikGeomFun", "rxode2_llikGeomFun", + "rxode2_llikUnifFun", "rxode2_llikUnifFun", "rxode2_llikUnifFun", + "rxode2_llikWeibullFun", "rxode2_llikWeibullFun", "rxode2_llikWeibullFun", + "rxode2_llikGammaFun", "rxode2_llikGammaFun", "rxode2_llikGammaFun", + "rxode2_llikCauchyFun", "rxode2_llikCauchyFun", "rxode2_llikCauchyFun" + ), package = c("rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", "rxode2", + "rxode2", "rxode2", "rxode2", "rxode2", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", "rxode2ll", + "rxode2ll", "rxode2ll"), packageFun = c("linCmtA", "linCmtB", + "linCmtC", "rxnorm", "rxbinom", "rxnbinom", "rxnbinomMu", + "rxcauchy", "rxchisq", "rxexp", "rxf", "rxgeom", "rxgamma", + "rxbeta", "rxpois", "rxt_", "rxunif", "rxweibull", "rinorm", + "ribinom", "rinbinom", "rinbinomMu", "ricauchy", "richisq", + "riexp", "rif", "rigeom", "rigamma", "ribeta", "ripois", + "rit_", "riunif", "riweibull", "ReLU", "dReLU", "GELU", "dGELU", + "d2GELU", "d3GELU", "d4GELU", "softplus", "dsoftplus", "d2softplus", + "d3softplus", "d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", + "Swish", "dSwish", "PReLU", "dPReLU", "dPReLUa", "dPReLUa1", + "ELU", "dELU", "d2ELU", "d2aELU", "dELUa", "d2ELUa", "phi", + "gammap", "gammaq", "gammapInv", "gammapInva", "gammaqInv", + "gammaqInva", "uppergamma", "lowergamma", "gammapDer", "logit", + "expit", "simeta", "simeps", "rxLlikNorm", "rxLlikNormDmean", + "rxLlikNormDsd", "rxLlikPois", "rxLlikPoisDlambda", "rxLlikBinom", + "rxLlikBinomDprob", "rxLlikNbinom", "rxLlikNbinomDprob", + "rxLlikNbinomMu", "rxLlikNbinomMuDmu", "rxLlikBeta", "rxLlikBetaDshape1", + "rxLlikBetaDshape2", "rxLlikT", "rxLlikTDdf", "rxLlikTDmean", + "rxLlikTDsd", "rxLlikChisq", "rxLlikChisqDdf", "rxLlikExp", + "rxLlikExpDrate", "rxLlikF", "rxLlikFDdf1", "rxLlikFDdf2", + "rxLlikGeom", "rxLlikGeomDp", "rxLlikUnif", "rxLlikUnifDalpha", + "rxLlikUnifDbeta", "rxLlikWeibull", "rxLlikWeibullDshape", + "rxLlikWeibullDscale", "rxLlikGamma", "rxLlikGammaDshape", + "rxLlikGammaDrate", "rxLlikCauchy", "rxLlikCauchyDlocation", + "rxLlikCauchyDscale"), argMin = c(20L, 21L, 20L, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, NA, 2L, 2L, 2L, + 2L, 2L, 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, 3L, 3L, 2L, 2L, + 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, 4L, 2L, 2L, + 2L, 2L, 3L, 3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, + 3L, 3L, 3L, 3L), argMax = c(20L, 21L, 20L, NA, NA, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, + NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, + 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, NA, 2L, 2L, 2L, 2L, 2L, + 2L, 2L, 2L, 2L, NA, NA, NA, NA, 3L, 3L, 3L, 2L, 2L, 3L, 3L, + 3L, 3L, 3L, 3L, 3L, 3L, 3L, 4L, 4L, 4L, 4L, 2L, 2L, 2L, 2L, + 3L, 3L, 3L, 2L, NA, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, 3L, + 3L, 3L), threadSafe = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, + 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L + )), row.names = c(NA, -114L), class = "data.frame") diff --git a/R/elu.R b/R/elu.R new file mode 100644 index 000000000..02bdb5c10 --- /dev/null +++ b/R/elu.R @@ -0,0 +1,102 @@ +#' Exponential Linear Unit (ELU) Activation Function +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' ELU(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=SELU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +ELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 1L) +} +#' Derivatives of the Exponential Linear Unit (ELU) Activation Function +#' +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative(s) of the ELU function has been applied +#' to each element of `x`. +#' @export +#' @author Matthew L. Fidler +#' @family Activation Functions +#' @examples +#' dELU(c(-1, 0, 1, 2), 2) +#' d2ELU(c(-1, 0, 1, 2), 2) +#' d2aELU(c(-1, 0, 1, 2), 2) +#' dELUa(c(-1, 0, 1, 2), 2) +#' d2ELUa(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' r <- rxode2({ +#' r1=dELU(time, 2) +#' r2=d2ELU(time, 2) +#' r2a=d2aELU(time, 2) +#' ra=dELUa(time, 2) +#' r2a=d2ELUa(time, 2) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(r, e) +dELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 2L) +} + +#' @rdname dELU +#' @export +d2ELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 3L) +} + +#' @rdname dELU +#' @export +d2aELU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 4L) +} + +#' @rdname dELU +#' @export +dELUa <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 5L) +} + +#' @rdname dELU +#' @export +d2ELUa <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 6L) +} diff --git a/R/gelu.R b/R/gelu.R new file mode 100644 index 000000000..e04a7e6f7 --- /dev/null +++ b/R/gelu.R @@ -0,0 +1,68 @@ + +#' GELU activation function +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @examples +#' +#' GELU(c(-2, -1, 0, 1, 2)) +#' +#' # you can use rxode2 as well +#' r <- rxode2({ +#' r = GELU(time) +#' }) +#' et <- et(c(-2, -1, 0, 1, 2)) +#' rxSolve(r, et) +#' +GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 1L) +} + + +#' Derivatives of GELU +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @examples +#' dGELU(c(-2, -1, 0, 1, 2)) +#' d2GELU(c(-2, -1, 0, 1, 2)) +#' d3GELU(c(-2, -1, 0, 1, 2)) +#' d4GELU(c(-2, -1, 0, 1, 2)) +#' # you can use rxode2 as well +#' r <- rxode2({ +#' r1 <- dGELU(time) +#' r2 <- d2GELU(time) +#' r3 <- d3GELU(time) +#' r4 <- d4GELU(time) +#' }) +#' et <- et(c(-2, -1, 0, 1, 2)) +#' rxSolve(r, et) +dGELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 9L) +} + +#' @rdname dGELU +#' @export +d2GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 10L) +} + +#' @rdname dGELU +#' @export +d3GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 11L) +} + +#' @rdname dGELU +#' @export +d4GELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 12L) +} diff --git a/R/lrelu.R b/R/lrelu.R new file mode 100644 index 000000000..306584ab0 --- /dev/null +++ b/R/lrelu.R @@ -0,0 +1,40 @@ +#' Leaky ReLU activation function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @examples +#' +#' lReLU(c(-1, 0, 1)) +#' +#' # Can use in rxode2 as well +#' +#' r <- rxode2({r <- lReLU(time)}) +#' e <- et(c(-1, 0, 1)) +#' rxSolve(r, e) +lReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 5L) +} + +#' Derivative of Leaky ReLU activation function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @examples +#' +#' dlReLU(c(-1, 0, 1)) +#' +#' # Can use in rxode2 as well +#' +#' r <- rxode2({r <- dlReLU(time)}) +#' e <- et(c(-1, 0, 1)) +#' rxSolve(r, e) +#' +dlReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 8L) +} diff --git a/R/parseFuns.R b/R/parseFuns.R index 397d90787..aded6523c 100644 --- a/R/parseFuns.R +++ b/R/parseFuns.R @@ -32,10 +32,14 @@ "llikXUnifDalpha", "llikXUnifDbeta", "llikXWeibull", "llikXWeibullDshape", "llikXWeibullDscale", "llikXGamma", "llikXGammaDshape", "llikXGammaDrate", "llikXCauchy", "llikXCauchyDlocation", "llikXCauchyDscale", "llikXNorm", -"llikXNormDmean", "llikXNormDsd", "linCmt", "rnorm", "rxnorm", -"rxbinom", "rbinom", "rxcauchy", "rcauchy", "rchisq", "rxchisq", -"rexp", "rxexp", "rbeta", "rxbeta", "rgeom", "rxgeom", "rxpois", -"rpois", "rxt", "rt") +"llikXNormDmean", "llikXNormDsd", "ReLU", "dReLU", "GELU", "dGELU", +"d2GELU", "d3GELU", "d4GELU", "ELU", "dELU", "d2ELU", "d2aELU", +"dELUa", "d2ELUa", "softplus", "dsoftplus", "d2softplus", "d3softplus", +"d4softplus", "SELU", "dSELU", "lReLU", "dlReLU", "PReLU", "dPReLU", +"d2PReLU", "dPReLUa", "dPReLUa1", "Swish", "dSwish", "linCmt", +"rnorm", "rxnorm", "rxbinom", "rbinom", "rxcauchy", "rcauchy", +"rchisq", "rxchisq", "rexp", "rxexp", "rbeta", "rxbeta", "rgeom", +"rxgeom", "rxpois", "rpois", "rxt", "rt") .parseEnv$.parseNum <- c(lgamma = 1, abs = 1, acos = 1, acosh = 1, asin = 1, asinh = 1, atan = 1, atan2 = 2, atanh = 1, beta = 2, cos = 1, cosh = 1, erf = 1, erfc = 1, exp = 1, gamma = 1, linCmtA = 20, linCmtC = 20, @@ -70,5 +74,10 @@ llikXFDdf1 = 4, llikXFDdf2 = 4, llikXGeom = 3, llikXGeomDprob = 3, llikXUnif = 4, llikXUnifDalpha = 4, llikXUnifDbeta = 4, llikXWeibull = 4, llikXWeibullDshape = 4, llikXWeibullDscale = 4, llikXGamma = 4, llikXGammaDshape = 4, llikXGammaDrate = 4, llikXCauchy = 4, llikXCauchyDlocation = 4, -llikXCauchyDscale = 4, llikXNorm = 4, llikXNormDmean = 4, llikXNormDsd = 4 -) +llikXCauchyDscale = 4, llikXNorm = 4, llikXNormDmean = 4, llikXNormDsd = 4, +ReLU = 1, dReLU = 1, GELU = 1, dGELU = 1, d2GELU = 1, d3GELU = 1, +d4GELU = 1, ELU = 2, dELU = 2, d2ELU = 2, d2aELU = 2, dELUa = 2, +d2ELUa = 2, softplus = 1, dsoftplus = 1, d2softplus = 1, d3softplus = 1, +d4softplus = 1, SELU = 1, dSELU = 1, lReLU = 1, dlReLU = 1, PReLU = 2, +dPReLU = 2, d2PReLU = 2, dPReLUa = 2, dPReLUa1 = 2, Swish = 1, +dSwish = 1) diff --git a/R/prelu.R b/R/prelu.R new file mode 100644 index 000000000..a0aaef006 --- /dev/null +++ b/R/prelu.R @@ -0,0 +1,82 @@ +#' Parametric ReLU Activation Function +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' PReLU(c(-1, 0, 1, 2), 2) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=PReLU(time, 2) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +PReLU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 7L) +} +#' Derivatives Parametric ReLU Activation Function +#' +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @param alpha A numeric scalar. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative(s) of the ELU function has been applied +#' to each element of `x`. +#' @export +#' @author Matthew L. Fidler +#' @family Activation Functions +#' @examples +#' +#' dPReLU(c(-1, 0, 1, 2), 2) +#' dPReLUa(c(-1, 0, 1, 2), 2) +#' dPReLUa1(c(-1, 0, 1, 2), 2) +#' +#' +#' # Can also be used in rxode2: +#' r <- rxode2({ +#' r1=dPReLU(time, 2) +#' r2a=dPReLUa(time, 2) +#' ra=dPReLUa1(time, 2) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(r, e) +dPReLU <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 8L) +} + +#' @rdname dPReLU +#' @export +dPReLUa <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 9L) +} + +#' @rdname dPReLU +#' @export +dPReLUa1 <- function(x, alpha=1) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + checkmate::assertNumeric(alpha, finite=TRUE, any.missing=FALSE) + .df <- data.frame(x=x, alpha=alpha) + .Call(`_rxode2_activationF2`, .df$x, .df$alpha, 10L) +} diff --git a/R/relu.R b/R/relu.R new file mode 100644 index 000000000..e8a1d236c --- /dev/null +++ b/R/relu.R @@ -0,0 +1,57 @@ +#' Rectified Linear Unit (ReLU) Activation Function +#' +#' This function applies the Rectified Linear Unit (ReLU) activation function to the input numeric vector. +#' The ReLU function is defined as the positive part of its argument: \eqn{f(x) = max(0, x)}. +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' ReLU(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=ReLU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +ReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 2L) +} +#' Derivative of the Rectified Linear Unit (ReLU) Activation Function +#' +#' This function applies the derivative of the Rectified Linear Unit +#' (ReLU) activation function to the input numeric vector. +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' +#' @family Activation Functions +#' +#' @return A numeric vector where the derivative of the ReLU function +#' @export +#' @examples +#' +#' dReLU(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=dReLU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +dReLU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 7L) +} diff --git a/R/selu.R b/R/selu.R new file mode 100644 index 000000000..804cc0f08 --- /dev/null +++ b/R/selu.R @@ -0,0 +1,47 @@ +#' Scaled Exponential Linear Unit (SELU) Activation Function +#' +#' @family Activation Functions +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' @author Matthew Fidler +#' @export +#' @examples +#' +#' SELU(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=SELU(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +SELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 4L) +} + +#' Derivative of the Scaled Exponential Linear Unit (SELU) Activation Function +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative of the SELU function +#' has been applied to each element of `x`. +#' @author Matthew Fidler +#' @family Activation Functions +#' @export +#' @examples +#' dSELU(c(-1, 0, 1, 2)) +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r=dSELU(time) +#' }) +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(x, e) +dSELU <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 17L) +} diff --git a/R/softplus.R b/R/softplus.R new file mode 100644 index 000000000..0a069706c --- /dev/null +++ b/R/softplus.R @@ -0,0 +1,69 @@ +#' Softplus Activation Function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @author Matthew L. Fidler +#' @examples +#' +#' softplus(c(-1, 0, 1, 2)) +#' +#' # You can use rxode2 too: +#' +#' r <- rxode2({ +#' s <- softplus(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(r, e) +#' +softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 3L) +} + +#' Default Softplus Activation Function +#' +#' @param x numeric vector +#' @return numeric vector +#' @family Activation Functions +#' @export +#' @author Matthew L. Fidler +#' @examples +#' +#' dsoftplus(c(-1, 0, 1, 2)) +#' +#' # You can use rxode2 too: +#' +#' r <- rxode2({ +#' s1 <- dsoftplus(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(r, e) +#' +dsoftplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 13L) +} + +#' @rdname dsoftplus +d2softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 14L) +} + +#' @rdname dsoftplus +d3softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 15L) +} + +#' @rdname dsoftplus +d4softplus <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 16L) +} diff --git a/R/swish.R b/R/swish.R new file mode 100644 index 000000000..737ac4a1e --- /dev/null +++ b/R/swish.R @@ -0,0 +1,58 @@ +#' Switch Activation Function +#' +#' The switch activation function is defined as: +#' +#' \deqn{f(x) = x \cdot \text{sigmoid}(x)} +#' +#' @family Activation Functions +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' +#' @return A numeric vector where the ReLU function has been applied +#' to each element of `x`. +#' +#' @author Matthew Fidler +#' +#' @export +#' +#' @examples +#' +#' Swish(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r<- Swish(time) +#' }) +#' +#' e <- et(c(-1, 0, 1, 2)) +#' +#' rxSolve(x, e) +#' +Swish <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 6L) +} + +#' Derivative of the Swish Activation Function +#' +#' @param x A numeric vector. All elements must be finite and +#' non-missing. +#' @return A numeric vector where the derivative of the SELU function +#' has been applied to each element of `x`. +#' @author Matthew Fidler +#' @family Activation Functions +#' @export +#' @examples +#' dSwish(c(-1, 0, 1, 2)) +#' +#' # Can also be used in rxode2: +#' x <- rxode2({ +#' r <- dSwish(time) +#' }) +#' e <- et(c(-1, 0, 1, 2)) +#' rxSolve(x, e) +dSwish <- function(x) { + checkmate::assertNumeric(x, finite=TRUE, any.missing=FALSE) + .Call(`_rxode2_activationF`, x, 18L) +} diff --git a/R/symengine.R b/R/symengine.R index 1bd0e3753..07bf6babc 100644 --- a/R/symengine.R +++ b/R/symengine.R @@ -718,7 +718,6 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, } } - .rxToSECurlyBrace <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { .x2 <- x[-1] if (progress) { @@ -1197,6 +1196,77 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, } } +.rxToSEd4GELU <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "d4GELU") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + return( + paste0("exp(-(", .x, ")^2/2)*(7*(", .x, ")^2 - 4 - (", .x, ")^4)/sqrt(2*pi)") + ) + } else { + stop("'d4GELU' can only take 1 argument", call. = FALSE) + } +} + +.rxToSEd4softplus <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "d4softplus") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + .ex1 <- paste0("(1.0 + exp(-(", .x, ")))") + return( + paste0("6.0*exp(-3.0*(", .x, "))/((", .ex1, + ")^4) - 6.0*exp(-2.0*(", .x, "))/((", .ex1, + ")^3) + exp(-(", .x, "))/((", .ex1, ")^2)") + ) + } else { + stop("'d4softplus' can only take 1 argument", call. = FALSE) + } +} + +.rxToSEdSELU <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "dSELU") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + return( + paste0("(rxGt(", .x, ", 0)*1.0507009873554804934193349852946 + 1.0507009873554804934193349852946*1.6732632423543772848170429916717*exp(", .x, ")*rxLeq(", .x, ", 0))") + ) + } else { + stop("'dSELU' can only take 1 argument", call. = FALSE) + } + +} + +.rxToSEdSwish <- function(x, envir=NULL, progress=FALSE, isEnv=TRUE) { + if (length(x) == 2) { + if (isEnv) { + .lastCall <- envir$..curCall + envir$..curCall <- c(envir$..curCall, "dSwish") + } + .x <- .rxToSE(x[[2]], envir = envir) + if (isEnv) envir$..curCall <- .lastCall + # x*exp(-x)/(1.0 + exp(-x))^2 + (1.0 + exp(-x))^(-1); + return( + paste0("((", .x, ")*exp(-(", .x, "))/(1.0 + exp(-(", .x, + ")))^2 + 1.0/(1.0 + exp(-(", .x, ")))") + ) + } else { + stop("'dSwish' can only take 1 argument", call. = FALSE) + } + +} + .rxToSETransit <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { if (length(x) == 4) { ## transit(n, mtt, bio) @@ -1256,6 +1326,7 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, return(.rxToSEMax(c(.ret, .xrest), min=min)) } + .rxToSECall <- function(x, envir = NULL, progress = FALSE, isEnv=TRUE) { if (identical(x[[1]], quote(`(`))) { return(paste0("(", .rxToSE(x[[2]], envir = envir), ")")) @@ -1315,6 +1386,14 @@ rxToSE <- function(x, envir = NULL, progress = FALSE, if (length(x) != 2) stop("abs only takes 1 argument", call.=FALSE) .r <- .rxToSE(x[[2]], envir = envir) return(paste0("(2.0*(", .r, ")*rxGt(", .r, ",0.0)-(", .r, "))")) + } else if (identical(x[[1]], quote(`d4GELU`))) { + return(.rxToSEd4GELU(x, envir = envir, progress = progress, isEnv=isEnv)) + } else if (identical(x[[1]], quote(`d4softplus`))) { + return(.rxToSEd4softplus(x, envir = envir, progress = progress, isEnv=isEnv)) + } else if (identical(x[[1]], quote(`dSELU`))) { + .rxToSEdSELU(x, envir=envir, progress=progress, isEnv=isEnv) + } else if (identical(x[[1]], quote(`dSwish`))) { + .rxToSEdSwish(x, envir=envir, progress=progress, isEnv=isEnv) } else { if (length(x[[1]]) == 1) { .x1 <- as.character(x[[1]]) diff --git a/_pkgdown.yml b/_pkgdown.yml index 2feb7bf3c..5053e5034 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -236,6 +236,24 @@ reference: - rxode2parseGetTranslation - summary.rxode2 - rxUiDeparse +- title: Neural Network Activation functions + contents: + - ELU + - GELU + - PReLU + - ReLU + - SELU + - Swish + - dELU + - dGELU + - dPReLU + - dReLU + - dSELU + - dSwish + - dlReLU + - dsoftplus + - lReLU + - softplus - title: Rxode2 data sets contents: - rxResidualError diff --git a/inst/include/rxode2_model_shared.c b/inst/include/rxode2_model_shared.c index 95f642cb1..7edcb5933 100644 --- a/inst/include/rxode2_model_shared.c +++ b/inst/include/rxode2_model_shared.c @@ -25,6 +25,37 @@ _rx_asgn _rxode2_rxAssignPtr =NULL; _rx_asgn _rxQr =NULL; rxode2_fn phi; +rxode2_fn ReLU; +rxode2_fn dReLU; +rxode2_fn GELU; +rxode2_fn dGELU; +rxode2_fn d2GELU; +rxode2_fn d3GELU; +rxode2_fn d4GELU; +rxode2_fn softplus; +rxode2_fn dsoftplus; +rxode2_fn d2softplus; +rxode2_fn d3softplus; +rxode2_fn d4softplus; +rxode2_fn SELU; +rxode2_fn dSELU; +rxode2_fn lReLU; +rxode2_fn dlReLU; +rxode2_fn Swish; +rxode2_fn dSwish; + +rxode2_fn2 PReLU; +rxode2_fn2 dPReLU; +rxode2_fn2 dPReLUa; +rxode2_fn2 dPReLUa1; + +rxode2_fn2 ELU; +rxode2_fn2 dELU; +rxode2_fn2 d2ELU; +rxode2_fn2 d2aELU; +rxode2_fn2 dELUa; +rxode2_fn2 d2ELUa; + rxode2_fn3 logit; rxode2_fn3 expit; rxode2_fn2 gammap; @@ -310,6 +341,34 @@ void _assignFuns0(void) { rit_ = (rxode2i2_fn)R_GetCCallable("rxode2","rit_"); riunif = (rxode2i2_fn2)R_GetCCallable("rxode2","riunif"); riweibull = (rxode2i2_fn2)R_GetCCallable("rxode2","riweibull"); + ReLU = (rxode2_fn)R_GetCCallable("rxode2","ReLU"); + dReLU = (rxode2_fn)R_GetCCallable("rxode2","dReLU"); + GELU = (rxode2_fn)R_GetCCallable("rxode2","GELU"); + dGELU = (rxode2_fn)R_GetCCallable("rxode2","dGELU"); + d2GELU = (rxode2_fn)R_GetCCallable("rxode2","d2GELU"); + d3GELU = (rxode2_fn)R_GetCCallable("rxode2","d3GELU"); + d4GELU = (rxode2_fn)R_GetCCallable("rxode2","d4GELU"); + softplus = (rxode2_fn)R_GetCCallable("rxode2","softplus"); + dsoftplus = (rxode2_fn)R_GetCCallable("rxode2","dsoftplus"); + d2softplus = (rxode2_fn)R_GetCCallable("rxode2","d2softplus"); + d3softplus = (rxode2_fn)R_GetCCallable("rxode2","d3softplus"); + d4softplus = (rxode2_fn)R_GetCCallable("rxode2","d4softplus"); + SELU = (rxode2_fn)R_GetCCallable("rxode2","SELU"); + dSELU = (rxode2_fn)R_GetCCallable("rxode2","dSELU"); + lReLU = (rxode2_fn)R_GetCCallable("rxode2","lReLU"); + dlReLU = (rxode2_fn)R_GetCCallable("rxode2","dlReLU"); + Swish = (rxode2_fn)R_GetCCallable("rxode2","Swish"); + dSwish = (rxode2_fn)R_GetCCallable("rxode2","dSwish"); + PReLU = (rxode2_fn2)R_GetCCallable("rxode2","PReLU"); + dPReLU = (rxode2_fn2)R_GetCCallable("rxode2","dPReLU"); + dPReLUa = (rxode2_fn2)R_GetCCallable("rxode2","dPReLUa"); + dPReLUa1 = (rxode2_fn2)R_GetCCallable("rxode2","dPReLUa1"); + ELU = (rxode2_fn2)R_GetCCallable("rxode2","ELU"); + dELU = (rxode2_fn2)R_GetCCallable("rxode2","dELU"); + d2ELU = (rxode2_fn2)R_GetCCallable("rxode2","d2ELU"); + d2aELU = (rxode2_fn2)R_GetCCallable("rxode2","d2aELU"); + dELUa = (rxode2_fn2)R_GetCCallable("rxode2","dELUa"); + d2ELUa = (rxode2_fn2)R_GetCCallable("rxode2","d2ELUa"); phi = (rxode2_fn)R_GetCCallable("rxode2","phi"); gammap = (rxode2_fn2) R_GetCCallable("rxode2","gammap"); diff --git a/inst/include/rxode2_model_shared.h b/inst/include/rxode2_model_shared.h index a7d520858..073000d55 100644 --- a/inst/include/rxode2_model_shared.h +++ b/inst/include/rxode2_model_shared.h @@ -19,140 +19,6 @@ static inline double _safe_log_(double a, rx_solve *rx) { } } -static inline double ReLU(double x) { - return (x > 0.0) ? x : 0.0; -} - -static inline double dReLU(double x) { - return (x > 0.0) ? 1.0 : 0.0; -} - -// -static inline double GELU(double x) { - return 0.5 * x * (1.0 + erf(x * M_SQRT1_2)); -} - -static inline double dGELU(double x) { - return 0.5 * (1.0 + erf(x * M_SQRT1_2)) + x * M_1_SQRT_2PI * exp(-0.5 * x * x); -} - -static inline double d2GELU(double x) { - return (2.0- x*x) * exp(-0.5* x * x)*M_1_SQRT_2PI; -} - -static inline double d3GELU(double x) { - return x * exp(-0.5 * x * x) * M_1_SQRT_2PI * (x * x - 4.0); -} - -static inline double d4GELU(double x) { - return exp(-0.5*x*x)*M_1_SQRT_2PI*(7.0*x*x - 4.0 - x*x*x*x); -} - -static inline double ELU(double x, double alpha) { - return (x > 0.0) ? x : (exp(x) - 1.0) * alpha; -} - -// derivative of ELU with respect to x -static inline double dELU(double x, double alpha) { - return (x > 0.0) ? 1.0 : exp(x)*alpha; -} - -// derivative of dELU with respect to x -static inline double d2ELU(double x, double alpha) { - return (x > 0.0) ? 0.0 : exp(x)*alpha; -} - -// derivative of dELU with respect to alpha -static inline double d2aELU(double x, double alpha) { - return (x > 0.0) ? 0.0 : exp(x); -} - -// derivative of ELU with respect to alpha -static inline double dELUa(double x, double alpha) { - return (x > 0.0) ? 0.0 : (exp(x) - 1.0); -} -// derivative of dELAa with respect to x -static inline double d2ELUa(double x, double alpha) { - return (x > 0.0) ? 0.0 : exp(x); -} - -static inline double softplus(double x) { - return log(1.0 + exp(x)); -} - -static inline double dsoftplus(double x) { - return 1.0 / (1.0 + exp(-x)); -} - -static inline double d2softplus(double x) { - double ex = exp(x); - return ex / ((1.0 + ex) * (1.0 + ex)); -} - -static inline double d3softplus(double x) { - double ex = exp(-x); - double ex1 = (1.0 + ex); - return 2.0*exp(-2.0*x)/(ex1*ex1*ex1) - 1.0*ex/(ex1*ex1); -} - -static inline double d4softplus(double x) { - double ex = exp(-x); - double ex1 = (1.0 + ex); - return 6.0*exp(-3.0*x)/(ex1*ex1*ex1*ex1) - - 6.0*exp(-2.0*x)/(ex1*ex1*ex1) + - 1.0*ex/(ex1*ex1); -} - -static inline double SELU(double x) { -#define alpha 1.6732632423543772848170429916717 -#define scale 1.0507009873554804934193349852946 - return (x > 0.0) ? scale * x : scale * alpha * (exp(x) - 1.0); -#undef alpha -#undef scale -} - -static inline double dSELU(double x) { -#define alpha 1.6732632423543772848170429916717 -#define scale 1.0507009873554804934193349852946 - return (x > 0.0) ? scale : scale * alpha * exp(x); -#undef alpha -#undef scale -} - -static inline double lReLU(double x) { - return (x > 0.0) ? x : 0.01 * x; -} - -static inline double dlReLU(double x) { - return (x > 0.0) ? 1.0 : 0.01; -} - -static inline double PReLU(double x, double alpha) { - return (x >= 0.0) ? x : alpha * x; -} - -static inline double dPReLU(double x, double alpha) { - return (x >= 0.0) ? 1.0 : alpha; -} - -static inline double dPReLUa(double x, double alpha) { - return (x >= 0.0) ? 0.0 : x; -} - -static inline double dPReLUa1(double x, double alpha) { - return (x >= 0.0) ? 0.0 : 1.0; -} - -static inline double Swish(double x) { - return x / (1.0 + exp(-x)); -} - -static inline double dSwish(double x) { - double ex = exp(x); - double den = 1.0 + ex; - return ex / (den * den) + x * ex / (den * den); -} - #define _safe_log(a) _safe_log_(a, _solveData) static inline double _div0_(double denom, rx_solve *rx) { if (rx->safeZero) { diff --git a/man/ELU.Rd b/man/ELU.Rd new file mode 100644 index 000000000..12b2ce0ab --- /dev/null +++ b/man/ELU.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/elu.R +\name{ELU} +\alias{ELU} +\title{Exponential Linear Unit (ELU) Activation Function} +\usage{ +ELU(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +Exponential Linear Unit (ELU) Activation Function +} +\examples{ + +ELU(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +x <- rxode2({ + r=SELU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/GELU.Rd b/man/GELU.Rd new file mode 100644 index 000000000..19b552b49 --- /dev/null +++ b/man/GELU.Rd @@ -0,0 +1,48 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gelu.R +\name{GELU} +\alias{GELU} +\title{GELU activation function} +\usage{ +GELU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +GELU activation function +} +\examples{ + +GELU(c(-2, -1, 0, 1, 2)) + +# you can use rxode2 as well +r <- rxode2({ + r = GELU(time) +}) +et <- et(c(-2, -1, 0, 1, 2)) +rxSolve(r, et) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\concept{Activation Functions} diff --git a/man/PReLU.Rd b/man/PReLU.Rd new file mode 100644 index 000000000..ea74e0def --- /dev/null +++ b/man/PReLU.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prelu.R +\name{PReLU} +\alias{PReLU} +\title{Parametric ReLU Activation Function} +\usage{ +PReLU(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +Parametric ReLU Activation Function +} +\examples{ + +PReLU(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +x <- rxode2({ + r=PReLU(time, 2) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/ReLU.Rd b/man/ReLU.Rd new file mode 100644 index 000000000..698f7d6c5 --- /dev/null +++ b/man/ReLU.Rd @@ -0,0 +1,56 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/relu.R +\name{ReLU} +\alias{ReLU} +\title{Rectified Linear Unit (ReLU) Activation Function} +\usage{ +ReLU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +This function applies the Rectified Linear Unit (ReLU) activation function to the input numeric vector. +The ReLU function is defined as the positive part of its argument: \eqn{f(x) = max(0, x)}. +} +\examples{ + +ReLU(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r=ReLU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/SELU.Rd b/man/SELU.Rd new file mode 100644 index 000000000..31bdef1be --- /dev/null +++ b/man/SELU.Rd @@ -0,0 +1,55 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/selu.R +\name{SELU} +\alias{SELU} +\title{Scaled Exponential Linear Unit (SELU) Activation Function} +\usage{ +SELU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +Scaled Exponential Linear Unit (SELU) Activation Function +} +\examples{ + +SELU(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r=SELU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/Swish.Rd b/man/Swish.Rd new file mode 100644 index 000000000..a12b2336b --- /dev/null +++ b/man/Swish.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/swish.R +\name{Swish} +\alias{Swish} +\title{Switch Activation Function} +\usage{ +Swish(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the ReLU function has been applied +to each element of \code{x}. +} +\description{ +The switch activation function is defined as: +} +\details{ +\deqn{f(x) = x \cdot \text{sigmoid}(x)} +} +\examples{ + +Swish(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r<- Swish(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dELU.Rd b/man/dELU.Rd new file mode 100644 index 000000000..f4c4d65ce --- /dev/null +++ b/man/dELU.Rd @@ -0,0 +1,75 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/elu.R +\name{dELU} +\alias{dELU} +\alias{d2ELU} +\alias{d2aELU} +\alias{dELUa} +\alias{d2ELUa} +\title{Derivatives of the Exponential Linear Unit (ELU) Activation Function} +\usage{ +dELU(x, alpha = 1) + +d2ELU(x, alpha = 1) + +d2aELU(x, alpha = 1) + +dELUa(x, alpha = 1) + +d2ELUa(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative(s) of the ELU function has been applied +to each element of \code{x}. +} +\description{ +Derivatives of the Exponential Linear Unit (ELU) Activation Function +} +\examples{ +dELU(c(-1, 0, 1, 2), 2) +d2ELU(c(-1, 0, 1, 2), 2) +d2aELU(c(-1, 0, 1, 2), 2) +dELUa(c(-1, 0, 1, 2), 2) +d2ELUa(c(-1, 0, 1, 2), 2) + +# Can also be used in rxode2: +r <- rxode2({ + r1=dELU(time, 2) + r2=d2ELU(time, 2) + r2a=d2aELU(time, 2) + ra=dELUa(time, 2) + r2a=d2ELUa(time, 2) +}) + +e <- et(c(-1, 0, 1, 2)) +rxSolve(r, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/man/dGELU.Rd b/man/dGELU.Rd new file mode 100644 index 000000000..e551f8be3 --- /dev/null +++ b/man/dGELU.Rd @@ -0,0 +1,60 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gelu.R +\name{dGELU} +\alias{dGELU} +\alias{d2GELU} +\alias{d3GELU} +\alias{d4GELU} +\title{Derivatives of GELU} +\usage{ +dGELU(x) + +d2GELU(x) + +d3GELU(x) + +d4GELU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Derivatives of GELU +} +\examples{ +dGELU(c(-2, -1, 0, 1, 2)) +d2GELU(c(-2, -1, 0, 1, 2)) +d3GELU(c(-2, -1, 0, 1, 2)) +d4GELU(c(-2, -1, 0, 1, 2)) +# you can use rxode2 as well +r <- rxode2({ + r1 <- dGELU(time) + r2 <- d2GELU(time) + r3 <- d3GELU(time) + r4 <- d4GELU(time) +}) +et <- et(c(-2, -1, 0, 1, 2)) +rxSolve(r, et) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\concept{Activation Functions} diff --git a/man/dPReLU.Rd b/man/dPReLU.Rd new file mode 100644 index 000000000..3b2f4295f --- /dev/null +++ b/man/dPReLU.Rd @@ -0,0 +1,67 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prelu.R +\name{dPReLU} +\alias{dPReLU} +\alias{dPReLUa} +\alias{dPReLUa1} +\title{Derivatives Parametric ReLU Activation Function} +\usage{ +dPReLU(x, alpha = 1) + +dPReLUa(x, alpha = 1) + +dPReLUa1(x, alpha = 1) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} + +\item{alpha}{A numeric scalar. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative(s) of the ELU function has been applied +to each element of \code{x}. +} +\description{ +Derivatives Parametric ReLU Activation Function +} +\examples{ + +dPReLU(c(-1, 0, 1, 2), 2) +dPReLUa(c(-1, 0, 1, 2), 2) +dPReLUa1(c(-1, 0, 1, 2), 2) + + +# Can also be used in rxode2: +r <- rxode2({ + r1=dPReLU(time, 2) + r2a=dPReLUa(time, 2) + ra=dPReLUa1(time, 2) +}) + +e <- et(c(-1, 0, 1, 2)) +rxSolve(r, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/man/dReLU.Rd b/man/dReLU.Rd new file mode 100644 index 000000000..1345fa681 --- /dev/null +++ b/man/dReLU.Rd @@ -0,0 +1,51 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/relu.R +\name{dReLU} +\alias{dReLU} +\title{Derivative of the Rectified Linear Unit (ReLU) Activation Function} +\usage{ +dReLU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative of the ReLU function +} +\description{ +This function applies the derivative of the Rectified Linear Unit +(ReLU) activation function to the input numeric vector. +} +\examples{ + +dReLU(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r=dReLU(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(x, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\concept{Activation Functions} diff --git a/man/dSELU.Rd b/man/dSELU.Rd new file mode 100644 index 000000000..f1e3005fa --- /dev/null +++ b/man/dSELU.Rd @@ -0,0 +1,50 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/selu.R +\name{dSELU} +\alias{dSELU} +\title{Derivative of the Scaled Exponential Linear Unit (SELU) Activation Function} +\usage{ +dSELU(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative of the SELU function +has been applied to each element of \code{x}. +} +\description{ +Derivative of the Scaled Exponential Linear Unit (SELU) Activation Function +} +\examples{ +dSELU(c(-1, 0, 1, 2)) +# Can also be used in rxode2: +x <- rxode2({ + r=dSELU(time) +}) +e <- et(c(-1, 0, 1, 2)) +rxSolve(x, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dSwish.Rd b/man/dSwish.Rd new file mode 100644 index 000000000..db244e29e --- /dev/null +++ b/man/dSwish.Rd @@ -0,0 +1,51 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/swish.R +\name{dSwish} +\alias{dSwish} +\title{Derivative of the Swish Activation Function} +\usage{ +dSwish(x) +} +\arguments{ +\item{x}{A numeric vector. All elements must be finite and +non-missing.} +} +\value{ +A numeric vector where the derivative of the SELU function +has been applied to each element of \code{x}. +} +\description{ +Derivative of the Swish Activation Function +} +\examples{ +dSwish(c(-1, 0, 1, 2)) + +# Can also be used in rxode2: +x <- rxode2({ + r <- dSwish(time) +}) +e <- et(c(-1, 0, 1, 2)) +rxSolve(x, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew Fidler +} +\concept{Activation Functions} diff --git a/man/dlReLU.Rd b/man/dlReLU.Rd new file mode 100644 index 000000000..cd1f2e188 --- /dev/null +++ b/man/dlReLU.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/lrelu.R +\name{dlReLU} +\alias{dlReLU} +\title{Derivative of Leaky ReLU activation function} +\usage{ +dlReLU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Derivative of Leaky ReLU activation function +} +\examples{ + +dlReLU(c(-1, 0, 1)) + +# Can use in rxode2 as well + +r <- rxode2({r <- dlReLU(time)}) +e <- et(c(-1, 0, 1)) +rxSolve(r, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\concept{Activation Functions} diff --git a/man/dsoftplus.Rd b/man/dsoftplus.Rd new file mode 100644 index 000000000..019db5954 --- /dev/null +++ b/man/dsoftplus.Rd @@ -0,0 +1,63 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/softplus.R +\name{dsoftplus} +\alias{dsoftplus} +\alias{d2softplus} +\alias{d3softplus} +\alias{d4softplus} +\title{Default Softplus Activation Function} +\usage{ +dsoftplus(x) + +d2softplus(x) + +d3softplus(x) + +d4softplus(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Default Softplus Activation Function +} +\examples{ + +dsoftplus(c(-1, 0, 1, 2)) + +# You can use rxode2 too: + +r <- rxode2({ + s1 <- dsoftplus(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(r, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{lReLU}()}, +\code{\link{softplus}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/man/lReLU.Rd b/man/lReLU.Rd new file mode 100644 index 000000000..6ea8fc124 --- /dev/null +++ b/man/lReLU.Rd @@ -0,0 +1,46 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/lrelu.R +\name{lReLU} +\alias{lReLU} +\title{Leaky ReLU activation function} +\usage{ +lReLU(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Leaky ReLU activation function +} +\examples{ + +lReLU(c(-1, 0, 1)) + +# Can use in rxode2 as well + +r <- rxode2({r <- lReLU(time)}) +e <- et(c(-1, 0, 1)) +rxSolve(r, e) +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{softplus}()} +} +\concept{Activation Functions} diff --git a/man/softplus.Rd b/man/softplus.Rd new file mode 100644 index 000000000..631e058b0 --- /dev/null +++ b/man/softplus.Rd @@ -0,0 +1,54 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/softplus.R +\name{softplus} +\alias{softplus} +\title{Softplus Activation Function} +\usage{ +softplus(x) +} +\arguments{ +\item{x}{numeric vector} +} +\value{ +numeric vector +} +\description{ +Softplus Activation Function +} +\examples{ + +softplus(c(-1, 0, 1, 2)) + +# You can use rxode2 too: + +r <- rxode2({ + s <- softplus(time) +}) + +e <- et(c(-1, 0, 1, 2)) + +rxSolve(r, e) + +} +\seealso{ +Other Activation Functions: +\code{\link{ELU}()}, +\code{\link{GELU}()}, +\code{\link{PReLU}()}, +\code{\link{ReLU}()}, +\code{\link{SELU}()}, +\code{\link{Swish}()}, +\code{\link{dELU}()}, +\code{\link{dGELU}()}, +\code{\link{dPReLU}()}, +\code{\link{dReLU}()}, +\code{\link{dSELU}()}, +\code{\link{dSwish}()}, +\code{\link{dlReLU}()}, +\code{\link{dsoftplus}()}, +\code{\link{lReLU}()} +} +\author{ +Matthew L. Fidler +} +\concept{Activation Functions} diff --git a/src/init.c b/src/init.c index ac12333ff..08f6d0ce7 100644 --- a/src/init.c +++ b/src/init.c @@ -112,6 +112,35 @@ SEXP _gammaqInva(SEXP, SEXP); double expit(double, double, double); double logit(double, double, double); +double ReLU(double x); +double dReLU(double x); +double GELU(double x); +double dGELU(double x); +double d2GELU(double x); +double d3GELU(double x); +double d4GELU(double x); +double ELU(double x, double alpha); +double dELU(double x, double alpha); +double d2ELU(double x, double alpha); +double d2aELU(double x, double alpha); +double dELUa(double x, double alpha); +double d2ELUa(double x, double alpha); +double softplus(double x); +double dsoftplus(double x); +double d2softplus(double x); +double d3softplus(double x); +double d4softplus(double x); +double SELU(double x); +double dSELU(double x); +double lReLU(double x); +double dlReLU(double x); +double PReLU(double x, double alpha); +double dPReLU(double x, double alpha); +double dPReLUa(double x, double alpha); +double dPReLUa1(double x, double alpha); +double Swish(double x); +double dSwish(double x); + SEXP _linCmtParse(SEXP vars, SEXP inStr, SEXP verbose); SEXP _rxode2_linCmtGen(SEXP linCmt, SEXP vars, SEXP linCmtSens, SEXP verbose); @@ -540,8 +569,12 @@ SEXP _rxode2_rxode2Ptr(void) { SEXP _rxode2_powerD(SEXP, SEXP, SEXP, SEXP, SEXP, SEXP); +SEXP _rxode2_activationF(SEXP xS, SEXP typeS); +SEXP _rxode2_activationF2(SEXP xS, SEXP aS, SEXP typeS); void R_init_rxode2(DllInfo *info){ R_CallMethodDef callMethods[] = { + {"_rxode2_activationF2", (DL_FUNC) &_rxode2_activationF2, 3}, + {"_rxode2_activationF", (DL_FUNC) &_rxode2_activationF, 2}, {"_rxode2_itoletter", (DL_FUNC) &_rxode2_itoletter, 2}, {"_rxode2_itostr", (DL_FUNC) &_rxode2_itostr, 2}, {"_rxode2_powerD", (DL_FUNC) &_rxode2_powerD, 6}, @@ -716,7 +749,34 @@ void R_init_rxode2(DllInfo *info){ R_RegisterCCallable("rxode2", "getSilentErr", (DL_FUNC) &getSilentErr); R_RegisterCCallable("rxode2", "logit", (DL_FUNC) &logit); R_RegisterCCallable("rxode2", "expit", (DL_FUNC) &expit); - + R_RegisterCCallable("rxode2", "ReLU", (DL_FUNC) &ReLU); + R_RegisterCCallable("rxode2", "dReLU", (DL_FUNC) &dReLU); + R_RegisterCCallable("rxode2", "GELU", (DL_FUNC) &GELU); + R_RegisterCCallable("rxode2", "dGELU", (DL_FUNC) &dGELU); + R_RegisterCCallable("rxode2", "d2GELU", (DL_FUNC) &d2GELU); + R_RegisterCCallable("rxode2", "d3GELU", (DL_FUNC) &d3GELU); + R_RegisterCCallable("rxode2", "d4GELU", (DL_FUNC) &d4GELU); + R_RegisterCCallable("rxode2", "ELU", (DL_FUNC) &ELU); + R_RegisterCCallable("rxode2", "dELU", (DL_FUNC) &dELU); + R_RegisterCCallable("rxode2", "d2ELU", (DL_FUNC) &d2ELU); + R_RegisterCCallable("rxode2", "d2aELU", (DL_FUNC) &d2aELU); + R_RegisterCCallable("rxode2", "dELUa", (DL_FUNC) &dELUa); + R_RegisterCCallable("rxode2", "d2ELUa", (DL_FUNC) &d2ELUa); + R_RegisterCCallable("rxode2", "softplus", (DL_FUNC) &softplus); + R_RegisterCCallable("rxode2", "dsoftplus", (DL_FUNC) &dsoftplus); + R_RegisterCCallable("rxode2", "d2softplus", (DL_FUNC) &d2softplus); + R_RegisterCCallable("rxode2", "d3softplus", (DL_FUNC) &d3softplus); + R_RegisterCCallable("rxode2", "d4softplus", (DL_FUNC) &d4softplus); + R_RegisterCCallable("rxode2", "SELU", (DL_FUNC) &SELU); + R_RegisterCCallable("rxode2", "dSELU", (DL_FUNC) &dSELU); + R_RegisterCCallable("rxode2", "lReLU", (DL_FUNC) &lReLU); + R_RegisterCCallable("rxode2", "dlReLU", (DL_FUNC) &dlReLU); + R_RegisterCCallable("rxode2", "PReLU", (DL_FUNC) &PReLU); + R_RegisterCCallable("rxode2", "dPReLU", (DL_FUNC) &dPReLU); + R_RegisterCCallable("rxode2", "dPReLUa", (DL_FUNC) &dPReLUa); + R_RegisterCCallable("rxode2", "dPReLUa1", (DL_FUNC) &dPReLUa1); + R_RegisterCCallable("rxode2", "Swish", (DL_FUNC) &Swish); + R_RegisterCCallable("rxode2", "dSwish", (DL_FUNC) &dSwish); R_RegisterCCallable("rxode2", "powerDi", (DL_FUNC) &powerDi); R_RegisterCCallable("rxode2", "powerD", (DL_FUNC) &powerD); R_RegisterCCallable("rxode2", "powerDD", (DL_FUNC) &powerDD); diff --git a/src/utilc.c b/src/utilc.c index c9e2c7364..1c0eabfc2 100644 --- a/src/utilc.c +++ b/src/utilc.c @@ -596,6 +596,262 @@ double probitInv(double alpha, double low, double high) { return _powerDi(alpha, 1.0, 6, low, high); } +double ReLU(double x) { + return (x > 0.0) ? x : 0.0; +} + +double dReLU(double x) { + return (x > 0.0) ? 1.0 : 0.0; +} + +// +double GELU(double x) { + return 0.5 * x * (1.0 + erf(x * M_SQRT1_2)); +} + +double dGELU(double x) { + return 0.5 * (1.0 + erf(x * M_SQRT1_2)) + x * M_1_SQRT_2PI * exp(-0.5 * x * x); +} + +double d2GELU(double x) { + return (2.0- x*x) * exp(-0.5* x * x)*M_1_SQRT_2PI; +} + +double d3GELU(double x) { + double x2 = x*x; + return -8.0*x*exp(-x2*0.5)*0.5*M_1_SQRT_2PI + 4.0*x2*x*exp(-x2*0.5)*0.25*M_1_SQRT_2PI; +} + +double d4GELU(double x) { + return exp(-0.5*x*x)*M_1_SQRT_2PI*(7.0*x*x - 4.0 - x*x*x*x); +} + +double ELU(double x, double alpha) { + return (x > 0.0) ? x : (exp(x) - 1.0) * alpha; +} + +// derivative of ELU with respect to x +double dELU(double x, double alpha) { + return (x > 0.0) ? 1.0 : exp(x)*alpha; +} + +// derivative of dELU with respect to x +double d2ELU(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x)*alpha; +} + +// derivative of dELU with respect to alpha +double d2aELU(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x); +} + +// derivative of ELU with respect to alpha +double dELUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : (exp(x) - 1.0); +} +// derivative of dELAa with respect to x +double d2ELUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : exp(x); +} + +double softplus(double x) { + return log(1.0 + exp(x)); +} + +double dsoftplus(double x) { + return 1.0 / (1.0 + exp(-x)); +} + +double d2softplus(double x) { + double ex = exp(x); + return ex / ((1.0 + ex) * (1.0 + ex)); +} + +double d3softplus(double x) { + double ex = exp(-x); + double ex1 = (1.0 + ex); + return 2.0*exp(-2.0*x)/(ex1*ex1*ex1) - 1.0*ex/(ex1*ex1); +} + +double d4softplus(double x) { + double ex = exp(-x); + double ex1 = (1.0 + ex); + return 6.0*exp(-3.0*x)/(ex1*ex1*ex1*ex1) - + 6.0*exp(-2.0*x)/(ex1*ex1*ex1) + + ex/(ex1*ex1); +} + +double SELU(double x) { +#define alpha 1.6732632423543772848170429916717 +#define scale 1.0507009873554804934193349852946 + return (x > 0.0) ? scale * x : scale * alpha * (exp(x) - 1.0); +#undef alpha +#undef scale +} + +double dSELU(double x) { +#define alpha 1.6732632423543772848170429916717 +#define scale 1.0507009873554804934193349852946 + return (x > 0.0) ? scale : scale * alpha * exp(x); +#undef alpha +#undef scale +} + +double lReLU(double x) { + return (x >= 0.0) ? x : 0.01 * x; +} + +double dlReLU(double x) { + return (x > 0.0) ? 1.0 : 0.01; +} + +double PReLU(double x, double alpha) { + return (x > 0.0) ? x : alpha * x; +} + +double dPReLU(double x, double alpha) { + return (x > 0.0) ? 1.0 : alpha; +} + +double dPReLUa(double x, double alpha) { + return (x > 0.0) ? 0.0 : x; +} + +double dPReLUa1(double x, double alpha) { + return (x > 0.0) ? 0.0 : 1.0; +} + +double Swish(double x) { + return x / (1.0 + exp(-x)); +} + +double dSwish(double x) { + double ex = exp(-x); + double den = (1.0 + ex); + return x*ex/(den*den) + 1.0/den; +} + +SEXP _rxode2_activationF2(SEXP xS, SEXP aS, SEXP typeS) { + int type = INTEGER(typeS)[0]; + int typex = TYPEOF(xS); + int typea = TYPEOF(aS); + int lenx = Rf_length(xS); + SEXP ret = PROTECT(Rf_allocVector(REALSXP, lenx)); + for (int i = 0; i < lenx; ++i) { + double x = (typex == REALSXP) ? REAL(xS)[i] : (double)INTEGER(xS)[i]; + double a = (typea == REALSXP) ? REAL(aS)[i] : (double)INTEGER(aS)[i]; + switch (type) { + case 1: + REAL(ret)[i] = ELU(x, a); + break; + case 2: + REAL(ret)[i] = dELU(x, a); + break; + case 3: + REAL(ret)[i] = d2ELU(x, a); + break; + case 4: + REAL(ret)[i] = d2aELU(x, a); + break; + case 5: + REAL(ret)[i] = dELUa(x, a); + break; + case 6: + REAL(ret)[i] = d2ELUa(x, a); + break; + case 7: + REAL(ret)[i] = PReLU(x, a); + break; + case 8: + REAL(ret)[i] = dPReLU(x, a); + break; + case 9: + REAL(ret)[i] = dPReLUa(x, a); + break; + case 10: + REAL(ret)[i] = dPReLUa1(x, a); + break; + default: + REAL(ret)[i] = NA_REAL; + break; + } + } + UNPROTECT(1); + return(ret); +} + +SEXP _rxode2_activationF(SEXP xS, SEXP typeS) { + int type = INTEGER(typeS)[0]; + int typex = TYPEOF(xS); + int lenx = Rf_length(xS); + SEXP ret = PROTECT(Rf_allocVector(REALSXP, lenx)); + for (int i = 0; i < lenx; ++i) { + double x = (typex == REALSXP) ? REAL(xS)[i] : (double)INTEGER(xS)[i]; + switch (type) { + case 1: + REAL(ret)[i] = GELU(x); + break; + case 2: + REAL(ret)[i] = ReLU(x); + break; + case 3: + REAL(ret)[i] = softplus(x); + break; + case 4: + REAL(ret)[i] = SELU(x); + break; + case 5: + REAL(ret)[i] = lReLU(x); + break; + case 6: + REAL(ret)[i] = Swish(x); + break; + case 7: + REAL(ret)[i] = dReLU(x); + break; + case 8: + REAL(ret)[i] = dlReLU(x); + break; + case 9: + REAL(ret)[i] = dGELU(x); + break; + case 10: + REAL(ret)[i] = d2GELU(x); + break; + case 11: + REAL(ret)[i] = d3GELU(x); + break; + case 12: + REAL(ret)[i] = d4GELU(x); + break; + case 13: + REAL(ret)[i] = dsoftplus(x); + break; + case 14: + REAL(ret)[i] = d2softplus(x); + break; + case 15: + REAL(ret)[i] = d3softplus(x); + break; + case 16: + REAL(ret)[i] = d4softplus(x); + break; + case 17: + REAL(ret)[i] = dSELU(x); + break; + case 18: + REAL(ret)[i] = dSwish(x); + break; + default: + REAL(ret)[i] = NA_REAL; + break; + } + } + UNPROTECT(1); + return(ret); +} + + SEXP _rxode2_powerD(SEXP xS, SEXP lowS, SEXP highS, SEXP lambdaS, SEXP yjS, SEXP inverseS) { int typex = TYPEOF(xS); int typelow = TYPEOF(lowS); diff --git a/tests/testthat/test-activation.R b/tests/testthat/test-activation.R new file mode 100644 index 000000000..4e8be6da0 --- /dev/null +++ b/tests/testthat/test-activation.R @@ -0,0 +1,167 @@ +test_that("activation functions", { + + r <- rxode2({ + r <- ReLU(time) + dr <- dReLU(time) + # + l <- lReLU(time) + dl <- dlReLU(time) + # + p <- PReLU(time, alpha) + dp <- dPReLU(time, alpha) + dpa <- dPReLUa(time, alpha) + dpa1 <- dPReLUa1(time, alpha) + # + g <- GELU(time) + dg <- dGELU(time) + d2g <- d2GELU(time) + d3g <- d3GELU(time) + d4g <- d4GELU(time) + # + s <- Swish(time) + ds <- dSwish(time) + # + sp <- softplus(time) + dsp <- dsoftplus(time) + # + el <- ELU(time, alpha) + del <- dELU(time, alpha) + d2el <- d2ELU(time, alpha) + d2a <- dELUa(time, alpha) + d2ael <- d2aELU(time, alpha) + dael <- d2aELU(time, alpha) + d2aela <- d2ELUa(time, alpha) + }) + + e <- et(seq(-10, 10, length.out = 41)) + + s <- rxSolve(r, e, c(alpha=2)) + + expect_equal(ReLU(e$time), s$r) + expect_equal(ReLU(e$time), vapply(e$time, + function(x) { + max(0, x) + }, double(1))) + expect_equal(dReLU(e$time), s$dr) + expect_equal(dReLU(e$time), vapply(e$time, + function(x) { + (x > 0)*1 + }, double(1))) + expect_equal(lReLU(e$time), s$l) + expect_equal(lReLU(e$time), vapply(e$time, + function(x) { + (x > 0)*x + (x <= 0)*0.01*x + }, double(1))) + expect_equal(dlReLU(e$time), s$dl) + expect_equal(dlReLU(e$time), vapply(e$time, + function(x) { + (x > 0) + (x <= 0)*0.01 + }, double(1))) + + expect_equal(PReLU(e$time, 2), s$p) + expect_equal(PReLU(e$time, 2), vapply(e$time, + function(x) { + (x > 0)*x + (x <= 0)*2*x + }, double(1))) + + expect_equal(dPReLU(e$time, 2), s$dp) + expect_equal(dPReLU(e$time, 2), vapply(e$time, + function(x) { + (x > 0) + (x <= 0)*2 + }, double(1))) + + expect_equal(GELU(e$time), s$g) + expect_equal(GELU(e$time), vapply(e$time, + function(x) { + phi(x)*x + }, double(1))) + + expect_equal(dGELU(e$time), s$dg) + expect_equal(dGELU(e$time), vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + 1.0*x*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2/sqrt(pi) + 0.5*(1 + erf(x*M_SQRT1_2)) + }, double(1))) + expect_equal(d2GELU(e$time), s$d2g) + expect_equal(d2GELU(e$time), + vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + 2.0*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2/sqrt(pi) - 2.0*x^2*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^3/sqrt(pi) + }, double(1))) + + expect_equal(d3GELU(e$time), s$d3g) + expect_equal(d3GELU(e$time), + vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + M_1_SQRT_2PI <- 1/sqrt(2*pi) + -8.0*x*exp(-x^2*0.5)*0.5*M_1_SQRT_2PI + 4.0*x^3*exp(-x^2*0.5)*0.25*M_1_SQRT_2PI + }, double(1))) + expect_equal(d4GELU(e$time), s$d4g) + expect_equal(d4GELU(e$time), + vapply(e$time, + function(x) { + M_SQRT1_2 <- 1/sqrt(2) + -8.0*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^3/sqrt(pi) + 28.0*x^2*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^5/sqrt(pi) - 8.0*x^4*exp(-x^2*M_SQRT1_2^2)*M_SQRT1_2^7/sqrt(pi) + }, double(1))) + + expect_equal(Swish(e$time), s$s) + expect_equal(Swish(e$time), vapply(e$time, + function(x) { + x/(1 + exp(-x)) + }, double(1))) + + expect_equal(dSwish(e$time), s$ds) + expect_equal(dSwish(e$time), vapply(e$time, + function(x) { + x*exp(-x)/(1 + exp(-x))^2 + (1 + exp(-x))^(-1) + }, double(1))) + + expect_equal(softplus(e$time), s$sp) + expect_equal(softplus(e$time), vapply(e$time, + function(x) { + log(1 + exp(x)) + }, double(1))) + + expect_equal(dsoftplus(e$time), s$dsp) + expect_equal(dsoftplus(e$time), vapply(e$time, + function(x) { + exp(x)/(1 + exp(x)) + }, double(1))) + + expect_equal(ELU(e$time, 2), s$el) + expect_equal(ELU(e$time, 2), vapply(e$time, + function(x) { + (x > 0)*x + (x <= 0)*2*(exp(x) - 1) + }, double(1))) + + # dELU/dx + expect_equal(dELU(e$time, 2), s$del) + expect_equal(dELU(e$time, 2), vapply(e$time, + function(x) { + (x > 0) + (x <= 0)*2*exp(x) + }, double(1))) + + # d2ELU/d2x + expect_equal(d2ELU(e$time, 2), s$d2el) + expect_equal(d2ELU(e$time, 2), vapply(e$time, + function(x) { + (x <= 0)*2*exp(x) + }, double(1))) + + # dELU/dalpha + expect_equal(dELUa(e$time, 2), s$d2a) + expect_equal(dELUa(e$time, 2), vapply(e$time, + function(x) { + (x <= 0)*(exp(x) - 1) + }, double(1))) + + # d2ELU/dalphad2x + expect_equal(d2ELUa(e$time, 2), s$d2ael) + expect_equal(d2ELUa(e$time, 2), vapply(e$time, + function(x) { + (x <= 0)*exp(x) + }, double(1))) + +}) diff --git a/tests/testthat/test-dsl.R b/tests/testthat/test-dsl.R index cd0e3e14c..4ad417168 100644 --- a/tests/testthat/test-dsl.R +++ b/tests/testthat/test-dsl.R @@ -269,6 +269,8 @@ rxTest({ expect_equal(rxFromSE("Derivative(d2GELU(x), x)"), "d3GELU(x)") expect_equal(rxFromSE("Derivative(d3GELU(x), x)"), "d4GELU(x)") + expect_equal(rxToSE("d4GELU(x)"), "exp(-(x)^2/2)*(7*(x)^2 - 4 - (x)^4)/sqrt(2*pi)") + expect_equal(rxFromSE("Derivative(ELU(x, alpha), x)"), "dELU(x, alpha)") expect_equal(rxFromSE("Derivative(ELU(x, alpha), alpha)"), "dELUa(x, alpha)") @@ -289,8 +291,14 @@ rxTest({ expect_equal(rxFromSE("Derivative(d2softplus(x), x)"), "d3softplus(x)") expect_equal(rxFromSE("Derivative(d3softplus(x), x)"), "d4softplus(x)") + expect_equal(rxToSE("d4softplus(x)"), + "6.0*exp(-3.0*(x))/(((1.0 + exp(-(x))))^4) - 6.0*exp(-2.0*(x))/(((1.0 + exp(-(x))))^3) + exp(-(x))/(((1.0 + exp(-(x))))^2)") + expect_equal(rxFromSE("Derivative(SELU(x), x)"), "dSELU(x)") + expect_equal(rxToSE("dSELU(x)"), + "(rxGt(x, 0)*1.0507009873554804934193349852946 + 1.0507009873554804934193349852946*1.6732632423543772848170429916717*exp(x)*rxLeq(x, 0))") + expect_equal(rxFromSE("Derivative(lReLU(x), x)"), "dlReLU(x)") expect_equal(rxFromSE("Derivative(dlReLU(x), x)"), "0") @@ -308,6 +316,9 @@ rxTest({ expect_equal(rxFromSE("Derivative(Swish(x), x)"), "dSwish(x)") + expect_equal(rxToSE("dSwish(x)"), + "((x)*exp(-(x))/(1.0 + exp(-(x)))^2 + 1.0/(1.0 + exp(-(x)))") + }) test_that("logic tests", {