gelu_function.hpp

Go to the documentation of this file.

#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP

#include <mlpack/prereqs.hpp>

namespace mlpack {
namespace ann  {

class GELUFunction
{
 public:
  static double Fn(const double x)
  {
    return 0.5 * x * (1 + std::tanh(std::sqrt(2 / M_PI) *
           (x + 0.044715 * std::pow(x, 3))));
  }

  template<typename InputVecType, typename OutputVecType>
  static void Fn(const InputVecType& x, OutputVecType& y)
  {
    y = 0.5 * x % (1 + arma::tanh(std::sqrt(2 / M_PI) *
        (x + 0.044715 * arma::pow(x, 3))));
  }

  static double Deriv(const double y)
  {
    return 0.5 * std::tanh(0.0356774 * std::pow(y, 3) + 0.797885 * y) +
           (0.0535161 * std::pow(y, 3) + 0.398942 * y) *
           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) +
           0.797885 * y), 2) + 0.5;
  }

  template<typename InputVecType, typename OutputVecType>
  static void Deriv(const InputVecType& y, OutputVecType& x)
  {
    x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) %
        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) +
        0.797885 * y), 2) + 0.5;
  }
}; // class GELUFunction

} // namespace ann
} // namespace mlpack

#endif