activation_functions_test.cpp File Reference

#include <mlpack/core.hpp>
#include <mlpack/methods/ann/layer/layer.hpp>
#include <mlpack/methods/ann/activation_functions/logistic_function.hpp>
#include <mlpack/methods/ann/activation_functions/identity_function.hpp>
#include <mlpack/methods/ann/activation_functions/softsign_function.hpp>
#include <mlpack/methods/ann/activation_functions/tanh_function.hpp>
#include <mlpack/methods/ann/activation_functions/rectifier_function.hpp>
#include <mlpack/methods/ann/activation_functions/softplus_function.hpp>
#include <mlpack/methods/ann/activation_functions/swish_function.hpp>
#include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
#include <mlpack/methods/ann/activation_functions/mish_function.hpp>
#include <mlpack/methods/ann/activation_functions/lisht_function.hpp>
#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
#include <mlpack/methods/ann/activation_functions/elliot_function.hpp>
#include <mlpack/methods/ann/activation_functions/elish_function.hpp>
#include <mlpack/methods/ann/activation_functions/inverse_quadratic_function.hpp>
#include <mlpack/methods/ann/activation_functions/quadratic_function.hpp>
#include <mlpack/methods/ann/activation_functions/multi_quadratic_function.hpp>
#include <mlpack/methods/ann/activation_functions/spline_function.hpp>
#include <mlpack/methods/ann/activation_functions/poisson1_function.hpp>
#include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
#include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
#include <mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp>
#include <mlpack/methods/ann/activation_functions/silu_function.hpp>
#include "catch.hpp"

Include dependency graph for activation_functions_test.cpp:

Functions
const arma::colvec	activationData ("-2 3.2 4.5 -100.2 1 -1 2 0")
template<class ActivationFunction >
void	CheckActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the activation function test. More...
template<class ActivationFunction >
void	CheckDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the activation function derivative test. More...
template<class ActivationFunction >
void	CheckInverseCorrect (const arma::colvec input)
	Implementation of the activation function inverse test. More...
void	CheckHardTanHActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the HardTanH activation function test. More...
void	CheckHardTanHDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the HardTanH activation function derivative test. More...
void	CheckLeakyReLUActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the LeakyReLU activation function test. More...
void	CheckLeakyReLUDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the LeakyReLU activation function derivative test. More...
void	CheckELUActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the ELU activation function test. More...
void	CheckELUDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the ELU activation function derivative test. More...
void	CheckPReLUActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the PReLU activation function test. More...
void	CheckPReLUDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the PReLU activation function derivative test. More...
void	CheckPReLUGradientCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the PReLU activation function gradient test. More...
void	CheckHardShrinkActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Hard Shrink activation function test. More...
void	CheckHardShrinkDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the HardShrink activation function derivative test. More...
void	CheckSoftShrinkActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Soft Shrink activation function test. More...
void	CheckSoftShrinkDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Soft Shrink activation function derivative test. More...
	TEST_CASE ("SELUFunctionNormalizedTest", "[ActivationFunctionsTest]")
	Simple SELU activation test to check whether the mean and variance remain invariant after passing normalized inputs through the function.
	TEST_CASE ("SELUFunctionUnnormalizedTest", "[ActivationFunctionsTest]")
	Simple SELU activation test to check whether the mean and variance vary significantly after passing unnormalized inputs through the function.
	TEST_CASE ("SELUFunctionDerivativeTest", "[ActivationFunctionsTest]")
	Simple SELU derivative test to check whether the derivatives produced by the activation function are correct.
void	CheckCELUActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the CELU activation function test. More...
void	CheckCELUDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the CELU activation function derivative test. More...
void	CheckISRLUActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the ISRLU activation function test. More...
void	CheckISRLUDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the ISRLU activation function derivative test. More...
void	CheckSoftminActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Softmin activation function test. More...
void	CheckSoftminDerivativeCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Softmin activation function derivative test. More...
void	CheckFlattenTSwishActivationCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Flatten T Swish activation function test. More...
void	CheckFlattenTSwishDerivateCorrect (const arma::colvec input, const arma::colvec target)
	Implementation of the Softmin activation function derivative test. More...
void	CheckReLU6Correct (const arma::colvec input, const arma::colvec ActivationTarget, const arma::colvec DerivativeTarget)
	Implementation of the ReLU6 activation function derivative test. More...
	TEST_CASE ("ReLU6FunctionTest", "[ActivationFunctionsTest]")
	Basic test of the ReLU6 function.
	TEST_CASE ("TanhFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the tanh function.
	TEST_CASE ("LogisticFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the logistic function.
	TEST_CASE ("SoftsignFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the softsign function.
	TEST_CASE ("IdentityFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the identity function.
	TEST_CASE ("RectifierFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the rectifier function.
	TEST_CASE ("LeakyReLUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the LeakyReLU function.
	TEST_CASE ("HardTanHFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the HardTanH function.
	TEST_CASE ("ELUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the ELU function.
	TEST_CASE ("SoftplusFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the softplus function.
	TEST_CASE ("PReLUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the PReLU function.
	TEST_CASE ("CReLUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the CReLU function.
	TEST_CASE ("SwishFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the swish function.
	TEST_CASE ("HardSigmoidFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the hard sigmoid function.
	TEST_CASE ("MishFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Mish function.
	TEST_CASE ("LiSHTFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the LiSHT function.
	TEST_CASE ("GELUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the GELU function.
	TEST_CASE ("HardShrinkFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Hard Shrink function.
	TEST_CASE ("ElliotFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Elliot function.
	TEST_CASE ("ElishFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the EliSH function.
	TEST_CASE ("SoftShrinkFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Soft Shrink function.
	TEST_CASE ("CELUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the CELU activation function.
	TEST_CASE ("ISRLUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the ISRLU activation function.
	TEST_CASE ("InverseQuadraticFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the inverse quadratic function.
	TEST_CASE ("QuadraticFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the quadratic function.
	TEST_CASE ("SplineFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Spline function.
	TEST_CASE ("MultiquadFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the multi quadratic function.
	TEST_CASE ("Poisson1FunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Poisson one function.
	TEST_CASE ("GaussianFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Gaussian activation function.
	TEST_CASE ("SoftminFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Softmin function.
	TEST_CASE ("HardSwishFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the Hard Swish function.
	TEST_CASE ("TanhExpFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the TanhExp function.
	TEST_CASE ("SILUFunctionTest", "[ActivationFunctionsTest]")
	Basic test of the SILU(Sigmoid Weighted Linear Unit) Function.
	TEST_CASE ("FlattenTSwishFunctionTest", "[ActivationFunctionsTest]")
	Basic test of Flatten T Swish function.

Detailed Description

Author

Marcus Edel

Dhawal Arora

Tests for the various activation functions.

mlpack is free software; you may redistribute it and/or modify it under the terms of the 3-clause BSD license. You should have received a copy of the 3-clause BSD license along with mlpack. If not, see http://www.opensource.org/licenses/BSD-3-Clause for more information.

Function Documentation

CheckActivationCorrect()

template<class ActivationFunction >

void CheckActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the activation function test.

Parameters

input	Input data used for evaluating the activation function.
target	Target data used to evaluate the activation.

Template Parameters

ActivationFunction

Activation function used for the check.

CheckCELUActivationCorrect()

void CheckCELUActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the CELU activation function test.

The function is implemented as CELU layer in the file celu.hpp.

Parameters

input	Input data used for evaluating the CELU activation function.
target	Target data used to evaluate the CELU activation.

CheckCELUDerivativeCorrect()

void CheckCELUDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the CELU activation function derivative test.

The function is implemented as CELU layer in the file celu.hpp.

Parameters

input	Input data used for evaluating the CELU activation function.
target	Target data used to evaluate the CELU activation.

CheckDerivativeCorrect()

template<class ActivationFunction >

void CheckDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the activation function derivative test.

Parameters

input	Input data used for evaluating the activation function.
target	Target data used to evaluate the activation.

Template Parameters

ActivationFunction

Activation function used for the check.

CheckELUActivationCorrect()

void CheckELUActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the ELU activation function test.

The function is implemented as ELU layer in the file elu.hpp

Parameters

input	Input data used for evaluating the ELU activation function.
target	Target data used to evaluate the ELU activation.

CheckELUDerivativeCorrect()

void CheckELUDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the ELU activation function derivative test.

The function is implemented as ELU layer in the file elu.hpp

Parameters

input	Input data used for evaluating the ELU activation function.
target	Target data used to evaluate the ELU activation.

CheckFlattenTSwishActivationCorrect()

void CheckFlattenTSwishActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Flatten T Swish activation function test.

The function is implemented as Flatten T Swish layer in the file flatten_t_swish.hpp.

Parameters

input	Input data used for evaluating the Flatten T Swish activation function.
target	Target data used to evaluate the Flatten T Swish activation.

CheckFlattenTSwishDerivateCorrect()

void CheckFlattenTSwishDerivateCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Softmin activation function derivative test.

The function is implemented as Softmin layer in the file softmin.hpp.

Parameters

input	Input data used for evaluating the Softmin activation function.
target	Target data used to evaluate the Softmin activation.

CheckHardShrinkActivationCorrect()

void CheckHardShrinkActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Hard Shrink activation function test.

The function is implemented as Hard Shrink layer in the file hardshrink.hpp

Parameters

input	Input data used for evaluating the Hard Shrink activation function.
target	Target data used to evaluate the Hard Shrink activation.

CheckHardShrinkDerivativeCorrect()

void CheckHardShrinkDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the HardShrink activation function derivative test.

The derivative function is implemented as HardShrink layer in the file hardshrink.hpp

Parameters

input	Input data used for evaluating the HardShrink activation function.
target	Target data used to evaluate the HardShrink activation.

CheckHardTanHActivationCorrect()

void CheckHardTanHActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the HardTanH activation function test.

The function is implemented as a HardTanH Layer in hard_tanh.hpp

Parameters

input	Input data used for evaluating the HardTanH activation function.
target	Target data used to evaluate the HardTanH activation.

CheckHardTanHDerivativeCorrect()

void CheckHardTanHDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the HardTanH activation function derivative test.

The derivative is implemented as HardTanH Layer in hard_tanh.hpp

Parameters

input	Input data used for evaluating the HardTanH activation function.
target	Target data used to evaluate the HardTanH activation.

CheckInverseCorrect()

template<class ActivationFunction >

void CheckInverseCorrect

(

const arma::colvec

input

)

Implementation of the activation function inverse test.

Parameters

input	Input data used for evaluating the activation function.
target	Target data used to evaluate the activation.

Template Parameters

ActivationFunction

Activation function used for the check.

CheckISRLUActivationCorrect()

void CheckISRLUActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the ISRLU activation function test.

The function is implemented as ISRLU layer in the file isrlu.hpp.

Parameters

input	Input data used for evaluating the ISRLU activation function.
target	Target data used to evaluate the ISRLU activation.

CheckISRLUDerivativeCorrect()

void CheckISRLUDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the ISRLU activation function derivative test.

The function is implemented as ISRLU layer in the file isrlu.hpp.

Parameters

input	Input data used for evaluating the ISRLU activation function.
target	Target data used to evaluate the ISRLU activation.

CheckLeakyReLUActivationCorrect()

void CheckLeakyReLUActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the LeakyReLU activation function test.

The function is implemented as LeakyReLU layer in the file leaky_relu.hpp

Parameters

input	Input data used for evaluating the LeakyReLU activation function.
target	Target data used to evaluate the LeakyReLU activation.

CheckLeakyReLUDerivativeCorrect()

void CheckLeakyReLUDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the LeakyReLU activation function derivative test.

The derivative function is implemented as LeakyReLU layer in the file leaky_relu_layer.hpp

Parameters

input	Input data used for evaluating the LeakyReLU activation function.
target	Target data used to evaluate the LeakyReLU activation.

CheckPReLUActivationCorrect()

void CheckPReLUActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the PReLU activation function test.

The function is implemented as PReLU layer in the file parametric_relu.hpp.

Parameters

input	Input data used for evaluating the PReLU activation function.
target	Target data used to evaluate the PReLU activation.

CheckPReLUDerivativeCorrect()

void CheckPReLUDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the PReLU activation function derivative test.

The function is implemented as PReLU layer in the file parametric_relu.hpp

Parameters

input	Input data used for evaluating the PReLU activation function.
target	Target data used to evaluate the PReLU activation.

CheckPReLUGradientCorrect()

void CheckPReLUGradientCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the PReLU activation function gradient test.

The function is implemented as PReLU layer in the file parametric_relu.hpp

Parameters

input	Input data used for evaluating the PReLU activation function.
target	Target data used to evaluate the PReLU gradient.

CheckReLU6Correct()

void CheckReLU6Correct	(	const arma::colvec	input,
		const arma::colvec	ActivationTarget,
		const arma::colvec	DerivativeTarget
	)

Implementation of the ReLU6 activation function derivative test.

The function is implemented as ReLU6 layer in the file relu6.hpp.

Parameters

input	Input data used for evaluating the ReLU6 activation function.
target	Target data used to evaluate the ReLU6 activation.

CheckSoftminActivationCorrect()

void CheckSoftminActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Softmin activation function test.

The function is implemented as Softmin layer in the file softmin.hpp.

Parameters

input	Input data used for evaluating the Softmin activation function.
target	Target data used to evaluate the Softmin activation.

CheckSoftminDerivativeCorrect()

void CheckSoftminDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Softmin activation function derivative test.

The function is implemented as Softmin layer in the file softmin.hpp.

Parameters

input	Input data used for evaluating the Softmin activation function.
target	Target data used to evaluate the Softmin activation.

CheckSoftShrinkActivationCorrect()

void CheckSoftShrinkActivationCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Soft Shrink activation function test.

The function is implemented as Soft Shrink layer in the file softshrink.hpp.

Parameters

input	Input data used for evaluating the Soft Shrink activation function.
target	Target data used to evaluate the Soft Shrink activation.

CheckSoftShrinkDerivativeCorrect()

void CheckSoftShrinkDerivativeCorrect	(	const arma::colvec	input,
		const arma::colvec	target
	)

Implementation of the Soft Shrink activation function derivative test.

The derivative function is implemented as Soft Shrink layer in the file softshrink.hpp

Parameters

input	Input data used for evaluating the Soft Shrink activation function.
target	Target data used to evaluate the Soft Shrink activation.