ml_expression_builder.hpp

Go to the documentation of this file.

//=======================================================================
// Copyright (c) 2014-2023 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "etl/impl/cce.hpp"
#include "etl/impl/bce.hpp"
#include "etl/impl/mse.hpp"

namespace etl::ml {

// Convolution wrappers

template <size_t S1 = 1, size_t S2 = 1, size_t P1 = 0, size_t P2 = 0, etl_expr A, etl_expr B>
conv_4d_valid_expr<detail::build_type<A>, detail::build_type<B>, S1, S2, P1, P2, true> convolution_forward(A&& a, B&& b) {
    return conv_4d_valid_expr<detail::build_type<A>, detail::build_type<B>, S1, S2, P1, P2, true>{std::forward<A>(a), std::forward<B>(b)};
}

template <etl_expr A, etl_expr B>
dyn_conv_4d_valid_expr<detail::build_type<A>, detail::build_type<B>, true> convolution_forward(
    A&& a, B&& b, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return dyn_conv_4d_valid_expr<detail::build_type<A>, detail::build_type<B>, true>{std::forward<A>(a), std::forward<B>(b), s1, s2, p1, p2};
}

template <size_t S1 = 1, size_t S2 = 1, size_t P1 = 0, size_t P2 = 0, etl_expr A, etl_expr B>
conv_4d_backward_expr<detail::build_type<A>, detail::build_type<B>, S1, S2, P1, P2, true> convolution_backward(A&& a, B&& b) {
    return conv_4d_backward_expr<detail::build_type<A>, detail::build_type<B>, S1, S2, P1, P2, true>{std::forward<A>(a), std::forward<B>(b)};
}

template <etl_expr A, etl_expr B>
dyn_conv_4d_backward_expr<detail::build_type<A>, detail::build_type<B>, true> convolution_backward(
    A&& a, B&& b, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return dyn_conv_4d_backward_expr<detail::build_type<A>, detail::build_type<B>, true>{std::forward<A>(a), std::forward<B>(b), s1, s2, p1, p2};
}

template <size_t S1 = 1, size_t S2 = 1, size_t P1 = 0, size_t P2 = 0, etl_expr A, etl_expr B>
conv_4d_backward_filter_expr<detail::build_type<A>, detail::build_type<B>, S1, S2, P1, P2, true> convolution_backward_filter(A&& a, B&& b) {
    return conv_4d_backward_filter_expr<detail::build_type<A>, detail::build_type<B>, S1, S2, P1, P2, true>{std::forward<A>(a), std::forward<B>(b)};
}

template <etl_expr A, etl_expr B>
dyn_conv_4d_backward_filter_expr<detail::build_type<A>, detail::build_type<B>, true> convolution_backward_filter(
    A&& a, B&& b, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return dyn_conv_4d_backward_filter_expr<detail::build_type<A>, detail::build_type<B>, true>{std::forward<A>(a), std::forward<B>(b), s1, s2, p1, p2};
}

// Pooling Wrappers

template <size_t C1, size_t C2, size_t S1 = C1, size_t S2 = C2, size_t P1 = 0, size_t P2 = 0, typename E>
auto max_pool_forward(E&& value) {
    return pool_2d_expr<detail::build_type<E>, C1, C2, S1, S2, P1, P2, impl::max_pool_2d>{value};
}

template <typename E>
auto max_pool_forward(E&& value, size_t c1, size_t c2) {
    return dyn_pool_2d_expr<detail::build_type<E>, impl::max_pool_2d>{value, c1, c2, c1, c2, 0, 0};
}

template <typename E>
auto max_pool_forward(E&& value, size_t c1, size_t c2, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return dyn_pool_2d_expr<detail::build_type<E>, impl::max_pool_2d>{value, c1, c2, s1, s2, p1, p2};
}

template <size_t C1, size_t C2, size_t C3, typename E>
auto max_pool_3d_forward(E&& value) {
    return pool_3d_expr<detail::build_type<E>, C1, C2, C3, C1, C2, C3, 0, 0, 0, impl::max_pool_3d>{value};
}

template <typename E>
auto max_pool_3d_forward(E&& value, size_t c1, size_t c2, size_t c3) {
    return dyn_pool_3d_expr<detail::build_type<E>, impl::max_pool_3d>{value, c1, c2, c3, c1, c2, c3, 0, 0, 0};
}

template <size_t C1, size_t C2, size_t S1 = C1, size_t S2 = C2, size_t P1 = 0, size_t P2 = 0, typename E>
auto avg_pool_forward(E&& value) {
    return pool_2d_expr<detail::build_type<E>, C1, C2, S1, S2, P1, P2, impl::avg_pool_2d>{value};
}

template <typename E>
auto avg_pool_forward(E&& value, size_t c1, size_t c2) {
    return dyn_pool_2d_expr<detail::build_type<E>, impl::avg_pool_2d>{value, c1, c2, c1, c2, 0, 0};
}

template <typename E>
auto avg_pool_forward(E&& value, size_t c1, size_t c2, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return dyn_pool_2d_expr<detail::build_type<E>, impl::avg_pool_2d>{value, c1, c2, s1, s2, p1, p2};
}

template <size_t C1, size_t C2, size_t C3, typename E>
auto avg_pool_3d_forward(E&& value) {
    return pool_3d_expr<detail::build_type<E>, C1, C2, C3, C1, C2, C3, 0, 0, 0, impl::avg_pool_3d>{value};
}

template <typename E>
auto avg_pool_3d_forward(E&& value, size_t c1, size_t c2, size_t c3) {
    return dyn_pool_3d_expr<detail::build_type<E>, impl::avg_pool_3d>{value, c1, c2, c3, c1, c2, c3, 0, 0, 0};
}

template <size_t C1, size_t C2, size_t S1, size_t S2, size_t P1, size_t P2, etl_expr A, etl_expr B, typename C>
pool_upsample_2d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, C1, C2, S1, S2, P1, P2, true> max_pool_backward(A&& input, B&& output, C&& errors) {
    return {input, output, errors};
}

template <etl_expr A, etl_expr B, typename C>
dyn_pool_upsample_2d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, true> max_pool_backward(
        A&& input, B&& output, C&& errors, size_t c1, size_t c2) {
    return {input, output, errors, c1, c2, c1, c2, 0, 0};
}

template <etl_expr A, etl_expr B, typename C>
dyn_pool_upsample_2d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, true> max_pool_backward(
        A&& input, B&& output, C&& errors, size_t c1, size_t c2, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return {input, output, errors, c1, c2, s1, s2, p1, p2};
}

template <size_t C1, size_t C2, size_t C3, etl_expr A, etl_expr B, typename C>
pool_upsample_3d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, C1, C2, C3, true> max_pool_3d_backward(A&& input,
                                                                                                                                  B&& output,
                                                                                                                                  C&& errors) {
    return {input, output, errors};
}

template <etl_expr A, etl_expr B, typename C>
dyn_pool_upsample_3d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, true> max_pool_3d_backward(
    A&& input, B&& output, C&& errors, size_t c1, size_t c2, size_t c3) {
    return {input, output, errors, c1, c2, c3};
}

template <size_t C1, size_t C2, size_t S1, size_t S2, size_t P1, size_t P2, etl_expr A, etl_expr B, typename C>
pool_upsample_2d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, C1, C2, S1, S2, P1, P2, false> avg_pool_backward(A&& input, B&& output, C&& errors) {
    return {input, output, errors};
}

template <etl_expr A, etl_expr B, typename C>
dyn_pool_upsample_2d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, false> avg_pool_backward(
    A&& input, B&& output, C&& errors, size_t c1, size_t c2) {
    return {input, output, errors, c1, c2, c1, c2, 0, 0};
}

template <etl_expr A, etl_expr B, typename C>
dyn_pool_upsample_2d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, false> avg_pool_backward(
    A&& input, B&& output, C&& errors, size_t c1, size_t c2, size_t s1, size_t s2, size_t p1 = 0, size_t p2 = 0) {
    return {input, output, errors, c1, c2, s1, s2, p1, p2};
}

template <size_t C1, size_t C2, size_t C3, etl_expr A, etl_expr B, typename C>
pool_upsample_3d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, C1, C2, C3, false> avg_pool_3d_backward(A&& input,
                                                                                                                                   B&& output,
                                                                                                                                   C&& errors) {
    return {input, output, errors};
}

template <etl_expr A, etl_expr B, typename C>
dyn_pool_upsample_3d_expr<detail::build_type<A>, detail::build_type<B>, detail::build_type<C>, false> avg_pool_3d_backward(
    A&& input, B&& output, C&& errors, size_t c1, size_t c2, size_t c3) {
    return {input, output, errors, c1, c2, c3};
}

// Derivatives with respect to output

template <typename E>
auto identity_derivative_out([[maybe_unused]] E&& value) {
    return 1.0;
}

template <etl_expr E>
auto sigmoid_derivative_out(E&& value) -> decltype(value >> (1.0 - value)) {
    return value >> (1.0 - value);
}

template <typename E>
auto softmax_derivative_out([[maybe_unused]] E&& e) {
    return 1.0;
}

template <etl_expr E>
auto tanh_derivative_out(E&& value) -> decltype(1.0 - (value >> value)) {
    return 1.0 - (value >> value);
}

template <etl_expr E>
auto relu_derivative_out(const E& value) -> detail::unary_helper<E, relu_derivative_op> {
    return detail::unary_helper<E, relu_derivative_op>{value};
}

// Fully-Fledged backward activation

template <typename O, typename E>
decltype(auto) identity_backward([[maybe_unused]] O&& output, E&& errors) {
    return std::forward<E>(errors);
}

template <typename O, typename E>
auto sigmoid_backward(O&& output, E&& errors) -> detail::left_binary_helper<O, E, sigmoid_derivative_binary_op> {
    return {output, errors};
}

template <typename O, typename E>
auto relu_backward(O&& output, E&& errors) -> detail::left_binary_helper<O, E, relu_derivative_binary_op> {
    return {output, errors};
}

template <typename O, typename E>
decltype(auto) softmax_backward([[maybe_unused]] O&& output, E&& errors) {
    return std::forward<E>(errors);
}

template <typename O, etl_expr E>
auto tanh_backward(O&& output, E&& errors) {
    return (1.0 - (output >> output)) >> errors;
}

template <etl_expr O, etl_expr L>
value_t<O> cce_loss(O&& output, L&& labels, value_t<O> scale) {
    return detail::cce_loss_impl::apply(std::forward<O>(output), std::forward<L>(labels), scale);
}

template <etl_expr O, etl_expr L>
value_t<O> cce_error(O&& output, L&& labels, value_t<O> scale) {
    return detail::cce_error_impl::apply(std::forward<O>(output), std::forward<L>(labels), scale);
}

template <etl_expr O, etl_expr L>
value_t<O> bce_loss(O&& output, L&& labels, value_t<O> scale) {
    return detail::bce_loss_impl::apply(std::forward<O>(output), std::forward<L>(labels), scale);
}

template <etl_expr O, etl_expr L>
value_t<O> bce_error(O&& output, L&& labels, value_t<O> scale) {
    return detail::bce_error_impl::apply(std::forward<O>(output), std::forward<L>(labels), scale);
}

template <etl_expr O, etl_expr L>
std::pair<value_t<O>, value_t<O>> bce(O&& output, L&& labels, value_t<O> alpha, value_t<O> beta) {
    return detail::bce_impl::apply(std::forward<O>(output), std::forward<L>(labels), alpha, beta);
}

template <etl_expr O, etl_expr L>
std::pair<value_t<O>, value_t<O>> cce(O&& output, L&& labels, value_t<O> alpha, value_t<O> beta) {
    return detail::cce_impl::apply(std::forward<O>(output), std::forward<L>(labels), alpha, beta);
}

template <etl_expr O, etl_expr L>
value_t<O> mse_loss(O&& output, L&& labels, value_t<O> scale) {
    return detail::mse_loss_impl::apply(std::forward<O>(output), std::forward<L>(labels), scale);
}

template <etl_expr O, etl_expr L>
value_t<O> mse_error(O&& output, L&& labels, value_t<O> scale) {
    return detail::mse_error_impl::apply(std::forward<O>(output), std::forward<L>(labels), scale);
}

template <etl_expr O, etl_expr L>
std::pair<value_t<O>, value_t<O>> mse(O&& output, L&& labels, value_t<O> alpha, value_t<O> beta) {
    return detail::mse_impl::apply(std::forward<O>(output), std::forward<L>(labels), alpha, beta);
}

} //end of namespace etl::ml