bias_batch_var_4d_expr.hpp

//=======================================================================
// Copyright (c) 2014-2023 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "etl/expr/base_temporary_expr.hpp"

#include "etl/impl/egblas/bias_batch_sum.hpp"

namespace etl {

template <etl_4d A, etl_1d B>
struct bias_batch_var_4d_expr : base_temporary_expr_bin<bias_batch_var_4d_expr<A, B>, A, B> {
    using value_type = value_t<A>;                               
    using this_type  = bias_batch_var_4d_expr<A, B>;             
    using base_type  = base_temporary_expr_bin<this_type, A, B>; 
    using sub_traits = decay_traits<A>;                          

    static constexpr auto storage_order = sub_traits::storage_order; 

    static constexpr bool gpu_computable =
               (impl::egblas::has_sbias_batch_var4 && all_row_major<A> && all_single_precision<A>)
            || (impl::egblas::has_dbias_batch_var4 && all_row_major<A> && all_double_precision<A>);

    explicit bias_batch_var_4d_expr(A a, B b) : base_type(a, b) {
        //Nothing else to init
    }

    template <etl_1d C>
    static void check([[maybe_unused]] const A& a, [[maybe_unused]] const B& b, [[maybe_unused]] const C& c) {
        if constexpr (all_fast<A, B, C>) {
            static_assert(etl::dim<1, A>() == etl::dim<0, C>(), "Invalid dimensions for bias_batch_var_4d");
            static_assert(etl::dim<0, B>() == etl::dim<0, C>(), "Invalid dimensions for bias_batch_var_4d");
        } else {
            cpp_assert(etl::dim<1>(a) == etl::dim<0>(c), "Invalid dimensions for bias_batch_var_4d");
            cpp_assert(etl::dim<0>(b) == etl::dim<0>(c), "Invalid dimensions for bias_batch_var_4d");
        }
    }

    // Assignment functions

    template <etl_expr L>
    void assign_to(L&& lhs) const {
        inc_counter("temp:assign");

        auto& a = this->a();
        auto& b = this->b();

        check(a, b, lhs);

        using T = value_t<A>;

        const auto N = etl::dim<0>(a);
        const auto K = etl::dim<1>(a);
        const auto F = static_cast<T>(etl::size(a) / etl::size(lhs));

        if constexpr (impl::egblas::has_sbias_batch_var4 && all_row_major<A> && all_floating<A, L>) {
            const auto W = etl::dim<2>(a);
            const auto H = etl::dim<3>(a);

            decltype(auto) t1 = smart_forward_gpu(a);
            decltype(auto) t2 = smart_forward_gpu(b);

            t1.ensure_gpu_up_to_date();
            t2.ensure_gpu_up_to_date();

            lhs.ensure_gpu_allocated();

            impl::egblas::bias_batch_var4(N, K, W, H, t1.gpu_memory(), t2.gpu_memory(), lhs.gpu_memory());

            lhs.validate_gpu();
            lhs.invalidate_cpu();
        } else {
            standard_evaluator::pre_assign_rhs(a);
            standard_evaluator::pre_assign_rhs(b);

            a.ensure_cpu_up_to_date();
            b.ensure_cpu_up_to_date();

            // Note: We use etl::sum directly instead of doing the sum manually
            // That way, we will access the already vectorized sum
            // Now, this means that evaluator decisions will be called several 
            // times. This could be an issue that could be looked at in the future

            auto batch_fun_k = [&](const size_t first, const size_t last) {
                CPU_SECTION {
                    for (size_t k = first; k < last; ++k) {
                        lhs(k) = 0;
                    }

                    for (size_t bb = 0; bb < N; ++bb) {
                        for (size_t k = first; k < last; ++k) {
                            lhs(k) += sum((a(bb)(k) - b(k)) >> (a(bb)(k) - b(k)));
                        }
                    }

                    for (size_t k = first; k < last; ++k) {
                        lhs(k) /= F;
                    }
                }
            };

            engine_dispatch_1d_serial(batch_fun_k, 0, K, 2UL);

            lhs.validate_cpu();
            lhs.invalidate_gpu();
        }
    }

    template <etl_expr L>
    void assign_add_to(L&& lhs) const {
        auto& a = this->a();
        auto& b = this->b();

        standard_evaluator::pre_assign_rhs(a);
        standard_evaluator::pre_assign_rhs(b);

        using T = value_t<A>;

        const auto N = etl::dim<0>(a);
        const auto K = etl::dim<1>(a);
        const auto F = static_cast<T>(etl::size(a) / etl::size(lhs));

        check(a, b, lhs);

        a.ensure_cpu_up_to_date();
        b.ensure_cpu_up_to_date();
        lhs.ensure_cpu_up_to_date();

        auto batch_fun_k = [&](const size_t first, const size_t last) {
            CPU_SECTION {
                for (size_t k = first; k < last; ++k) {
                    T var = 0;

                    for (size_t bb = 0; bb < N; ++bb) {
                        var += sum((a(bb)(k) - b(k)) >> (a(bb)(k) - b(k)));
                    }

                    lhs(k) += var / F;
                }
            }
        };

        engine_dispatch_1d_serial(batch_fun_k, 0, K, 2UL);

        lhs.validate_cpu();
        lhs.invalidate_gpu();
    }

    template <etl_expr L>
    void assign_sub_to(L&& lhs) const {
        auto& a = this->a();
        auto& b = this->b();

        standard_evaluator::pre_assign_rhs(a);
        standard_evaluator::pre_assign_rhs(b);

        using T = value_t<A>;

        const auto N = etl::dim<0>(a);
        const auto K = etl::dim<1>(a);
        const auto F = static_cast<T>(etl::size(a) / etl::size(lhs));

        check(a, b, lhs);

        a.ensure_cpu_up_to_date();
        b.ensure_cpu_up_to_date();
        lhs.ensure_cpu_up_to_date();

        auto batch_fun_k = [&](const size_t first, const size_t last) {
            CPU_SECTION {
                for (size_t k = first; k < last; ++k) {
                    T var = 0;

                    for (size_t bb = 0; bb < N; ++bb) {
                        var += sum((a(bb)(k) - b(k)) >> (a(bb)(k) - b(k)));
                    }

                    lhs(k) -= var / F;
                }
            }
        };

        engine_dispatch_1d_serial(batch_fun_k, 0, K, 2UL);

        lhs.validate_cpu();
        lhs.invalidate_gpu();
    }

    template <etl_expr L>
    void assign_mul_to(L&& lhs) const {
        auto& a = this->a();
        auto& b = this->b();

        standard_evaluator::pre_assign_rhs(a);
        standard_evaluator::pre_assign_rhs(b);

        using T = value_t<A>;

        const auto N = etl::dim<0>(a);
        const auto K = etl::dim<1>(a);
        const auto F = static_cast<T>(etl::size(a) / etl::size(lhs));

        check(a, b, lhs);

        a.ensure_cpu_up_to_date();
        b.ensure_cpu_up_to_date();
        lhs.ensure_cpu_up_to_date();

        auto batch_fun_k = [&](const size_t first, const size_t last) {
            CPU_SECTION {
                for (size_t k = first; k < last; ++k) {
                    T var = 0;

                    for (size_t bb = 0; bb < N; ++bb) {
                        var += sum((a(bb)(k) - b(k)) >> (a(bb)(k) - b(k)));
                    }

                    lhs(k) *= var / F;
                }
            }
        };

        engine_dispatch_1d_serial(batch_fun_k, 0, K, 2UL);

        lhs.validate_cpu();
        lhs.invalidate_gpu();
    }

    template <etl_expr L>
    void assign_div_to(L&& lhs) const {
        auto& a = this->a();
        auto& b = this->b();

        standard_evaluator::pre_assign_rhs(a);
        standard_evaluator::pre_assign_rhs(b);

        using T = value_t<A>;

        const auto N = etl::dim<0>(a);
        const auto K = etl::dim<1>(a);
        const auto F = static_cast<T>(etl::size(a) / etl::size(lhs));

        check(a, b, lhs);

        a.ensure_cpu_up_to_date();
        b.ensure_cpu_up_to_date();
        lhs.ensure_cpu_up_to_date();

        auto batch_fun_k = [&](const size_t first, const size_t last) {
            CPU_SECTION {
                for (size_t k = first; k < last; ++k) {
                    T var = 0;

                    for (size_t bb = 0; bb < N; ++bb) {
                        var += sum((a(bb)(k) - b(k)) >> (a(bb)(k) - b(k)));
                    }

                    lhs(k) /= var / F;
                }
            }
        };

        engine_dispatch_1d_serial(batch_fun_k, 0, K, 2UL);

        lhs.validate_cpu();
        lhs.invalidate_gpu();
    }

    template <etl_expr L>
    void assign_mod_to(L&& lhs) const {
        auto& a = this->a();
        auto& b = this->b();

        standard_evaluator::pre_assign_rhs(a);
        standard_evaluator::pre_assign_rhs(b);

        using T = value_t<A>;

        const auto N = etl::dim<0>(a);
        const auto K = etl::dim<1>(a);
        const auto F = static_cast<T>(etl::size(a) / etl::size(lhs));

        check(a, b, lhs);

        a.ensure_cpu_up_to_date();
        b.ensure_cpu_up_to_date();
        lhs.ensure_cpu_up_to_date();

        auto batch_fun_k = [&](const size_t first, const size_t last) {
            CPU_SECTION {
                for (size_t k = first; k < last; ++k) {
                    T var = 0;

                    for (size_t bb = 0; bb < N; ++bb) {
                        var += sum((a(bb)(k) - b(k)) >> (a(bb)(k) - b(k)));
                    }

                    lhs(k) %= var / F;
                }
            }
        };

        engine_dispatch_1d_serial(batch_fun_k, 0, K, 2UL);

        lhs.validate_cpu();
        lhs.invalidate_gpu();
    }

    friend std::ostream& operator<<(std::ostream& os, const bias_batch_var_4d_expr& expr) {
        return os << "bias_batch_var_4d(" << expr._a << ")";
    }
};

template <typename A, typename B>
struct etl_traits<etl::bias_batch_var_4d_expr<A, B>> {
    using expr_t     = etl::bias_batch_var_4d_expr<A, B>; 
    using sub_expr_t = std::decay_t<A>;                   
    using sub_traits = etl_traits<sub_expr_t>;            
    using value_type = value_t<A>;                        

    static constexpr bool is_etl         = true;                                 
    static constexpr bool is_transformer = false;                                
    static constexpr bool is_view        = false;                                
    static constexpr bool is_magic_view  = false;                                
    static constexpr bool is_fast        = sub_traits::is_fast;                  
    static constexpr bool is_linear      = false;                                
    static constexpr bool is_thread_safe = true;                                 
    static constexpr bool is_value       = false;                                
    static constexpr bool is_direct      = true;                                 
    static constexpr bool is_generator   = false;                                
    static constexpr bool is_padded      = false;                                
    static constexpr bool is_aligned     = true;                                 
    static constexpr bool is_temporary   = true;                                 
    static constexpr order storage_order = sub_traits::storage_order;            
    static constexpr bool gpu_computable = is_gpu_t<value_type> && cuda_enabled; 

    template <vector_mode_t V>
    static constexpr bool vectorizable = true;

    template <size_t DD>
    static constexpr size_t dim() requires(DD == 0) {
        return decay_traits<A>::template dim<1>();
    }

    static size_t dim(const expr_t& e, [[maybe_unused]] size_t d) {
        cpp_assert(d == 0, "Invalid dimensions access");
        return etl::dim<1>(e._a);
    }

    static size_t size(const expr_t& e) {
        return etl::dim<1>(e._a);
    }

    static constexpr size_t size() {
        return decay_traits<A>::template dim<1>();
    }

    static constexpr size_t dimensions() {
        return 1;
    }

    static constexpr int complexity() noexcept {
        return -1;
    }
};

template <etl_4d A, etl_1d B>
bias_batch_var_4d_expr<detail::build_type<A>, detail::build_type<B>> bias_batch_var_4d(const A& a, const B& b) {
    return bias_batch_var_4d_expr<detail::build_type<A>, detail::build_type<B>>{a, b};
}

} //end of namespace etl