gevm_expr.hpp

//=======================================================================
// Copyright (c) 2014-2023 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

#include "etl/expr/base_temporary_expr.hpp"

//The implementations
#include "etl/impl/std/gemm.hpp"
#include "etl/impl/blas/gemm.hpp"
#include "etl/impl/vec/gevm.hpp"
#include "etl/impl/vec/gemm_conv.hpp"
#include "etl/impl/cublas/gemm.hpp"

namespace etl {

template <etl_expr A, etl_expr B>
struct gevm_expr : base_temporary_expr_bin<gevm_expr<A, B>, A, B> {
    using value_type  = value_t<A>;                               
    using this_type   = gevm_expr<A, B>;                          
    using base_type   = base_temporary_expr_bin<this_type, A, B>; 
    using left_traits = decay_traits<A>;                          

    static constexpr auto storage_order = left_traits::storage_order; 

    static constexpr bool gpu_computable = cublas_enabled && all_homogeneous<A, B>;

    explicit gevm_expr(A a, B b) : base_type(a, b) {
        //Nothing else to init
    }

    template <etl_expr C>
    static void check([[maybe_unused]] const A& a, [[maybe_unused]] const B& b, [[maybe_unused]] const C& c) {
        if constexpr (all_fast<A, B, C>) {
            static_assert(dim<0, A>() == dim<0, B>()         //exterior dimension 1
                              && dim<1, B>() == dim<0, C>(), //exterior dimension 2
                          "Invalid sizes for multiplication");
        } else {
            cpp_assert(dim<0>(a) == dim<0>(b)         //exterior dimension 1
                           && dim<1>(b) == dim<0>(c), //exterior dimension 2
                       "Invalid sizes for multiplication");
        }
    }

    // Assignment functions

    template <etl_expr C>
    static constexpr gemm_impl select_default_gevm_impl(bool no_gpu) {
        constexpr bool vec_possible = vectorize_impl && all_vectorizable_t<vector_mode, A, B, C> && vec_enabled;
        constexpr bool homo         = all_homogeneous<A, B, C>;

        if (cublas_enabled && homo && !no_gpu) {
            return gemm_impl::CUBLAS;
        }

        if (vec_possible && homo) {
            return gemm_impl::VEC;
        }

        if (cblas_enabled && homo) {
            return gemm_impl::BLAS;
        }

        return gemm_impl::STD;
    }

#ifdef ETL_MANUAL_SELECT

    template <etl_expr C>
    static inline gemm_impl select_gevm_impl() {
        if (local_context().gemm_selector.forced) {
            auto forced = local_context().gemm_selector.impl;

            switch (forced) {
                //CUBLAS cannot always be used
                case gemm_impl::CUBLAS:
                    if (!cublas_enabled || !all_homogeneous<A, B, C> || local_context().cpu) { //COVERAGE_EXCLUDE_LINE
                        std::cerr << "Forced selection to CUBLAS gevm implementation, but not possible for this expression"
                                  << std::endl;                                  //COVERAGE_EXCLUDE_LINE
                        return select_default_gevm_impl<C>(local_context().cpu); //COVERAGE_EXCLUDE_LINE
                    }                                                            //COVERAGE_EXCLUDE_LINE

                    return forced;

                //BLAS cannot always be used
                case gemm_impl::BLAS:
                    if (!cblas_enabled || !all_homogeneous<A, B, C>) {                                                                  //COVERAGE_EXCLUDE_LINE
                        std::cerr << "Forced selection to BLAS gevm implementation, but not possible for this expression" << std::endl; //COVERAGE_EXCLUDE_LINE
                        return select_default_gevm_impl<C>(local_context().cpu);                                                        //COVERAGE_EXCLUDE_LINE
                    }                                                                                                                   //COVERAGE_EXCLUDE_LINE

                    return forced;

                //VEC cannot always be used
                case gemm_impl::VEC:
                    if (!vec_enabled || !vectorize_impl || !all_vectorizable<vector_mode, A, B, C> || !all_homogeneous<A, B, C>) {                        //COVERAGE_EXCLUDE_LINE
                        std::cerr << "Forced selection to VEC gevm implementation, but not possible for this expression" << std::endl; //COVERAGE_EXCLUDE_LINE
                        return select_default_gevm_impl<C>(local_context().cpu);                                                       //COVERAGE_EXCLUDE_LINE
                    }                                                                                                                  //COVERAGE_EXCLUDE_LINE

                    return forced;

                //In other cases, simply use the forced impl
                default:
                    return forced;
            }
        }

        return select_default_gevm_impl<C>(local_context().cpu);
    }

#else

    template <etl_expr C>
    static constexpr gemm_impl select_gevm_impl() {
        return select_default_gevm_impl<C>(false);
    }

#endif

    template <etl_expr AA, etl_expr BB, etl_expr C>
    static void apply_raw(AA&& a, BB&& b, C&& c) {
        constexpr_select auto impl = select_gevm_impl<C>();

        // clang-format off
        if constexpr (is_transpose_expr<BB>) {
            if constexpr_select (impl == gemm_impl::STD) {
                inc_counter("impl:std");
                etl::impl::standard::vm_mul(smart_forward(a), smart_forward(b), c);
            } else if constexpr_select (impl == gemm_impl::BLAS) {
                inc_counter("impl:blas");
                etl::impl::blas::gevm_t(smart_forward(a), smart_forward(b.a()), c);
            } else if constexpr_select (impl == gemm_impl::VEC) {
                inc_counter("impl:vec");
                etl::impl::vec::gevm_t(smart_forward(a), smart_forward(b.a()), c);
            } else if constexpr_select (impl == gemm_impl::CUBLAS) {
                inc_counter("impl:cublas");
                etl::impl::cublas::gevm_t(smart_forward_gpu(a), smart_forward_gpu(b.a()), c);
            } else {
                cpp_unreachable("Invalid selection for gevm");
            }
        } else {
            if constexpr_select (impl == gemm_impl::STD) {
                inc_counter("impl:std");
                etl::impl::standard::vm_mul(smart_forward(a), smart_forward(b), c);
            } else if constexpr_select (impl == gemm_impl::BLAS) {
                inc_counter("impl:blas");
                etl::impl::blas::gevm(smart_forward(a), smart_forward(b), c);
            } else if constexpr_select (impl == gemm_impl::VEC) {
                inc_counter("impl:vec");
                etl::impl::vec::gevm(smart_forward(a), smart_forward(b), c);
            } else if constexpr_select (impl == gemm_impl::CUBLAS) {
                inc_counter("impl:cublas");
                etl::impl::cublas::gevm(smart_forward_gpu(a), smart_forward_gpu(b), c);
            } else {
                cpp_unreachable("Invalid selection for gevm");
            }
        }
        // clang-format on
    }

    template <etl_expr C>
    void assign_to(C&& c) const {
        inc_counter("temp:assign");

        check(this->a(), this->b(), c);

        apply_raw(this->a(), this->b(), c);
    }

    template <etl_expr L>
    void assign_add_to(L&& lhs) const {
        std_add_evaluate(*this, lhs);
    }

    template <etl_expr L>
    void assign_sub_to(L&& lhs) const {
        std_sub_evaluate(*this, lhs);
    }

    template <etl_expr L>
    void assign_mul_to(L&& lhs) const {
        std_mul_evaluate(*this, lhs);
    }

    template <etl_expr L>
    void assign_div_to(L&& lhs) const {
        std_div_evaluate(*this, lhs);
    }

    template <etl_expr L>
    void assign_mod_to(L&& lhs) const {
        std_mod_evaluate(*this, lhs);
    }

    friend std::ostream& operator<<(std::ostream& os, const gevm_expr& expr) {
        return os << expr._a << " * " << expr._b;
    }
};

template <etl_expr A, etl_expr B>
struct etl_traits<etl::gevm_expr<A, B>> {
    using expr_t       = etl::gevm_expr<A, B>;     
    using left_expr_t  = std::decay_t<A>;          
    using right_expr_t = std::decay_t<B>;          
    using left_traits  = etl_traits<left_expr_t>;  
    using right_traits = etl_traits<right_expr_t>; 
    using value_type   = value_t<A>;               

    static constexpr bool is_etl         = true;                                          
    static constexpr bool is_transformer = false;                                         
    static constexpr bool is_view        = false;                                         
    static constexpr bool is_magic_view  = false;                                         
    static constexpr bool is_fast        = left_traits::is_fast && right_traits::is_fast; 
    static constexpr bool is_linear      = false;                                         
    static constexpr bool is_thread_safe = true;                                          
    static constexpr bool is_value       = false;                                         
    static constexpr bool is_direct      = true;                                          
    static constexpr bool is_generator   = false;                                         
    static constexpr bool is_padded      = false;                                         
    static constexpr bool is_aligned     = true;                                          
    static constexpr bool is_temporary   = true;                                          
    static constexpr bool gpu_computable = is_gpu_t<value_type> && cublas_enabled;        
    static constexpr order storage_order = left_traits::storage_order;                    

    template <vector_mode_t V>
    static constexpr bool vectorizable = true;

    template <size_t DD>
    static constexpr size_t dim() {
        return decay_traits<B>::template dim<1>();
    }

    static size_t dim(const expr_t& e, [[maybe_unused]] size_t d) {
        return etl::dim(e._b, 1);
    }

    static size_t size(const expr_t& e) {
        return etl::dim(e._b, 1);
    }

    static constexpr size_t size() {
        return decay_traits<B>::template dim<1>();
    }

    static constexpr size_t dimensions() {
        return 1;
    }

    static constexpr int complexity() noexcept {
        return -1;
    }
};

template <etl_1d A, etl_2d B>
auto operator*(A&& a, B&& b) {
    return gevm_expr<detail::build_type<A>, detail::build_type<B>>{a, b};
}

template <etl_1d A, etl_2d B>
auto mul(A&& a, B&& b) {
    return gevm_expr<detail::build_type<A>, detail::build_type<B>>{a, b};
}

template <etl_1d A, etl_2d B, etl_1d C>
auto mul(A&& a, B&& b, C&& c) {
    c = mul(a, b);
    return c;
}

} //end of namespace etl