block_discontinuity.hpp

// Copyright (c) 2017-2022 Advanced Micro Devices, Inc. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

#ifndef ROCPRIM_BLOCK_BLOCK_DISCONTINUITY_HPP_
#define ROCPRIM_BLOCK_BLOCK_DISCONTINUITY_HPP_


#include "detail/block_adjacent_difference_impl.hpp"

#include "../config.hpp"
#include "../detail/various.hpp"




BEGIN_ROCPRIM_NAMESPACE

template<
    class T,
    unsigned int BlockSizeX,
    unsigned int BlockSizeY = 1,
    unsigned int BlockSizeZ = 1
>
class block_discontinuity
#ifndef DOXYGEN_SHOULD_SKIP_THIS // hide implementation detail from documentation
    : private detail::block_adjacent_difference_impl<T, BlockSizeX, BlockSizeY, BlockSizeZ>
#endif // DOXYGEN_SHOULD_SKIP_THIS
{
    using base_type = detail::block_adjacent_difference_impl<T, BlockSizeX, BlockSizeY, BlockSizeZ>;

    static constexpr unsigned BlockSize = base_type::BlockSize;
    // Struct used for creating a raw_storage object for this primitive's temporary storage.
    struct storage_type_
    {
        typename base_type::storage_type left;
        typename base_type::storage_type right;
    };

public:

    #ifndef DOXYGEN_SHOULD_SKIP_THIS // hides storage_type implementation for Doxygen
    using storage_type = detail::raw_storage<storage_type_>;
    #else
    using storage_type = storage_type_;
    #endif

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_heads(Flag (&head_flags)[ItemsPerThread],
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op,
                    storage_type& storage)
    {
        static constexpr auto as_flags         = true;
        static constexpr auto reversed         = false;
        static constexpr auto with_predecessor = false;
        base_type::template apply_left<as_flags, reversed, with_predecessor>(
            input, head_flags, flag_op, input[0] /* predecessor */, storage.get().left);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_heads(Flag (&head_flags)[ItemsPerThread],
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_heads(head_flags, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_heads(Flag (&head_flags)[ItemsPerThread],
                    T tile_predecessor_item,
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op,
                    storage_type& storage)
    {
        static constexpr auto as_flags         = true;
        static constexpr auto reversed         = false;
        static constexpr auto with_predecessor = true;
        base_type::template apply_left<as_flags, reversed, with_predecessor>(
            input, head_flags, flag_op, tile_predecessor_item, storage.get().left);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_heads(Flag (&head_flags)[ItemsPerThread],
                    T tile_predecessor_item,
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_heads(head_flags, tile_predecessor_item, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_tails(Flag (&tail_flags)[ItemsPerThread],
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op,
                    storage_type& storage)
    {
        static constexpr auto as_flags       = true;
        static constexpr auto reversed       = false;
        static constexpr auto with_successor = false;
        base_type::template apply_right<as_flags, reversed, with_successor>(
            input, tail_flags, flag_op, input[0] /* successor */, storage.get().right);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_tails(Flag (&tail_flags)[ItemsPerThread],
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_tails(tail_flags, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_tails(Flag (&tail_flags)[ItemsPerThread],
                    T tile_successor_item,
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op,
                    storage_type& storage)
    {
        static constexpr auto as_flags       = true;
        static constexpr auto reversed       = false;
        static constexpr auto with_successor = true;
        base_type::template apply_right<as_flags, reversed, with_successor>(
            input, tail_flags, flag_op, tile_successor_item, storage.get().right);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_tails(Flag (&tail_flags)[ItemsPerThread],
                    T tile_successor_item,
                    const T (&input)[ItemsPerThread],
                    FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_tails(tail_flags, tile_successor_item, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              Flag (&tail_flags)[ItemsPerThread],
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op,
                              storage_type& storage)
    {
        static constexpr auto as_flags         = true;
        static constexpr auto reversed         = false;
        static constexpr auto with_predecessor = false;
        static constexpr auto with_successor   = false;

        // Copy items in case head_flags is aliased with input
        T items[ItemsPerThread];

        ROCPRIM_UNROLL
        for(unsigned int i = 0; i < ItemsPerThread; ++i) {
            items[i] = input[i];
        }

        base_type::template apply_left<as_flags, reversed, with_predecessor>(
            items, head_flags, flag_op, items[0] /*predecessor*/, storage.get().left);

        base_type::template apply_right<as_flags, reversed, with_successor>(
            items, tail_flags, flag_op, items[0] /*successor*/, storage.get().right);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              Flag (&tail_flags)[ItemsPerThread],
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_heads_and_tails(head_flags, tail_flags, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              Flag (&tail_flags)[ItemsPerThread],
                              T tile_successor_item,
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op,
                              storage_type& storage)
    {
        static constexpr auto as_flags         = true;
        static constexpr auto reversed         = false;
        static constexpr auto with_predecessor = false;
        static constexpr auto with_successor   = true;

        // Copy items in case head_flags is aliased with input
        T items[ItemsPerThread];

        ROCPRIM_UNROLL
        for(unsigned int i = 0; i < ItemsPerThread; ++i) {
            items[i] = input[i];
        }

        base_type::template apply_left<as_flags, reversed, with_predecessor>(
            items, head_flags, flag_op, items[0] /*predecessor*/, storage.get().left);

        base_type::template apply_right<as_flags, reversed, with_successor>(
            items, tail_flags, flag_op, tile_successor_item, storage.get().right);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              Flag (&tail_flags)[ItemsPerThread],
                              T tile_successor_item,
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_heads_and_tails(head_flags, tail_flags, tile_successor_item, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              T tile_predecessor_item,
                              Flag (&tail_flags)[ItemsPerThread],
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op,
                              storage_type& storage)
    {
        static constexpr auto as_flags         = true;
        static constexpr auto reversed         = false;
        static constexpr auto with_predecessor = true;
        static constexpr auto with_successor   = false;

        // Copy items in case head_flags is aliased with input
        T items[ItemsPerThread];

        ROCPRIM_UNROLL
        for(unsigned int i = 0; i < ItemsPerThread; ++i) {
            items[i] = input[i];
        }

        base_type::template apply_left<as_flags, reversed, with_predecessor>(
            items, head_flags, flag_op, tile_predecessor_item, storage.get().left);

        base_type::template apply_right<as_flags, reversed, with_successor>(
            items, tail_flags, flag_op, items[0] /*successor*/, storage.get().right);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              T tile_predecessor_item,
                              Flag (&tail_flags)[ItemsPerThread],
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_heads_and_tails(head_flags, tile_predecessor_item, tail_flags, input, flag_op, storage);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              T tile_predecessor_item,
                              Flag (&tail_flags)[ItemsPerThread],
                              T tile_successor_item,
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op,
                              storage_type& storage)
    {
        static constexpr auto as_flags         = true;
        static constexpr auto reversed         = false;
        static constexpr auto with_predecessor = true;
        static constexpr auto with_successor   = true;

        // Copy items in case head_flags is aliased with input
        T items[ItemsPerThread];

        ROCPRIM_UNROLL
        for(unsigned int i = 0; i < ItemsPerThread; ++i) {
            items[i] = input[i];
        }

        base_type::template apply_left<as_flags, reversed, with_predecessor>(
            items, head_flags, flag_op, tile_predecessor_item, storage.get().left);

        base_type::template apply_right<as_flags, reversed, with_successor>(
            items, tail_flags, flag_op, tile_successor_item, storage.get().right);
    }

    template<unsigned int ItemsPerThread, class Flag, class FlagOp>
    ROCPRIM_DEVICE ROCPRIM_FORCE_INLINE
    void flag_heads_and_tails(Flag (&head_flags)[ItemsPerThread],
                              T tile_predecessor_item,
                              Flag (&tail_flags)[ItemsPerThread],
                              T tile_successor_item,
                              const T (&input)[ItemsPerThread],
                              FlagOp flag_op)
    {
        ROCPRIM_SHARED_MEMORY storage_type storage;
        flag_heads_and_tails(
            head_flags, tile_predecessor_item, tail_flags, tile_successor_item,
            input, flag_op, storage
        );
    }
};

END_ROCPRIM_NAMESPACE

// end of group blockmodule

#endif // ROCPRIM_BLOCK_BLOCK_DISCONTINUITY_HPP_