launch_configuration.hpp

Go to the documentation of this file.

#pragma once
#ifndef CUDA_API_WRAPPERS_LAUNCH_CONFIGURATION_CUH_
#define CUDA_API_WRAPPERS_LAUNCH_CONFIGURATION_CUH_

#include "constants.hpp"
#include "types.hpp"

#include <type_traits>
#include <utility>

namespace cuda {

class device_t;
class event_t;
class kernel_t;

namespace detail_ {

inline void validate_block_dimensions(grid::block_dimensions_t block_dims)
{
    if (block_dims.volume() == 0) {
        throw ::std::invalid_argument("Zero-volume grid-of-blocks dimensions provided");
    }
}

inline void validate_grid_dimensions(grid::dimensions_t grid_dims)
{
    if (grid_dims.volume() == 0) {
        throw ::std::invalid_argument("Zero-volume block dimensions provided");
    }
}

// Note: The reason for the verbose name is the identity of the block and grid dimension types
void validate_block_dimension_compatibility(const device_t &device, grid::block_dimensions_t block_dims);
void validate_block_dimension_compatibility(const kernel_t &kernel, grid::block_dimensions_t block_dims);

void validate_compatibility(const kernel_t &kernel, memory::shared::size_t shared_mem_size);
void validate_compatibility(const device_t &device, memory::shared::size_t shared_mem_size);

} // namespace detail_

#if CUDA_VERSION >= 12000
enum class cluster_scheduling_policy_t {
    default_ = CU_CLUSTER_SCHEDULING_POLICY_DEFAULT,
    spread = CU_CLUSTER_SCHEDULING_POLICY_SPREAD,
    load_balance = CU_CLUSTER_SCHEDULING_POLICY_LOAD_BALANCING
};
#endif

struct launch_configuration_t {
    grid::composite_dimensions_t dimensions { grid::dimensions_t{ 0u, 0u, 0u }, grid::block_dimensions_t{ 0u, 0u, 0u } };

    memory::shared::size_t dynamic_shared_memory_size { 0u };

    bool block_cooperation { false };

#if CUDA_VERSION >= 12000

    bool programmatically_dependent_launch { true };

    struct programmatic_completion_t {
        event_t* event { nullptr };
        // unsigned flags; WHAT ABOUT THE FLAGS?
        bool trigger_event_at_block_start { true };
#if __cplusplus >= 202002L
        constexpr bool operator==(const programmatic_completion_t&) const noexcept = default;
#endif
    } programmatic_completion;

    bool in_remote_memory_synchronization_domain { false };

    struct clustering_t {
        grid::dimensions_t cluster_dimensions { 1, 1, 1 };
        cluster_scheduling_policy_t scheduling_policy { cluster_scheduling_policy_t::default_ };
#if __cplusplus >= 202002L
        constexpr bool operator==(const clustering_t &) const noexcept = default;
#endif
    } clustering;
#endif // CUDA_VERSION >= 12000

#if __cplusplus >= 202002L
    constexpr bool operator==(const launch_configuration_t&) const noexcept = default;
#endif
public: // non-mutators

    bool has_nondefault_attributes() const
    {
        if (block_cooperation) { return true; }
#if CUDA_VERSION >= 12000
        return  programmatically_dependent_launch or programmatic_completion.event
            or in_remote_memory_synchronization_domain or clustering.cluster_dimensions != grid::dimensions_t::point();
#else
        return false;
#endif
    }

    // In C++11, an inline initializer for a struct's field costs us a lot
    // of its defaulted constructors; but - we must initialize the shared
    // memory size to 0, as otherwise, people might be tempted to initialize
    // a launch configuration with { num_blocks, num_threads } - and get an
    // uninitialized shared memory size which they did not expect. So,
    // we do have the inline initializers above regardless of the language
    // standard version, and we just have to "pay the price" of spelling things out:
    launch_configuration_t() = delete;
    constexpr launch_configuration_t(const launch_configuration_t&) = default;
    constexpr launch_configuration_t(launch_configuration_t&&) = default;

    constexpr launch_configuration_t(
        grid::composite_dimensions_t grid_and_block_dimensions,
        memory::shared::size_t dynamic_shared_mem = 0u
    ) :
        dimensions{grid_and_block_dimensions},
        dynamic_shared_memory_size{dynamic_shared_mem}
    { }

    constexpr launch_configuration_t(
        grid::dimensions_t grid_dims,
        grid::dimensions_t block_dims,
        memory::shared::size_t dynamic_shared_mem = 0u
    ) : launch_configuration_t( {grid_dims, block_dims}, dynamic_shared_mem) { }

    // A "convenience" delegating ctor to avoid narrowing-conversion warnings
    constexpr launch_configuration_t(
        int grid_dims,
        int block_dims,
        memory::shared::size_t dynamic_shared_mem = 0u
    ) : launch_configuration_t(
        grid::dimensions_t(grid_dims),
        grid::block_dimensions_t(block_dims),
        dynamic_shared_mem)
    { }

    CPP14_CONSTEXPR launch_configuration_t& operator=(const launch_configuration_t& other) = default;
    CPP14_CONSTEXPR launch_configuration_t& operator=(launch_configuration_t&&) = default;
};

#if __cplusplus < 202002L
constexpr bool operator==(const launch_configuration_t lhs, const launch_configuration_t& rhs) noexcept
{
    return
        lhs.dimensions == rhs.dimensions
        and lhs.dynamic_shared_memory_size == rhs.dynamic_shared_memory_size
        and lhs.block_cooperation == rhs.block_cooperation
#if CUDA_VERSION >= 12000
        and lhs.programmatically_dependent_launch == rhs.programmatically_dependent_launch
        and lhs.programmatic_completion.event == rhs.programmatic_completion.event
        and lhs.programmatic_completion.trigger_event_at_block_start == rhs.programmatic_completion.trigger_event_at_block_start
        and lhs.in_remote_memory_synchronization_domain == rhs.in_remote_memory_synchronization_domain
        and lhs.clustering.cluster_dimensions == rhs.clustering.cluster_dimensions
        and lhs.clustering.scheduling_policy == rhs.clustering.scheduling_policy
#endif // CUDA_VERSION >= 12000
        ;
}

constexpr bool operator!=(const launch_configuration_t lhs, const launch_configuration_t& rhs) noexcept
{
    return not (lhs == rhs);
}
#endif

namespace detail_ {

// Note: This will not check anything related to the device or the kernel
// with which the launch configuration is to be used
inline void validate(const launch_configuration_t& launch_config) noexcept(false)
{
    validate_block_dimensions(launch_config.dimensions.block);
    validate_grid_dimensions(launch_config.dimensions.grid);
}

inline void validate_compatibility(
    const device_t& device,
    launch_configuration_t launch_config) noexcept(false)
{
    validate(launch_config);
    validate_block_dimension_compatibility(device, launch_config.dimensions.block);
    //  Uncomment if we actually get such checks
    //  validate_grid_dimension_compatibility(device, launch_config.dimensions.grid);
}

void validate_compatibility(
    const kernel_t& kernel,
    launch_configuration_t launch_config) noexcept(false);

using launch_attribute_index_t = unsigned int;

// ensure we have the same number here as the number of attribute insertions in marsha()
constexpr launch_attribute_index_t maximum_possible_kernel_launch_attributes = 7;

#if CUDA_VERSION >= 12000
// Note: The attribute_storage must have a capacity of maximum_possible_kernel_launch_attributes+1 at least
CUlaunchConfig marshal(
    const launch_configuration_t& config,
    const stream::handle_t stream_handle,
    span<CUlaunchAttribute> attribute_storage) noexcept(true);
#endif // CUDA_VERSION >= 12000

} // namespace detail_

} // namespace cuda

#endif // CUDA_API_WRAPPERS_LAUNCH_CONFIGURATION_CUH_