context.hpp

Go to the documentation of this file.

#pragma once
#ifndef CUDA_API_WRAPPERS_CONTEXT_HPP_
#define CUDA_API_WRAPPERS_CONTEXT_HPP_

#include "current_context.hpp"
#include "versions.hpp"
#include "error.hpp"
#include "constants.hpp"
#include "types.hpp"

#include <string>
#include <utility>

namespace cuda {

class device_t;
class event_t;
class context_t;
class stream_t;
class module_t;

namespace link {
struct options_t;
} // namespace link

namespace context {

using limit_t = CUlimit;

using limit_value_t = size_t;

using shared_memory_bank_size_t = CUsharedconfig;

struct stream_priority_range_t {
    stream::priority_t least;

    stream::priority_t greatest;

    constexpr bool is_trivial() const
    {
        return least == stream::default_priority and greatest == stream::default_priority;
    }
};

context_t wrap(
    device::id_t       device_id,
    context::handle_t  context_id,
    bool               take_ownership = false) noexcept;

namespace detail_ {

::std::string identify(const context_t& context);

inline limit_value_t get_limit(limit_t limit_id)
{
    limit_value_t limit_value;
    auto status = cuCtxGetLimit(&limit_value, limit_id);
    throw_if_error_lazy(status, "Failed obtaining CUDA context limit value");
    return limit_value;
}

inline void set_limit(limit_t limit_id, limit_value_t new_value)
{
    auto status = cuCtxSetLimit(limit_id, new_value);
    throw_if_error_lazy(status, "Failed obtaining CUDA context limit value");
}

constexpr flags_t inline make_flags(
    host_thread_sync_scheduling_policy_t   sync_scheduling_policy,
    bool                                   keep_larger_local_mem_after_resize)
{
    return
          sync_scheduling_policy // this enum value is also a valid bitmask
        | (keep_larger_local_mem_after_resize    ? CU_CTX_LMEM_RESIZE_TO_MAX : 0);
}

// consider renaming this: device_id_of
inline device::id_t get_device_id(handle_t context_handle)
{
    auto needed_push = current::detail_::push_if_not_on_top(context_handle);
    auto device_id = current::detail_::get_device_id();
    if (needed_push) {
        current::detail_::pop();
    }
    return device_id;
}


context_t from_handle(
    context::handle_t  context_handle,
    bool               take_ownership = false);

inline size_t total_memory(handle_t handle)
{
    size_t total_mem_in_bytes;
    auto status = cuMemGetInfo(nullptr, &total_mem_in_bytes);
    throw_if_error_lazy(status, "Failed determining amount of total memory for " + identify(handle));
    return total_mem_in_bytes;

}

inline size_t free_memory(handle_t handle)
{
    size_t free_mem_in_bytes;
    auto status = cuMemGetInfo(&free_mem_in_bytes, nullptr);
    throw_if_error_lazy(status, "Failed determining amount of free memory for " + identify(handle));
    return free_mem_in_bytes;
}

inline void set_cache_preference(handle_t handle, multiprocessor_cache_preference_t preference)
{
    auto status = cuCtxSetCacheConfig(static_cast<CUfunc_cache>(preference));
    throw_if_error_lazy(status,
        "Setting the multiprocessor L1/Shared Memory cache distribution preference to " +
        ::std::to_string(static_cast<unsigned>(preference)) + " for " + identify(handle));
}

inline multiprocessor_cache_preference_t cache_preference(handle_t handle)
{
    CUfunc_cache preference;
    auto status = cuCtxGetCacheConfig(&preference);
    throw_if_error_lazy(status,
        "Obtaining the multiprocessor L1/Shared Memory cache distribution preference for " + identify(handle));
    return static_cast<multiprocessor_cache_preference_t>(preference);
}

#if CUDA_VERSION < 12030
inline shared_memory_bank_size_t shared_memory_bank_size(handle_t handle)
{
    CUsharedconfig bank_size;
    auto status = cuCtxGetSharedMemConfig(&bank_size);
    throw_if_error_lazy(status, "Obtaining the multiprocessor shared memory bank size for " + identify(handle));
    return static_cast<shared_memory_bank_size_t>(bank_size);
}
#endif // CUDA_VERSION < 12030

#if CUDA_VERSION < 12030
inline void set_shared_memory_bank_size(handle_t handle, shared_memory_bank_size_t bank_size)
{
    auto status = cuCtxSetSharedMemConfig(static_cast<CUsharedconfig>(bank_size));
    throw_if_error_lazy(status, "Setting the multiprocessor shared memory bank size for " + identify(handle));
}
#endif // CUDA_VERSION < 12030


inline void synchronize(context::handle_t handle)
{
    CAW_SET_SCOPE_CONTEXT(handle);
    context::current::detail_::synchronize(handle);
}

inline void synchronize(device::id_t device_id, context::handle_t handle)
{
    CAW_SET_SCOPE_CONTEXT(handle);
    context::current::detail_::synchronize(device_id, handle);
}

inline status_t destroy_nothrow(handle_t handle) noexcept
{
    return cuCtxDestroy(handle);
}

inline void destroy(handle_t handle)
{
    auto status = destroy_nothrow(handle);
    throw_if_error_lazy(status, "Failed destroying " + identify(handle));
}

inline void destroy(handle_t handle, device::id_t device_index)
{
    auto status = destroy_nothrow(handle);
    throw_if_error_lazy(status, "Failed destroying " + identify(handle, device_index));
}

inline context::flags_t get_flags(handle_t handle)
{
    CAW_SET_SCOPE_CONTEXT(handle);
    return context::current::detail_::get_flags();
}

} // namespace detail_

} // namespace context

inline void synchronize(const context_t& context);

class context_t {
protected: // types
    using flags_type = context::flags_t;

public: // types

    static_assert(
        ::std::is_same<::std::underlying_type<CUsharedconfig>::type, ::std::underlying_type<cudaSharedMemConfig>::type>::value,
        "Unexpected difference between enumerators used for the same purpose by the CUDA runtime and the CUDA driver");

public: // inner classes

    class global_memory_type {
#if CUDA_VERSION >= 11040
    public: // data types
        using execution_graph_related_attribute_t = CUgraphMem_attribute;
#endif // CUDA_VERSION >= 11040


    protected: // data members
        const device::id_t device_id_;
        const context::handle_t context_handle_;

    public:
        global_memory_type(device::id_t device_id, context::handle_t context_handle)
            : device_id_(device_id), context_handle_(context_handle)
        {}

        device_t associated_device() const;

        context_t associated_context() const;

        memory::region_t allocate(size_t size_in_bytes) const;

        memory::region_t allocate_managed(
            size_t size_in_bytes,
            cuda::memory::managed::initial_visibility_t initial_visibility =
            cuda::memory::managed::initial_visibility_t::to_supporters_of_concurrent_managed_access) const;

        size_t amount_total() const
        {
            CAW_SET_SCOPE_CONTEXT(context_handle_);
            return context::detail_::total_memory(context_handle_);
        }

        size_t amount_free() const
        {
            CAW_SET_SCOPE_CONTEXT(context_handle_);
            return context::detail_::free_memory(context_handle_);
        }

#if CUDA_VERSION >= 11040

        void free_unused_execution_graph_memory() const
        {
            auto status = cuDeviceGraphMemTrim(device_id_);
            throw_if_error_lazy(status,
                "Trimming memory used for CUDA execution graphs on " + device::detail_::identify(device_id_));
        }

        size_t get_execution_graph_related_attribute(execution_graph_related_attribute_t attribute) const
        {
            cuuint64_t result;
            auto status = cuDeviceGetGraphMemAttribute(device_id_, attribute, &result);
            throw_if_error_lazy(status, "Failed obtaining an execution-graph-related memory attribute for "
                                        + device::detail_::identify(device_id_));
            return result;
        }

        void reset_execution_graph_usage_high_watermark() const
        {
            cuuint64_t value_{0};
            auto status = cuDeviceSetGraphMemAttribute(device_id_, CU_GRAPH_MEM_ATTR_USED_MEM_HIGH, &value_);
            throw_if_error_lazy(status, "Failed setting an execution-graph-related memory attribute for "
                                        + device::detail_::identify(device_id_));
        }
#endif // CUDA_VERSION >= 11040
    }; // class global_memory_type


public: // data member non-mutator getters

    context::handle_t handle() const noexcept { return handle_; }
    device::id_t device_id() const noexcept { return device_id_; }
    device_t device() const;

    bool is_owning() const noexcept { return owning_;  }

    size_t total_memory() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::total_memory(handle_);
    }

    size_t free_memory() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::free_memory(handle_);
    }

public: // other non-mutator methods

    stream_t default_stream() const;

    template <typename Kernel, typename ... KernelParameters>
    void launch(
        Kernel                  kernel,
        launch_configuration_t  launch_configuration,
        KernelParameters...     parameters) const;

    multiprocessor_cache_preference_t cache_preference() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::cache_preference(handle_);
    }

    size_t stack_size() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(CU_LIMIT_STACK_SIZE);
    }

    context::limit_value_t printf_buffer_size() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(CU_LIMIT_PRINTF_FIFO_SIZE);
    }

    context::limit_value_t memory_allocation_heap_size() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(CU_LIMIT_MALLOC_HEAP_SIZE);
    }

    context::limit_value_t maximum_depth_of_child_grid_sync_calls() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(CU_LIMIT_DEV_RUNTIME_SYNC_DEPTH);
    }

    global_memory_type memory() const
    {
        return { device_id_, handle_ };
    }

    context::limit_value_t maximum_outstanding_kernel_launches() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(CU_LIMIT_DEV_RUNTIME_PENDING_LAUNCH_COUNT);
    }

#if CUDA_VERSION >= 10000

    context::limit_value_t l2_fetch_granularity() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(CU_LIMIT_MAX_L2_FETCH_GRANULARITY);
    }
#endif

#if CUDA_VERSION < 12030

    context::shared_memory_bank_size_t shared_memory_bank_size() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::shared_memory_bank_size(handle_);
    }
#endif // CUDA_VERSION < 12030

    bool is_current() const
    {
        return context::current::detail_::is_(handle_);
    }

    bool is_primary() const;

    context::stream_priority_range_t stream_priority_range() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        context::stream_priority_range_t result;
        auto status = cuCtxGetStreamPriorityRange(&result.least, &result.greatest);
        throw_if_error_lazy(status, "Obtaining the priority range for streams within " +
            context::detail_::identify(*this));
        return result;
    }

    context::limit_value_t get_limit(context::limit_t limit_id) const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::get_limit(limit_id);
    }

    version_t api_version() const
    {
        unsigned int raw_version;
        auto status = cuCtxGetApiVersion(handle_, &raw_version);
        throw_if_error_lazy(status, "Failed obtaining the API version for " + context::detail_::identify(*this));
        return version_t::from_single_number(static_cast<combined_version_t>(raw_version));
    }

protected:
    context::flags_t flags() const
    {
        return context::detail_::get_flags(handle_);
    }

public: // methods which mutate the context, but not its wrapper
    context::host_thread_sync_scheduling_policy_t sync_scheduling_policy() const
    {
        return context::host_thread_sync_scheduling_policy_t(flags() & CU_CTX_SCHED_MASK);
    }

    bool keeping_larger_local_mem_after_resize() const
    {
        return flags() & CU_CTX_LMEM_RESIZE_TO_MAX;
    }

    stream_t create_stream(
        bool                will_synchronize_with_default_stream,
        stream::priority_t  priority = cuda::stream::default_priority) const;

    event_t create_event(
        bool uses_blocking_sync = event::sync_by_busy_waiting, // Yes, that's the runtime default
        bool records_timing     = event::do_record_timings,
        bool interprocess       = event::not_interprocess) const;

    template <typename ContiguousContainer,
        cuda::detail_::enable_if_t<detail_::is_kinda_like_contiguous_container<ContiguousContainer>::value, bool> = true>
    module_t create_module(ContiguousContainer module_data, const link::options_t& link_options) const;

    template <typename ContiguousContainer,
        cuda::detail_::enable_if_t<detail_::is_kinda_like_contiguous_container<ContiguousContainer>::value, bool> = true>
    module_t create_module(ContiguousContainer module_data) const;

public: // Methods which don't mutate the context, but affect the device itself


    void enable_access_to(const context_t& peer) const;

    void disable_access_to(const context_t& peer) const;

    void reset_persisting_l2_cache() const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
#if (CUDA_VERSION >= 11000)
        auto status = cuCtxResetPersistingL2Cache();
        throw_if_error_lazy(status, "Failed resetting/clearing the persisting L2 cache memory");
#endif
        throw cuda::runtime_error(
            cuda::status::insufficient_driver,
            "Resetting/clearing the persisting L2 cache memory is not supported when compiling CUDA versions lower than 11.0");
    }

public: // other methods which don't mutate this class as a reference, but do mutate the context

#if CUDA_VERSION < 12030

    void set_shared_memory_bank_size(context::shared_memory_bank_size_t bank_size) const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        context::detail_::set_shared_memory_bank_size(handle_, bank_size);
    }
#endif // CUDA_VERSION < 12030

    void set_cache_preference(multiprocessor_cache_preference_t preference) const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        context::detail_::set_cache_preference(handle_, preference);
    }

    void set_limit(context::limit_t limit_id, context::limit_value_t new_value) const
    {
        CAW_SET_SCOPE_CONTEXT(handle_);
        return context::detail_::set_limit(limit_id, new_value);
    }

    void stack_size(context::limit_value_t new_value) const
    {
        return set_limit(CU_LIMIT_STACK_SIZE, new_value);
    }

    void printf_buffer_size(context::limit_value_t new_value) const
    {
        return set_limit(CU_LIMIT_PRINTF_FIFO_SIZE, new_value);
    }

    void memory_allocation_heap_size(context::limit_value_t new_value) const
    {
        return set_limit(CU_LIMIT_MALLOC_HEAP_SIZE, new_value);
    }

    void set_maximum_depth_of_child_grid_sync_calls(context::limit_value_t new_value) const
    {
        return set_limit(CU_LIMIT_DEV_RUNTIME_SYNC_DEPTH, new_value);
    }

    void set_maximum_outstanding_kernel_launches(context::limit_value_t new_value) const
    {
        return set_limit(CU_LIMIT_DEV_RUNTIME_PENDING_LAUNCH_COUNT, new_value);
    }

    void synchronize() const
    {
        cuda::synchronize(*this);
    }

protected: // constructors

    context_t(
        device::id_t       device_id,
        context::handle_t  context_id,
        bool               take_ownership) noexcept
        : device_id_(device_id), handle_(context_id), owning_(take_ownership)
    { }

public: // friendship

    friend context_t context::wrap(
        device::id_t       device_id,
        context::handle_t  context_id,
        bool               take_ownership) noexcept;

public: // constructors and destructor

    context_t(const context_t& other) :
        context_t(other.device_id_, other.handle_, false)
    { };

    context_t(context_t&& other) noexcept:
        context_t(other.device_id_, other.handle_, other.owning_)
    {
        other.owning_ = false;
    };

    ~context_t() DESTRUCTOR_EXCEPTION_SPEC
    {
        if (not owning_) { return; }
#if THROW_IN_DESTRUCTORS
        context::detail_::destroy(handle_, device_id_);
#else
        context::detail_::destroy_nothrow(handle_);
#endif
    }

public: // operators

    context_t& operator=(const context_t&) = delete;
    context_t& operator=(context_t&& other) noexcept
    {
        ::std::swap(device_id_, other.device_id_);
        ::std::swap(handle_, other.handle_);
        ::std::swap(owning_, other.owning_);
        return *this;
    }

protected: // data members
    device::id_t       device_id_;
    context::handle_t  handle_;
    bool               owning_;
        // this field is mutable only for enabling move construction; other
        // than in that case it must not be altered

    // TODO: Should we hold a field indicating whether this context is
    // primary or not?
};

inline bool operator==(const context_t& lhs, const context_t& rhs) noexcept
{
    // Note: Contexts on different devices cannot have the same context handle,
    // so this is redundant, but let's be extra safe:
    return lhs.device_id() == rhs.device_id() and lhs.handle() == rhs.handle();
}

inline bool operator!=(const context_t& lhs, const context_t& rhs) noexcept
{
    return not (lhs == rhs);
}

namespace context {

inline context_t wrap(
    device::id_t       device_id,
    handle_t           context_id,
    bool               take_ownership) noexcept
{
    return { device_id, context_id, take_ownership };
}

namespace detail_ {

inline context_t from_handle(
    context::handle_t  context_handle,
    bool               take_ownership)
{
    device::id_t device_id = get_device_id(context_handle);
    return wrap(device_id, context_handle, take_ownership);
}

inline handle_t create_and_push(
    device::id_t                           device_id,
    host_thread_sync_scheduling_policy_t   sync_scheduling_policy = automatic,
    bool                                   keep_larger_local_mem_after_resize = false)
{
    auto flags = context::detail_::make_flags(
        sync_scheduling_policy,
        keep_larger_local_mem_after_resize);
    handle_t handle;
#if CUDA_VERSION >= 13000
    // TODO: Introduce support for setting meaningful values for these parameters
    CUctxCreateParams creation_params = {};
    auto status = cuCtxCreate(&handle, &creation_params, flags, device_id);
#else
    auto status = cuCtxCreate(&handle, flags, device_id);
#endif
    throw_if_error_lazy(status, "failed creating a CUDA context associated with "
        + device::detail_::identify(device_id));
    return handle;
}

} // namespace detail_

context_t create(
    const device_t&                        device,
    host_thread_sync_scheduling_policy_t   sync_scheduling_policy = heuristic,
    bool                                   keep_larger_local_mem_after_resize = false);

context_t create_and_push(
    const device_t&                        device,
    host_thread_sync_scheduling_policy_t   sync_scheduling_policy = heuristic,
    bool                                   keep_larger_local_mem_after_resize = false);

namespace current {

inline context_t get()
{
    auto handle = detail_::get_handle();
    if (handle == context::detail_::none) {
        throw ::std::runtime_error("Attempt to obtain the current CUDA context when no context is current.");
    }
    return context::detail_::from_handle(handle);
}

inline void set(const context_t& context)
{
    return detail_::set(context.handle());
}

inline bool push_if_not_on_top(const context_t& context)
{
    return context::current::detail_::push_if_not_on_top(context.handle());
}

inline void push(const context_t& context)
{
    return context::current::detail_::push(context.handle());
}

inline context_t pop()
{
    static constexpr const bool do_not_take_ownership { false };
    // Unfortunately, since we don't store the device IDs of contexts
    // on the stack, this incurs an extra API call beyond just the popping...
    auto handle = context::current::detail_::pop();
    auto device_id = context::detail_::get_device_id(handle);
    return context::wrap(device_id, handle, do_not_take_ownership);
}

namespace detail_ {

handle_t push_default_if_missing();

inline context_t get_with_fallback_push()
{
    auto handle = push_default_if_missing();
    return context::detail_::from_handle(handle);
}


} // namespace detail_

} // namespace current

bool is_primary(const context_t& context);

namespace detail_ {

inline ::std::string identify(const context_t& context)
{
    return identify(context.handle(), context.device_id());
}

} // namespace detail_

} // namespace context

inline void synchronize(const context_t& context)
{
    context::detail_::synchronize(context.device_id(), context.handle());
}

} // namespace cuda

#endif // CUDA_API_WRAPPERS_CONTEXT_HPP_