1 #ifndef DASH__IO__HDF5__STORAGEDRIVER_H__ 2 #define DASH__IO__HDF5__STORAGEDRIVER_H__ 4 #include <dash/internal/Config.h> 6 #ifdef DASH_ENABLE_HDF5 8 #ifndef DASH_MPI_IMPL_ID 9 #error "HDF5 module requires dart-mpi" 12 #include <dash/Exception.h> 13 #include <dash/Init.h> 14 #include <dash/Array.h> 15 #include <dash/Matrix.h> 16 #include <dash/Shared.h> 32 #include <type_traits> 44 hid_t get_h5_datatype();
47 using type_converter_fun_type = std::function<hid_t()>;
82 template <
class pattern_t>
83 static constexpr
bool _compatible_pattern() {
91 template <
class ViewType>
92 static constexpr
bool _is_origin_view() {
96 static std::vector<std::string> _split_string(
const std::string& str,
98 std::vector<std::string> elems;
102 while (std::getline(ss, item, delim)) {
104 elems.push_back(item);
121 template <
typename View_t>
126 std::string filename,
128 std::string datapath,
132 type_converter_fun_type to_h5_dt_converter = get_h5_datatype<
134 using Container_t =
typename dash::view_traits<View_t>::origin_type;
135 using pattern_t =
typename Container_t::pattern_type;
136 using extent_t =
typename pattern_t::size_type;
137 using index_t =
typename Container_t::index_type;
138 using value_t =
typename Container_t::value_type;
143 _verify_container_dims(array);
144 static_assert(std::is_same<index_t, typename pattern_t::index_type>::value,
145 "Specified index_t differs from pattern_t::index_type");
150 auto h5datatype = to_h5_dt_converter();
152 std::list<hid_t> open_groups;
154 auto path_vec = _split_string(datapath,
'/');
155 auto dataset = path_vec.back();
168 plist_id = H5Pcreate(H5P_FILE_ACCESS);
173 if (team.myid() == 0) {
174 if (access(filename.c_str(), F_OK) != -1) {
176 f_exists.
set(static_cast<int>(H5Fis_hdf5(filename.c_str())));
183 if (foptions.overwrite_file || (f_exists.
get() <= 0)) {
186 H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, plist_id);
189 file_id = H5Fopen(filename.c_str(), H5F_ACC_RDWR, plist_id);
197 for (std::string elem : path_vec) {
198 if (H5Lexists(loc_id, elem.c_str(), H5P_DEFAULT)) {
200 DASH_LOG_DEBUG(
"Open Group", elem);
201 loc_id = H5Gopen2(loc_id, elem.c_str(), H5P_DEFAULT);
204 DASH_LOG_DEBUG(
"Create Group", elem);
205 loc_id = H5Gcreate2(loc_id, elem.c_str(), H5P_DEFAULT, H5P_DEFAULT,
208 if (loc_id != file_id) {
209 open_groups.push_back(loc_id);
214 auto filespace_extents = _get_container_extents(array);
217 filespace = H5Screate_simple(ndim, filespace_extents.extent, NULL);
218 internal_type = H5Tcopy(h5datatype);
220 if (foptions.modify_dataset) {
222 h5dset = H5Dopen(loc_id,
dataset.c_str(), H5P_DEFAULT);
225 h5dset = H5Dcreate(loc_id,
dataset.c_str(), internal_type, filespace,
226 H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
234 _write_dataset_impl(array, h5dset, internal_type);
239 if (foptions.store_pattern && _is_origin_view<Container_t>()) {
240 DASH_LOG_DEBUG(
"store pattern in hdf5 file");
241 _store_pattern(array, h5dset, foptions);
246 H5Tclose(internal_type);
249 [](hid_t& group_id) { H5Gclose(group_id); });
264 template <
typename Container_t>
265 typename std::enable_if<
266 _compatible_pattern<typename Container_t::pattern_type>() &&
267 _is_origin_view<Container_t>(),
273 std::string filename,
275 std::string datapath,
279 type_converter_fun_type to_h5_dt_converter =
280 get_h5_datatype<typename Container_t::value_type>) {
281 using pattern_t =
typename Container_t::pattern_type;
282 using extent_t =
typename pattern_t::size_type;
283 using index_t =
typename Container_t::index_type;
284 using value_t =
typename Container_t::value_type;
288 static_assert(std::is_same<index_t, typename pattern_t::index_type>::value,
289 "Specified index_t differs from pattern_t::index_type");
298 hsize_t data_dimsf[
ndim];
306 bool is_alloc = (matrix.size() != 0);
309 plist_id = H5Pcreate(H5P_FILE_ACCESS);
321 file_id = H5Fopen(filename.c_str(), H5P_DEFAULT, plist_id);
327 h5dset = H5Dopen(file_id, datapath.c_str(), H5P_DEFAULT);
330 filespace = H5Dget_space(h5dset);
331 rank = H5Sget_simple_extent_ndims(filespace);
333 DASH_ASSERT_EQ(rank, ndim,
334 "Data dimension of HDF5 dataset does not match matrix " 337 status = H5Sget_simple_extent_dims(filespace, data_dimsf, NULL);
339 std::array<extent_t, ndim> size_extents;
342 for (
int i = 0; i <
ndim; ++i) {
343 size_extents[i] = data_dimsf[i];
347 auto pat_key = foptions.pattern_metadata_key.c_str();
350 && foptions.restore_pattern
351 && H5Aexists(h5dset, pat_key))
353 _restore_pattern(matrix, h5dset, foptions);
354 }
else if (is_alloc) {
355 DASH_LOG_DEBUG(
"Matrix already allocated");
357 auto container_extents = _get_container_extents(matrix);
358 for (
int i = 0; i <
ndim; ++i) {
359 DASH_ASSERT_EQ(size_extents[i], container_extents.extent[i],
360 "Container extents do not match data extents");
368 matrix.allocate(pattern);
371 h5datatype = to_h5_dt_converter();
372 internal_type = H5Tcopy(h5datatype);
376 _read_dataset_impl(matrix, h5dset, internal_type);
382 H5Tclose(internal_type);
385 matrix.team().barrier();
388 template <
class Container_t>
389 typename std::enable_if<
390 !(_compatible_pattern<typename Container_t::pattern_type>() &&
391 _is_origin_view<Container_t>()),
397 std::string filename,
399 std::string datapath,
403 type_converter_fun_type to_h5_dt_converter =
404 get_h5_datatype<typename Container_t::value_type>) {}
410 template <dim_t ndim>
412 std::array<hsize_t, ndim> count{{0}};
413 std::array<hsize_t, ndim> stride{{0}};
414 std::array<hsize_t, ndim> offset{{0}};
415 std::array<hsize_t, ndim>
block{{0}};
423 template <dim_t ndim>
428 template <dim_t ndim>
432 std::array<hsize_t, ndim> data_extf{{0}};
433 std::array<hsize_t, ndim> data_extm{{0}};
435 size_t contrib_data = 0;
436 bool contrib_blocks =
false;
444 enum class Mode : uint16_t {
449 template <
class BlockSpec_t,
typename index_t>
450 index_t
static inline _blockspec_at(
const BlockSpec_t& lblockspec,
451 const std::array<index_t, 1>& coords) {
455 template <
class BlockSpec_t,
typename index_t, std::
size_t ndim>
456 index_t
static inline _blockspec_at(
const BlockSpec_t& lblockspec,
457 const std::array<index_t, ndim>& coords) {
458 return lblockspec.at(coords);
465 template <
class pattern_t>
468 const pattern_t& pattern,
470 const std::vector<dim_t> dimensions) {
471 using index_t =
typename pattern_t::index_type;
474 auto & ms = hs.memory;
475 auto & ts = hs.dataset;
477 const auto & lblockspec = pattern.local_blockspec();
481 if (lblockspec.size() == 0) {
484 std::array<index_t, ndim> coords{{0}};
486 for (
auto d : dimensions) {
487 coords[d] = lblockspec.extent(d) - 1;
489 lblckidx = _blockspec_at(lblockspec, coords);
491 hs.contrib_blocks =
true;
493 for (
int i = 0; i <
ndim; ++i) {
494 auto tilesize = pattern.blocksize(i);
495 auto num_tiles = pattern.local_extent(i) / tilesize;
498 if (
std::find(dimensions.begin(), dimensions.end(), i) !=
502 ts.block[i] = pattern.local_block(lblckidx).extent(i);
503 if (pattern.local_extent(i) == num_tiles * tilesize) {
509 if (num_tiles == 0) {
513 ts.count[i] = num_tiles;
514 ts.block[i] = tilesize;
516 ts.offset[i] = pattern.local_block(lblckidx).offset(i);
519 ts.stride[i] = pattern.teamspec().extent(i) * ts.block[i];
524 ms.count[i] = ts.count[i];
525 ms.block[i] = ts.block[i];
526 ms.offset[i] = pattern.local_block_local(lblckidx).offset(i);
527 ms.stride[i] = tilesize;
529 hs.data_extm[i] = pattern.local_extent(i);
530 hs.contrib_data *= ts.count[i] * ts.block[i];
532 DASH_LOG_DEBUG(
"dimensions", dimensions);
533 DASH_LOG_DEBUG(
"ts.count", i, ts.count[i]);
534 DASH_LOG_DEBUG(
"ts.offset", i, ts.offset[i]);
535 DASH_LOG_DEBUG(
"ts.block", i, ts.block[i]);
536 DASH_LOG_DEBUG(
"ts.stride", i, ts.stride[i]);
537 DASH_LOG_DEBUG(
"ms.count", i, ms.count[i]);
538 DASH_LOG_DEBUG(
"ms.block", i, ms.block[i]);
539 DASH_LOG_DEBUG(
"ms.offset", i, ms.offset[i]);
540 DASH_LOG_DEBUG(
"ms.stride", i, ms.stride[i]);
542 hs.contrib_blocks =
true;
550 template <
typename PatternT>
551 const static inline std::vector<hdf5_hyperslab_spec<PatternT::ndim()>>
552 _get_hdf_slabs_with_underfilled(
const PatternT& pattern) {
554 std::vector<hdf5_hyperslab_spec<ndim>> specs_hyperslab;
555 std::vector<dim_t> dimensions;
558 specs_hyperslab.push_back(_get_hdf_slab_body(pattern, dimensions));
560 dimensions.push_back(-1);
561 while (dimensions.size() > 0) {
563 if (ndim == dimensions.back()) {
564 dimensions.pop_back();
566 }
else if (pattern.underfilled_blocksize(dimensions.back()) == 0) {
569 auto& current_hs = _get_hdf_slab_body(pattern, dimensions);
570 if (current_hs.contrib_blocks) {
571 specs_hyperslab.push_back(current_hs);
573 dimensions.push_back(dimensions.back());
576 return specs_hyperslab;
583 template <dim_t ndim, MemArrange Arr,
typename index_t>
584 const static inline std::vector<hdf5_hyperslab_spec<ndim>>
588 return _get_hdf_slabs_with_underfilled(pattern);
595 template <dim_t ndim, MemArrange Arr,
typename index_t>
596 const static inline std::vector<hdf5_hyperslab_spec<ndim>>
600 return _get_hdf_slabs_with_underfilled(pattern);
606 template <
class pattern_t>
607 const static inline std::vector<hdf5_hyperslab_spec<pattern_t::ndim()>>
608 _get_hdf_slabs(
const pattern_t& pattern) {
610 return std::vector<hdf5_hyperslab_spec<pattern_t::ndim()>>(
611 1, _get_hdf_slab_body(pattern, {}));
614 template <
typename value_t>
618 fs.extent[0] = array.
size();
622 template <
class Container_t>
624 _get_container_extents(Container_t& container) {
627 for (
int i = 0; i <
ndim; ++i) {
628 fs.extent[i] = container.extent(i);
635 template <
typename ViewType >
637 _get_container_extents(ViewType & container){
640 for(
int i=0; i<
ndim; ++i){
641 fs.extent[i] = dash::extent<i>(container);
647 template <dim_t ndim,
typename value_t,
typename index_t,
typename pattern_t>
648 static inline void _verify_container_dims(
651 "Pattern dimension has to match matrix dimension");
653 template <
class Container_t>
654 static inline void _verify_container_dims(
const Container_t& container) {
658 template <
typename Container_t>
659 typename std::enable_if<
660 _is_origin_view<Container_t>(),
661 void>::type
static _store_pattern(Container_t& container, hid_t h5dset,
663 using pattern_t =
typename Container_t::pattern_type;
664 using extent_t =
typename pattern_t::size_type;
667 auto& pattern = container.pattern();
670 extent_t pattern_spec[ndim * 4];
674 H5Adelete(h5dset, pat_key);
678 for (
int i = 0; i <
ndim; ++i) {
679 pattern_spec[i] = pattern.sizespec().extent(i);
680 pattern_spec[i +
ndim] = pattern.teamspec().extent(i);
681 pattern_spec[i + (ndim * 2)] = pattern.blockspec().extent(i);
682 pattern_spec[i + (ndim * 3)] = pattern.blocksize(i);
685 hsize_t attr_len[] = {
static_cast<hsize_t
>(ndim * 4)};
686 hid_t attrspace = H5Screate_simple(1, attr_len, NULL);
687 hid_t attribute_id = H5Acreate(h5dset, pat_key, H5T_NATIVE_LONG, attrspace,
688 H5P_DEFAULT, H5P_DEFAULT);
689 H5Awrite(attribute_id, H5T_NATIVE_LONG, &pattern_spec);
690 H5Aclose(attribute_id);
694 template <
typename Container_t>
695 typename std::enable_if<
696 !_is_origin_view<Container_t>(),
697 void>::type
static _store_pattern(Container_t& container, hid_t h5dset,
700 template <
typename Container_t>
701 typename std::enable_if<
702 _is_origin_view<Container_t>(),
703 void>::type
static _restore_pattern(Container_t& container, hid_t h5dset,
705 using pattern_t =
typename Container_t::pattern_type;
706 using extent_t =
typename pattern_t::size_type;
709 std::array<extent_t, ndim> size_extents;
710 std::array<extent_t, ndim> team_extents;
711 std::array<dash::Distribution, ndim> dist_extents;
712 extent_t hdf_dash_pattern[ndim * 4];
714 hsize_t attr_len[] = {ndim * 4};
715 hid_t attrspace = H5Screate_simple(1, attr_len, NULL);
719 H5Aread(attribute_id, H5T_NATIVE_LONG, hdf_dash_pattern);
720 H5Aclose(attribute_id);
723 for (
int i = 0; i <
ndim; ++i) {
724 size_extents[i] =
static_cast<extent_t
>(hdf_dash_pattern[i]);
725 team_extents[i] =
static_cast<extent_t
>(hdf_dash_pattern[i +
ndim]);
726 dist_extents[i] = dash::TILE(hdf_dash_pattern[i + (ndim * 3)]);
728 DASH_LOG_DEBUG(
"Created pattern according to metadata");
736 container.allocate(pattern);
739 template <
typename Container_t>
740 typename std::enable_if<
741 !_is_origin_view<Container_t>(),
742 void>::type
static _restore_pattern(Container_t& container, hid_t h5dset,
755 template <
class Container_t>
756 typename std::enable_if<
757 _is_origin_view<Container_t>() &&
758 _compatible_pattern<typename Container_t::pattern_type>(),
759 void>::type
static _write_dataset_impl(Container_t& container,
761 const hid_t& internal_type) {
762 _process_dataset_impl_zero_copy(StoreHDF::Mode::WRITE, container, h5dset,
772 template <
class Container_t>
773 typename std::enable_if<
774 !(_is_origin_view<Container_t>() &&
775 _compatible_pattern<typename Container_t::pattern_type>()),
776 void>::type
static _write_dataset_impl(Container_t& container,
778 const hid_t& internal_type) {
779 _write_dataset_impl_buffered(container, h5dset, internal_type);
782 template <
class Container_t>
783 static void _process_dataset_impl_zero_copy(StoreHDF::Mode io_mode,
784 Container_t& container,
786 const hid_t& internal_type);
788 template <
class Container_t>
789 static void _write_dataset_impl_buffered(Container_t& container,
791 const hid_t& internal_type);
799 static void _write_dataset_impl_nd_block(
802 const hid_t& internal_type);
814 template <
class Container_t>
815 typename std::enable_if<
816 _compatible_pattern<typename Container_t::pattern_type>() &&
817 _is_origin_view<Container_t>(),
818 void>::type
static inline _read_dataset_impl(Container_t& container,
820 const hid_t& internal_type) {
821 _process_dataset_impl_zero_copy(StoreHDF::Mode::READ, container, h5dset,
830 #include <dash/io/hdf5/internal/DriverImplZeroCopy.h> 831 #include <dash/io/hdf5/internal/DriverImplBuffered.h> 832 #include <dash/io/hdf5/internal/DriverImplNdBlock.h> 834 #include <dash/io/hdf5/internal/StorageDriver-inl.h> 836 #endif // DASH_ENABLE_HDF5 838 #endif // DASH__IO__HDF5__STORAGEDRIVER_H__ constexpr dim_t rank(const DimensionalType &d)
static std::enable_if< !(_compatible_pattern< typename Container_t::pattern_type >) &&_is_origin_view< Container_t >)), void >::type read(Container_t &matrix, std::string filename, std::string datapath, hdf5_options foptions=hdf5_options(), type_converter_fun_type to_h5_dt_converter=get_h5_datatype< typename Container_t::value_type >)
static void write(View_t &array, std::string filename, std::string datapath, hdf5_options foptions=hdf5_options(), type_converter_fun_type to_h5_dt_converter=get_h5_datatype< typename dash::view_traits< View_t >::origin_type::value_type >)
Store all dash::Matrix values in an HDF5 file using parallel IO.
Defines how a list of global indices is mapped to single units within a Team.
This class is a simple memory pool which holds allocates elements of size ValueType.
bool modify_dataset
Modify an already existing HDF5 dataset.
Specifies cartesian extents in a specific number of dimensions.
int dim_t
Scalar type for a dimension value, with 0 indicating the first dimension.
Stream manipulator class to set whether the selected dataset should be overwritten.
Defines how a list of global indices is mapped to single units within a Team.
bool overwrite_file
Overwrite HDF5 file if already existing.
reference get()
Get a reference on the shared value.
void set(const value_type &val)
Set the value of the shared element.
std::string pattern_metadata_key
Metadata attribute key in HDF5 file.
hdf5 pattern specification for parallel IO
A Team instance specifies a subset of all available units.
A set of utility routines used to provide parallel io support.
constexpr dim_t ndim(const DimensionalType &d)
Specifies the arrangement of team units in a specified number of dimensions.
Stream manipulator class to set whether the pattern should be stored as metadata of the hdf5 dataset ...
Stream manipulator class to specify the hdf5 dataset.
constexpr size_type size() const noexcept
The size of the array.
static std::enable_if< _compatible_pattern< typename Container_t::pattern_type >) &&_is_origin_view< Container_t >), void >::type read(Container_t &matrix, std::string filename, std::string datapath, hdf5_options foptions=hdf5_options(), type_converter_fun_type to_h5_dt_converter=get_h5_datatype< typename Container_t::value_type >)
Read an HDF5 dataset into a dash container using parallel IO if the matrix is already allocated...
dart_ret_t dart__io__hdf5__prep_mpio(hid_t plist_id, dart_team_t teamid)
setup hdf5 for parallel io using mpi-io
DASH wrapper to store an dash::Array or dash::Matrix in an HDF5 file using parallel IO...
GlobIter find(GlobIter first, GlobIter last, const ElementType &value)
Returns an iterator to the first element in the range [first,last) that compares equal to val...
Stream manipulator class to set whether the pattern should be restored from the hdf5 dataset metadata...
constexpr DimensionalType::extent_type extent(const DimensionalType &d)
Shared access to a value in global memory across a team.
hdf5 pattern specification for parallel IO
static Team & All()
The invariant Team instance containing all available units.
Options which can be passed to dash::io::StoreHDF::write to specify how existing structures are treat...
void for_each(const GlobInputIt &first, const GlobInputIt &last, UnaryFunction func)
Invoke a function on every element in a range distributed by a pattern.
dart_team_t dart_id() const
Index of this team relative to global team dash::Team::All().
DistributionSpec describes distribution patterns of all dimensions,.
constexpr auto block(OffsetT block_idx, const ViewType &view) -> typename std::enable_if<(!dash::view_traits< ViewType >::is_local::value), ViewBlockMod< ViewType > >::type
Blocks view from global view.