Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
LimaGroup
Lima2
Commits
13a23596
Commit
13a23596
authored
Feb 17, 2022
by
Alejandro Homs Puron
Browse files
[IO] Add full/partial data write & read to H5 wrapper API
parent
03767d3f
Changes
4
Hide whitespace changes
Inline
Side-by-side
include/lima/io/h5/wrapper.hpp
View file @
13a23596
...
...
@@ -152,7 +152,31 @@ namespace io
bool
is_open
()
const
{
return
(
bool
)
member
;
}
/// Returns a copy of the dataspace of the dataset
dataspace
space
()
const
{
return
dataspace
(
H5Dget_space
(
member
.
get
()));
}
dataspace
space
()
const
{
return
H5Dget_space
(
member
.
get
());
}
/// Returns the datatype of the dataset
predef_datatype
datatype
()
const
{
return
H5Dget_type
(
member
.
get
());
}
/// Writes data to the dataset
herr_t
write
(
const
void
*
buffer
,
predef_datatype
dtype
,
hid_t
mem_dspace
=
H5S_ALL
,
hid_t
file_dspace
=
H5S_ALL
,
hid_t
dtpl_id
=
H5P_DEFAULT
)
{
return
H5Dwrite
(
member
.
get
(),
dtype
,
mem_dspace
,
file_dspace
,
dtpl_id
,
buffer
);
}
/// Writes chunk to the dataset
herr_t
write_chunk
(
const
void
*
buffer
,
uint32_t
filter_mask
,
hsize_t
offset
[],
size_t
data_size
,
hid_t
dtpl_id
=
H5P_DEFAULT
)
{
return
H5Dwrite_chunk
(
member
.
get
(),
dtpl_id
,
filter_mask
,
offset
,
data_size
,
buffer
);
}
/// Reads from the dataset
herr_t
read
(
void
*
buffer
,
predef_datatype
dtype
,
hid_t
mem_dspace
=
H5S_ALL
,
hid_t
file_dspace
=
H5S_ALL
,
hid_t
dtpl_id
=
H5P_DEFAULT
)
{
return
H5Dread
(
member
.
get
(),
dtype
,
mem_dspace
,
file_dspace
,
dtpl_id
,
buffer
);
}
/// Changes the sizes of a dataset's dimensions.
herr_t
extent
(
const
hsize_t
size
[])
{
return
H5Dset_extent
(
member
.
get
(),
size
);
}
...
...
include/lima/io/h5/wrapper/dataspace.hpp
View file @
13a23596
...
...
@@ -43,6 +43,13 @@ namespace io
return
H5Sget_simple_extent_dims
(
m_hid
.
get
(),
dims
,
maxdims
);
}
/// Selects a hyperslab of a dataspace.
herr_t
select_hyperslab
(
H5S_seloper_t
op
,
const
hsize_t
start
[],
const
hsize_t
stride
[],
const
hsize_t
count
[],
const
hsize_t
block
[])
{
return
H5Sselect_hyperslab
(
m_hid
.
get
(),
op
,
start
,
stride
,
count
,
block
);
}
operator
hid_t
()
const
{
return
m_hid
.
get
();
}
shared_dataspace_hid_t
m_hid
;
...
...
include/lima/io/h5/wrapper/datatype.hpp
View file @
13a23596
...
...
@@ -11,6 +11,8 @@
#include
<hdf5.h>
#include
<lima/exceptions.hpp>
#include
<lima/core/enums.hpp>
#include
<lima/io/h5/wrapper/handle.hpp>
namespace
lima
...
...
@@ -36,6 +38,27 @@ namespace io
template
<
>
struct
deduct
<
float
>
{
static
const
hid_t
type
()
{
return
H5T_NATIVE_FLOAT
;
}
};
template
<
>
struct
deduct
<
double
>
{
static
const
hid_t
type
()
{
return
H5T_NATIVE_DOUBLE
;
}
};
// clang-format on
inline
pixel_enum
pixel_type
(
hid_t
datatype
)
{
// H55_NATIVE_xxxx are not constants (expand to function calls): cannot use switch
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_UINT8
)
>
0
)
return
pixel_enum
::
gray8
;
else
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_INT8
)
>
0
)
return
pixel_enum
::
gray8s
;
else
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_UINT16
)
>
0
)
return
pixel_enum
::
gray16
;
else
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_INT16
)
>
0
)
return
pixel_enum
::
gray16s
;
else
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_UINT32
)
>
0
)
return
pixel_enum
::
gray32
;
else
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_INT32
)
>
0
)
return
pixel_enum
::
gray32s
;
else
if
(
H5Tequal
(
datatype
,
H5T_NATIVE_FLOAT
)
>
0
)
return
pixel_enum
::
gray32f
;
else
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"Unsupported H5 data type"
));
}
}
// namespace detail
struct
predef_datatype
...
...
@@ -49,6 +72,10 @@ namespace io
return
detail
::
deduct
<
std
::
decay_t
<
T
>>::
type
();
}
pixel_enum
pixel_type
()
{
return
detail
::
pixel_type
(
m_hid
);
}
size_t
size
()
{
return
H5Tget_size
(
m_hid
);
}
operator
hid_t
()
const
{
return
m_hid
;
}
private:
...
...
include/lima/io/h5/writer.hpp
View file @
13a23596
...
...
@@ -245,10 +245,10 @@ namespace io
}
// Set dimensions
hsize_t
dims
[]
=
{
(
hsize_t
)
nb_frames
,
m_dimensions
.
y
,
m_dimensions
.
x
};
hsize_t
dims
[]
=
{
hsize_t
(
nb_frames
)
,
m_dimensions
.
y
,
m_dimensions
.
x
};
// Set chunk size
hsize_t
chunk
[]
=
{
(
hsize_t
)
nb_frames_per_chunk
,
m_dimensions
.
y
,
m_dimensions
.
x
};
hsize_t
chunk
[]
=
{
hsize_t
(
nb_frames_per_chunk
)
,
m_dimensions
.
y
,
m_dimensions
.
x
};
res
=
H5Pset_chunk
(
dcpl
,
lengthof
(
chunk
),
chunk
);
// Create dataspace
...
...
@@ -296,14 +296,14 @@ namespace io
size_t
data_size
=
nb_elems
*
elem_size
;
//TODO Make this alignement aware but how?
const
data_type
*
data_ptr
=
&
v
(
0
,
0
);
if
(
elem_size
!=
H5Tget_size
(
m_dtype
))
if
(
elem_size
!=
m_dtype
.
size
(
))
LIMA_THROW_EXCEPTION
(
lima
::
runtime_error
(
"View pixel / H5Tget_size mismatch"
));
std
::
vector
<
unsigned
char
>
comp_buffer
;
switch
(
m_compression
)
{
case
compression_enum
::
none
:
{
herr_t
res
=
H5D
write_chunk
(
m_dset
,
H5P_DEFAULT
,
filter_mask
,
offset
,
data_size
,
data_ptr
);
herr_t
res
=
m_dset
.
write_chunk
(
data_ptr
,
filter_mask
,
offset
,
data_size
);
if
(
res
<
0
)
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"H5Dwrite_chunk error"
)
<<
boost
::
errinfo_errno
(
res
));
break
;
...
...
@@ -328,7 +328,7 @@ namespace io
<<
" (ratio="
<<
((
double
)
data_size
/
(
double
)
comp_size
)
<<
")"
;
// Write chunk
herr_t
res
=
H5D
write_chunk
(
m_dset
,
H5P_DEFAULT
,
filter_mask
,
offset
,
comp_size
,
comp_data
);
herr_t
res
=
m_dset
.
write_chunk
(
comp_data
,
filter_mask
,
offset
,
comp_size
);
if
(
res
<
0
)
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"H5Dwrite_chunk error"
)
<<
boost
::
errinfo_errno
(
res
));
#endif //defined(LIMA_ENABLE_ZLIB)
...
...
@@ -355,7 +355,7 @@ namespace io
<<
" (ratio="
<<
((
double
)
data_size
/
(
double
)
comp_size
)
<<
")"
;
// Write chunk
herr_t
res
=
H5D
write_chunk
(
m_dset
,
H5P_DEFAULT
,
filter_mask
,
offset
,
comp_size
+
12
,
comp_data
);
herr_t
res
=
m_dset
.
write_chunk
(
comp_data
,
filter_mask
,
offset
,
comp_size
+
12
);
if
(
res
<
0
)
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"H5Dwrite_chunk error"
)
<<
boost
::
errinfo_errno
(
res
));
#endif //defined(LIMA_ENABLE_BSHUF_LZ4)
...
...
@@ -377,7 +377,7 @@ namespace io
uint32_t
filter_mask
=
0
;
// Write chunk
herr_t
res
=
H5D
write_chunk
(
m_dset
,
H5P_DEFAULT
,
filter_mask
,
offset
,
comp_size
,
comp_data
);
herr_t
res
=
m_dset
.
write_chunk
(
comp_data
,
filter_mask
,
offset
,
comp_size
);
if
(
res
<
0
)
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"H5Dwrite_chunk error"
)
<<
boost
::
errinfo_errno
(
res
));
}
...
...
@@ -553,10 +553,10 @@ namespace io
// Background
{
// Set dimensions
hsize_t
dims
[]
=
{
m_
nb_frames
,
m_nb_bins
};
hsize_t
dims
[]
=
{
hsize_t
(
nb_frames
)
,
m_nb_bins
};
// Set chunk size
hsize_t
chunk
[]
=
{
(
hsize_t
)
nb_frames_per_chunk
,
m_nb_bins
};
hsize_t
chunk
[]
=
{
hsize_t
(
nb_frames_per_chunk
)
,
m_nb_bins
};
res
=
H5Pset_chunk
(
dcpl
,
lengthof
(
chunk
),
chunk
);
// Create dataspace
...
...
@@ -581,7 +581,7 @@ namespace io
// Frame idx
{
// Set dimensions
hsize_t
dims
[]
=
{
(
hsize_t
)
nb_frames
+
1
};
hsize_t
dims
[]
=
{
hsize_t
(
nb_frames
)
+
1
};
// Create dataspace
auto
file_space
=
dataspace
::
create_simple
(
lengthof
(
dims
),
dims
);
...
...
@@ -637,9 +637,9 @@ namespace io
hsize_t
offset
[]
=
{
frame_idx
};
hsize_t
stride
[]
=
{
1
};
hsize_t
count
[]
=
{
1
};
H5S
select_hyperslab
(
file_space
,
H5S_SELECT_SET
,
offset
,
stride
,
count
,
NULL
);
file_space
.
select_hyperslab
(
H5S_SELECT_SET
,
offset
,
stride
,
count
,
nullptr
);
H5Dwrite
(
m_frame_idx
,
m_index_dtype
,
mem_space
,
file_space
,
H5P_DEFAULT
,
&
val
);
m_frame_idx
.
write
(
&
val
,
m_index_dtype
,
mem_space
,
file_space
);
}
void
write_peak_indices
(
std
::
vector
<
int
>
const
&
peak_indices
)
...
...
@@ -661,10 +661,10 @@ namespace io
hsize_t
offset
[]
=
{
file_dims
[
0
]};
hsize_t
stride
[]
=
{
1
};
hsize_t
count
[]
=
{
peak_indices
.
size
()};
H5S
select_hyperslab
(
file_space
,
H5S_SELECT_SET
,
offset
,
stride
,
count
,
NULL
);
file_space
.
select_hyperslab
(
H5S_SELECT_SET
,
offset
,
stride
,
count
,
nullptr
);
predef_datatype
index_dtype
(
H5T_NATIVE_INT32
);
H5Dwrite
(
m_pixel_idx
,
index_dtype
,
mem_space
,
file_space
,
H5P_DEFAULT
,
&
peak_indices
[
0
]
);
m_pixel_idx
.
write
(
&
peak_indices
[
0
]
,
index_dtype
,
mem_space
,
file_space
);
}
void
write_peak_values
(
std
::
vector
<
float
>
const
&
peak_val
)
...
...
@@ -686,9 +686,9 @@ namespace io
hsize_t
offset
[]
=
{
file_dims
[
0
]};
hsize_t
stride
[]
=
{
1
};
hsize_t
count
[]
=
{
peak_val
.
size
()};
H5S
select_hyperslab
(
file_space
,
H5S_SELECT_SET
,
offset
,
stride
,
count
,
NULL
);
file_space
.
select_hyperslab
(
H5S_SELECT_SET
,
offset
,
stride
,
count
,
nullptr
);
H5Dwrite
(
m_pixel
_val
,
m_peak_value_dtype
,
mem_space
,
file_space
,
H5P_DEFAULT
,
&
peak_val
[
0
]
);
m_pixel_val
.
write
(
&
peak
_val
[
0
]
,
m_peak_value_dtype
,
mem_space
,
file_space
);
}
void
write_frame_peaks
(
hsize_t
frame_idx
,
std
::
vector
<
int
>
const
&
peak_indices
,
...
...
@@ -718,9 +718,9 @@ namespace io
hsize_t
offset
[]
=
{
frame_idx
,
0
};
hsize_t
stride
[]
=
{
1
,
1
};
hsize_t
count
[]
=
{
1
,
m_nb_bins
};
H5S
select_hyperslab
(
file_space
,
H5S_SELECT_SET
,
offset
,
stride
,
count
,
NULL
);
file_space
.
select_hyperslab
(
H5S_SELECT_SET
,
offset
,
stride
,
count
,
nullptr
);
H5D
write
(
m_
background_avg
,
m_bckg_dtype
,
mem_space
,
file_space
,
H5P_DEFAULT
,
&
background_avg
[
0
]
);
m_background_avg
.
write
(
&
background_avg
[
0
]
,
m_bckg_dtype
,
mem_space
,
file_space
);
}
void
write_background_std
(
hsize_t
frame_idx
,
std
::
vector
<
float
>
const
&
background_std
)
...
...
@@ -737,9 +737,9 @@ namespace io
hsize_t
offset
[]
=
{
frame_idx
,
0
};
hsize_t
stride
[]
=
{
1
,
1
};
hsize_t
count
[]
=
{
1
,
m_nb_bins
};
H5S
select_hyperslab
(
file_space
,
H5S_SELECT_SET
,
offset
,
stride
,
count
,
NULL
);
file_space
.
select_hyperslab
(
H5S_SELECT_SET
,
offset
,
stride
,
count
,
nullptr
);
H5D
write
(
m_
background_std
,
m_bckg_dtype
,
mem_space
,
file_space
,
H5P_DEFAULT
,
&
background_std
[
0
]
);
m_background_std
.
write
(
&
background_std
[
0
]
,
m_bckg_dtype
,
mem_space
,
file_space
);
}
protected:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment