Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
LimaGroup
Lima2
Commits
78651737
Commit
78651737
authored
Feb 24, 2022
by
Alejandro Homs Puron
Browse files
[IO] Use h5::saving_params to build writer/multi & io_hdf5_node
parent
328f746d
Changes
11
Hide whitespace changes
Inline
Side-by-side
bench/io/bench_io_hdf5.cpp
View file @
78651737
...
...
@@ -105,7 +105,9 @@ static void graph_io_hdf5_raw(benchmark::State& state)
const
int
nb_frames
=
10000
;
auto
src
=
input_node
(
g
,
nb_frames
);
lima
::
io
::
h5
::
writer
writer
(
"test_raw.h5"
,
nb_frames
,
1
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
);
lima
::
io
::
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
nb_frames
;
lima
::
io
::
h5
::
writer
writer
(
"test_writer_raw.h5"
,
params
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
);
flow
::
function_node
<
frame_t
>
write_view
(
g
,
flow
::
serial
,
[
&
writer
](
frame_t
in
)
{
writer
.
write_view
(
lima
::
view
(
in
),
in
.
metadata
.
idx
);
});
...
...
@@ -157,8 +159,10 @@ static void graph_io_hdf5_chunk_raw(benchmark::State& state)
}
});
lima
::
io
::
h5
::
writer
writer
(
"test_raw.h5"
,
nb_frames
,
nb_frames_per_chunk
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
);
lima
::
io
::
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
nb_frames
;
params
.
nb_frames_per_chunk
=
nb_frames_per_chunk
;
lima
::
io
::
h5
::
writer
writer
(
"test_writer_raw.h5"
,
params
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
);
flow
::
function_node
<
chunk_ptr_t
>
write_chunk
(
g
,
flow
::
serial
,
[
&
writer
](
chunk_ptr_t
in
)
{
writer
.
write_chunk
(
in
->
data
.
data
(),
in
->
data
.
size
(),
in
->
id
);
});
...
...
@@ -210,8 +214,10 @@ static void graph_io_hdf5_comp(benchmark::State& state)
return
res
;
});
lima
::
io
::
h5
::
writer
writer
(
"test_comp.h5"
,
nb_frames
,
1
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
,
lima
::
io
::
h5
::
compression_enum
::
bshuf_lz4
);
lima
::
io
::
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
nb_frames
;
params
.
compression
=
lima
::
io
::
h5
::
compression_enum
::
bshuf_lz4
;
lima
::
io
::
h5
::
writer
writer
(
"test_writer_raw.h5"
,
params
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
);
flow
::
function_node
<
chunk_ptr_t
>
write_chunk
(
g
,
flow
::
serial
,
[
&
writer
](
chunk_ptr_t
in
)
{
writer
.
write_chunk
(
in
->
data
.
data
(),
in
->
data
.
size
(),
in
->
id
);
});
...
...
@@ -290,8 +296,11 @@ static void graph_io_hdf5_chunk_comp(benchmark::State& state)
return
res
;
});
lima
::
io
::
h5
::
writer
writer
(
"test_chunk_comp.h5"
,
nb_frames
,
nb_frames_per_chunk
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
,
lima
::
io
::
h5
::
compression_enum
::
bshuf_lz4
);
lima
::
io
::
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
nb_frames
;
params
.
nb_frames_per_chunk
=
nb_frames_per_chunk
;
params
.
compression
=
lima
::
io
::
h5
::
compression_enum
::
bshuf_lz4
;
lima
::
io
::
h5
::
writer
writer
(
"test_writer_raw.h5"
,
params
,
dimensions_t
{
512
,
512
},
lima
::
pixel_enum
::
gray8
);
flow
::
function_node
<
chunk_ptr_t
>
write_chunk
(
g
,
flow
::
serial
,
[
&
writer
,
&
nb_frames_per_chunk
](
chunk_ptr_t
in
)
{
writer
.
write_chunk
(
in
->
data
.
data
(),
in
->
data
.
size
(),
in
->
id
);
...
...
include/lima/io/file_exists_policy.hpp
View file @
78651737
...
...
@@ -22,7 +22,8 @@ namespace io
/// Policy defining the action when the writer finds an existing file
struct
write_file_exists_policy
{
write_file_exists_policy
(
file_exists_policy_enum
file_exists_policy
)
:
m_file_exists_policy
(
file_exists_policy
)
template
<
typename
Params
>
write_file_exists_policy
(
Params
const
&
params
)
:
m_file_exists_policy
(
params
.
file_exists_policy
)
{
}
...
...
include/lima/io/h5/writer.hpp
View file @
78651737
...
...
@@ -36,6 +36,7 @@
#include
<lima/io/file_exists_policy.hpp>
#include
<lima/io/h5/enums.hpp>
#include
<lima/io/h5/nexus.hpp>
#include
<lima/io/h5/params.hpp>
namespace
boost
{
...
...
@@ -126,31 +127,31 @@ namespace io
{
public:
using
dimensions_t
=
lima
::
point
<
hsize_t
>
;
using
params_t
=
saving_params
;
using
file_exists_policy_t
=
write_file_exists_policy
;
writer
(
std
::
filesystem
::
path
const
&
filename
,
// Common
int
nb_frames
,
int
nb_frames_per_chunk
,
// Data
point_t
const
&
dim
,
pixel_enum
const
&
px
,
// HDF5
compression_enum
comp
=
compression_enum
::
none
,
//
h5
::
path
entry_name
=
"/entry_0000"
,
// Nexus
h5
::
path
detector_name
=
"Simulator"
,
//
double
exposure_time
=
0.1
,
//
double
latency_time
=
1.e-8
,
//
std
::
string
acq_mode
=
"Single"
,
//
std
::
string
trigger_mode
=
"IntTrig"
,
//
std
::
string
pixel_type
=
"Bpp32"
,
// Detector
int
image_xsize
=
1024
,
//
int
image_ysize
=
1024
,
//
std
::
string
model
=
"Generator"
,
//
std
::
string
name
=
"Simulator"
,
//
std
::
string
type
=
"Simulator"
,
//
double
pixel_xsize
=
1e-6
,
//
double
pixel_ysize
=
1e-6
//
writer
(
std
::
filesystem
::
path
const
&
filename
,
// Common
saving_params
const
&
params
,
// Saving params
point_t
const
&
dim
,
pixel_enum
const
&
px
,
// HDF5
h5
::
path
entry_name
=
"/entry_0000"
,
// Nexus
h5
::
path
detector_name
=
"Simulator"
,
//
double
exposure_time
=
0.1
,
//
double
latency_time
=
1.e-8
,
//
std
::
string
acq_mode
=
"Single"
,
//
std
::
string
trigger_mode
=
"IntTrig"
,
//
std
::
string
pixel_type
=
"Bpp32"
,
// Detector
int
image_xsize
=
1024
,
//
int
image_ysize
=
1024
,
//
std
::
string
model
=
"Generator"
,
//
std
::
string
name
=
"Simulator"
,
//
std
::
string
type
=
"Simulator"
,
//
double
pixel_xsize
=
1e-6
,
//
double
pixel_ysize
=
1e-6
//
)
:
m_dimensions
(
cast_dimensions
<
dimensions_t
>
(
dim
)),
m_dtype
(
h5
::
datatype
(
px
)),
m_nb_frames
(
nb_frames
),
m_compression
(
comp
)
m_nb_frames
(
params
.
nb_frames
_per_file
),
m_compression
(
params
.
compression
)
{
using
namespace
detail
;
using
namespace
std
::
string_literals
;
...
...
@@ -180,6 +181,9 @@ namespace io
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"H5Fcreate error"
)
<<
boost
::
errinfo_filesystem_path
(
filename
));
auto
nb_frames
=
params
.
nb_frames_per_file
;
auto
nb_frames_per_chunk
=
params
.
nb_frames_per_chunk
;
// Create NEXUS entry
nx
::
group
entry
=
f
.
create_entry
(
entry_name
,
detector_name
,
nb_frames
,
exposure_time
,
latency_time
,
acq_mode
,
trigger_mode
,
pixel_type
,
image_xsize
,
image_ysize
,
model
,
...
...
@@ -397,37 +401,35 @@ namespace io
{
public:
using
dimensions_t
=
lima
::
point
<
hsize_t
>
;
using
params_t
=
saving_params
;
using
file_exists_policy_t
=
write_file_exists_policy
;
writer_sparse
(
std
::
filesystem
::
path
const
&
filename
,
// Common
int
nb_frames
,
int
nb_frames_per_chunk
,
//
int
nb_bins
,
// Data
point_t
const
&
dim
,
// HDF5
lima
::
frame
radius1d
,
//
lima
::
frame
radius2d_mask
,
//
compression_enum
comp
=
compression_enum
::
none
,
//
h5
::
path
entry_name
=
"/entry_0000"
,
// Nexus
h5
::
path
detector_name
=
"Simulator"
,
//
double
exposure_time
=
0.1
,
//
double
latency_time
=
1.e-8
,
//
std
::
string
acq_mode
=
"Single"
,
//
std
::
string
trigger_mode
=
"IntTrig"
,
//
std
::
string
pixel_type
=
"Bpp32"
,
// Detector
int
image_xsize
=
1024
,
//
int
image_ysize
=
1024
,
//
std
::
string
model
=
"Generator"
,
//
std
::
string
name
=
"Simulator"
,
//
std
::
string
type
=
"Simulator"
,
//
double
pixel_xsize
=
1e-6
,
//
double
pixel_ysize
=
1e-6
//
writer_sparse
(
std
::
filesystem
::
path
const
&
filename
,
// Common
saving_params
const
&
params
,
// Saving params
int
nb_bins
,
// Data
point_t
const
&
dim
,
// HDF5
h5
::
path
entry_name
=
"/entry_0000"
,
// Nexus
h5
::
path
detector_name
=
"Simulator"
,
//
double
exposure_time
=
0.1
,
//
double
latency_time
=
1.e-8
,
//
std
::
string
acq_mode
=
"Single"
,
//
std
::
string
trigger_mode
=
"IntTrig"
,
//
std
::
string
pixel_type
=
"Bpp32"
,
// Detector
int
image_xsize
=
1024
,
//
int
image_ysize
=
1024
,
//
std
::
string
model
=
"Generator"
,
//
std
::
string
name
=
"Simulator"
,
//
std
::
string
type
=
"Simulator"
,
//
double
pixel_xsize
=
1e-6
,
//
double
pixel_ysize
=
1e-6
//
)
:
m_dimensions
(
cast_dimensions
<
dimensions_t
>
(
dim
)),
m_bckg_dtype
(
H5T_NATIVE_FLOAT
),
m_peak_value_dtype
(
H5T_NATIVE_FLOAT
),
m_index_dtype
(
H5T_NATIVE_UINT32
),
m_nb_frames
(
nb_frames
),
m_nb_frames
(
params
.
nb_frames
_per_file
),
m_nb_bins
(
nb_bins
),
m_compression
(
comp
)
m_compression
(
params
.
compression
)
{
using
namespace
detail
;
using
namespace
std
::
string_literals
;
...
...
@@ -439,6 +441,9 @@ namespace io
LIMA_THROW_EXCEPTION
(
lima
::
hdf5_error
(
"H5Fcreate error"
)
<<
boost
::
errinfo_filesystem_path
(
filename
));
auto
nb_frames
=
params
.
nb_frames_per_file
;
auto
nb_frames_per_chunk
=
params
.
nb_frames_per_chunk
;
// Create NEXUS entry
nx
::
group
entry
=
f
.
create_entry
(
entry_name
,
detector_name
,
nb_frames
,
exposure_time
,
latency_time
,
acq_mode
,
trigger_mode
,
pixel_type
,
image_xsize
,
image_ysize
,
model
,
...
...
include/lima/io/hdf5.hpp
View file @
78651737
...
...
@@ -29,7 +29,8 @@ namespace io
/// Saves the currently instantiated frame to an HDF5 file.
inline
void
h5_write_frame
(
std
::
filesystem
::
path
const
&
filename
,
frame
const
&
frm
)
{
h5
::
writer
m
(
filename
,
1
,
1
,
frm
.
dimensions
(),
frm
.
pixel_type
());
h5
::
saving_params
params
{};
// default constructor: 1 frame/file, 1 frame/chunk, no comp
h5
::
writer
m
(
filename
,
params
,
frm
.
dimensions
(),
frm
.
pixel_type
());
m
.
write_view
(
const_view
(
frm
));
}
...
...
@@ -37,7 +38,9 @@ namespace io
/// Saves the currently instantiated frame to an HDF5 file with zip compression.
inline
void
h5_zip_write_frame
(
std
::
filesystem
::
path
const
&
filename
,
frame
const
&
frm
)
{
h5
::
writer
m
(
filename
,
1
,
1
,
frm
.
dimensions
(),
frm
.
pixel_type
(),
h5
::
compression_enum
::
zip
);
h5
::
saving_params
params
{};
// default constructor: 1 frame/file, 1 frame/chunk
params
.
compression
=
h5
::
compression_enum
::
zip
;
h5
::
writer
m
(
filename
,
params
,
frm
.
dimensions
(),
frm
.
pixel_type
());
m
.
write_view
(
const_view
(
frm
));
}
...
...
@@ -45,7 +48,9 @@ namespace io
/// Saves the currently instantiated frame to an HDF5 file with bshuf/LZ4 compression.
inline
void
h5_bshuf_lz4_write_frame
(
std
::
filesystem
::
path
const
&
filename
,
frame
const
&
frm
)
{
h5
::
writer
m
(
filename
,
1
,
1
,
frm
.
dimensions
(),
frm
.
pixel_type
(),
h5
::
compression_enum
::
bshuf_lz4
);
h5
::
saving_params
params
{};
// default constructor: 1 frame/file, 1 frame/chunk
params
.
compression
=
h5
::
compression_enum
::
bshuf_lz4
;
h5
::
writer
m
(
filename
,
params
,
frm
.
dimensions
(),
frm
.
pixel_type
());
m
.
write_view
(
const_view
(
frm
));
}
...
...
include/lima/io/multi.hpp
View file @
78651737
...
...
@@ -24,8 +24,7 @@
#include
<lima/exceptions.hpp>
#include
<lima/logging.hpp>
#include
<lima/core/frame.hpp>
#include
<lima/io/const.hpp>
#include
<lima/io/enums.hpp>
#include
<lima/io/params.hpp>
namespace
lima
{
...
...
@@ -47,36 +46,29 @@ namespace io
public:
using
driver_t
=
Driver
;
using
params_t
=
typename
driver_t
::
params_t
;
using
file_exists_policy_t
=
typename
driver_t
::
file_exists_policy_t
;
/// Construct a multi file driver
/// \tparams Args Parameters that are forwarded to the underlying driver
template
<
typename
...
Args
>
multi
(
std
::
filesystem
::
path
base_path
,
std
::
string
filename_format
,
std
::
string
filename_prefix
,
std
::
string
filename_suffix
,
int
start_number
,
int
rank
,
file_exists_policy_enum
file_exists_policy
,
int
nb_frames
,
int
nb_frames_per_file
,
int
nb_frames_per_chunk
,
Args
...
args
)
:
m_nb_frames
(
nb_frames
),
m_nb_frames_per_file
(
nb_frames_per_file
),
m_nb_frames_per_chunk
(
nb_frames_per_chunk
),
m_start_number
(
start_number
),
m_rank
(
rank
),
m_base_path
(
base_path
),
m_filename_format
(
filename_format
),
m_filename_prefix
(
filename_prefix
),
m_filename_suffix
(
filename_suffix
),
m_file_exists_policy
(
file_exists_policy
)
multi
(
params_t
const
&
params
,
int
rank
,
int
nb_frames
,
Args
...
args
)
:
m_params
(
params
),
m_rank
(
rank
),
m_file_exists_policy
(
params
),
m_nb_frames
(
nb_frames
)
{
if
(
m_params
.
nb_frames_per_file
<
1
)
LIMA_THROW_EXCEPTION
(
lima
::
invalid_argument
(
"Invalid nb_frames_per_file"
));
auto
int_ceiling
=
[](
int
x
,
int
y
)
{
return
1
+
((
x
-
1
)
/
y
);
};
int
number_of_files
=
int_ceiling
(
m_nb_frames
,
m_nb_frames_per_file
);
int
number_of_files
=
int_ceiling
(
m_nb_frames
,
m_
params
.
nb_frames_per_file
);
LIMA_LOG
(
trace
,
io
)
<<
"Expecting "
<<
number_of_files
<<
" files"
;
// Reserve drivers
m_drivers
.
resize
(
number_of_files
);
// Create the factory (bind the driver specific arguments)
m_driver_factory
=
[
nb_frames_per_file
,
nb_frames_per_chunk
,
args
...](
std
::
string
const
&
filename
)
{
return
driver_t
{
filename
,
nb_frames_per_file
,
nb_frames_per_chunk
,
args
...};
m_driver_factory
=
[
params
,
args
...](
std
::
string
const
&
filename
)
{
return
driver_t
{
filename
,
params
,
args
...};
};
// Create the initial driver
...
...
@@ -88,7 +80,7 @@ namespace io
void
apply
(
int
frame_idx
,
int
nb_frames
,
Callable
func
)
{
// Check if the driver is available
auto
driver_idx
=
frame_idx
/
m_nb_frames_per_file
;
auto
driver_idx
=
frame_idx
/
m_
params
.
nb_frames_per_file
;
if
(
!
is_driver_available
(
driver_idx
))
// Create the next driver
open
(
driver_idx
);
...
...
@@ -97,13 +89,13 @@ namespace io
auto
&
dcounter
=
m_drivers
[
driver_idx
];
// Write the data
func
(
dcounter
->
driver
,
frame_idx
%
m_nb_frames_per_file
);
func
(
dcounter
->
driver
,
frame_idx
%
m_
params
.
nb_frames_per_file
);
// Increment counter
dcounter
->
nb_frames_xferred
+=
nb_frames
;
// Release the driver if we are done with it
if
(
dcounter
->
nb_frames_xferred
>=
m_nb_frames_per_file
)
if
(
dcounter
->
nb_frames_xferred
>=
m_
params
.
nb_frames_per_file
)
dcounter
.
reset
();
}
...
...
@@ -121,7 +113,7 @@ namespace io
/// \param chunk_idx is the frame idx of the first frame in the chunk
void
write_chunk
(
const
void
*
chunk_data
,
std
::
size_t
chunk_size
,
int
chunk_idx
=
0
)
{
apply
(
chunk_idx
,
m_nb_frames_per_chunk
,
[
&
](
auto
&
writer
,
int
frame_idx
)
{
apply
(
chunk_idx
,
m_
params
.
nb_frames_per_chunk
,
[
&
](
auto
&
writer
,
int
frame_idx
)
{
// Write the data
writer
.
write_chunk
(
chunk_data
,
chunk_size
,
frame_idx
);
});
...
...
@@ -132,7 +124,7 @@ namespace io
void
close
(
int
frame_idx
)
{
// Check if the driver is available
auto
driver_idx
=
frame_idx
/
m_nb_frames_per_file
;
auto
driver_idx
=
frame_idx
/
m_
params
.
nb_frames_per_file
;
if
(
is_driver_available
(
driver_idx
))
{
LIMA_LOG
(
trace
,
io
)
<<
"Closing I/O driver "
<<
driver_idx
;
// Close driver
...
...
@@ -142,16 +134,10 @@ namespace io
protected:
// Settings
int
m_nb_frames
;
//!< The number of frames expected to be read/saved in the file
int
m_nb_frames_per_file
;
//!< The number of frames per file
int
m_nb_frames_per_chunk
;
//!< The number of frames per chunk
int
m_start_number
;
//!< The start number of the
params_t
m_params
;
//!< The driver parameters
int
m_rank
;
//!< The MPI rank of the process
std
::
filesystem
::
path
m_base_path
;
//!< The directory where to save the files
std
::
string
m_filename_format
;
//!< The format of the
std
::
string
m_filename_prefix
;
//!< The file prefix
std
::
string
m_filename_suffix
;
//!< The file suffix
file_exists_policy_t
m_file_exists_policy
;
//!< Behavior when the file already exists
int
m_nb_frames
;
//!< The number of frames expected to be read/saved in the file
file_exists_policy_t
m_file_exists_policy
;
//!< The policy verifying before opening the file
private:
std
::
vector
<
std
::
shared_ptr
<
driver_counter
>>
m_drivers
;
// Vector elements must be copy-constructible
...
...
@@ -162,8 +148,9 @@ namespace io
{
using
namespace
fmt
::
literals
;
return
fmt
::
format
(
m_filename_format
,
"filename_prefix"
_a
=
m_filename_prefix
,
"rank"
_a
=
m_rank
,
"file_number"
_a
=
m_start_number
+
driver_idx
,
"filename_suffix"
_a
=
m_filename_suffix
);
return
fmt
::
format
(
m_params
.
filename_format
,
"filename_prefix"
_a
=
m_params
.
filename_prefix
,
"rank"
_a
=
m_rank
,
"file_number"
_a
=
m_params
.
start_number
+
driver_idx
,
"filename_suffix"
_a
=
m_params
.
filename_suffix
);
}
bool
is_driver_available
(
std
::
size_t
driver_idx
)
{
return
(
bool
)
m_drivers
[
driver_idx
];
}
...
...
@@ -174,7 +161,7 @@ namespace io
if
(
driver_idx
>=
m_drivers
.
size
())
LIMA_THROW_EXCEPTION
(
io_error
(
"Unexpected I/O driver index"
));
std
::
filesystem
::
path
filepath
=
m_base_path
/
filename
(
driver_idx
);
std
::
filesystem
::
path
filepath
=
m_
params
.
base_path
/
filename
(
driver_idx
);
LIMA_LOG
(
trace
,
io
)
<<
"Opening file "
<<
filepath
;
//Check whether the file exists, etc
...
...
include/lima/io/params.hpp
View file @
78651737
...
...
@@ -11,7 +11,6 @@
#include
<lima/io/const.hpp>
#include
<lima/io/enums.hpp>
#include
<lima/io/h5/enums.hpp>
namespace
lima
{
...
...
include/lima/processing/nodes/io_hdf5.hpp
View file @
78651737
...
...
@@ -197,17 +197,10 @@ namespace processing
struct
write_chunk_body
{
using
writer_t
=
io
::
multi
<
io
::
h5
::
writer
>
;
write_chunk_body
(
std
::
filesystem
::
path
base_path
,
std
::
string
filename_format
,
std
::
string
filename_prefix
,
std
::
string
filename_suffix
,
int
start_number
,
int
rank
,
io
::
file_exists_policy_enum
file_exists_policy
,
int
nb_frames
,
int
nb_frames_per_file
,
int
nb_frames_per_chunk
,
dimensions_t
const
&
dimensions
,
pixel_enum
pixel
,
io
::
h5
::
compression_enum
compression
)
:
m_writer
(
std
::
make_shared
<
writer_t
>
(
base_path
,
filename_format
,
filename_prefix
,
filename_suffix
,
start_number
,
rank
,
file_exists_policy
,
nb_frames
,
nb_frames_per_file
,
nb_frames_per_chunk
,
dimensions
,
pixel
,
compression
)),
m_nb_frames_per_chunk
(
nb_frames_per_chunk
)
write_chunk_body
(
io
::
h5
::
saving_params
const
&
params
,
int
rank
,
int
nb_frames
,
dimensions_t
const
&
dimensions
,
pixel_enum
pixel
)
:
m_writer
(
std
::
make_shared
<
writer_t
>
(
params
,
rank
,
nb_frames
,
dimensions
,
pixel
)),
m_nb_frames_per_chunk
(
params
.
nb_frames_per_chunk
)
{
}
...
...
@@ -250,27 +243,21 @@ namespace processing
};
public:
io_hdf5_node
(
tbb
::
flow
::
graph
&
g
,
std
::
filesystem
::
path
base_path
,
std
::
string
filename_format
,
std
::
string
filename_prefix
,
std
::
string
filename_suffix
,
int
start_number
,
int
rank
,
io
::
file_exists_policy_enum
file_exists_policy
,
int
nb_frames
,
int
nb_frames_per_file
,
int
nb_frames_per_chunk
,
dimensions_t
const
&
dimensions
,
pixel_enum
px
,
io
::
h5
::
compression_enum
comp
)
:
io_hdf5_node
(
tbb
::
flow
::
graph
&
g
,
io
::
h5
::
saving_params
const
&
params
,
int
rank
,
int
nb_frames
,
dimensions_t
const
&
dimensions
,
pixel_enum
px
)
:
// Contract
boost
::
contract
::
constructor_precondition
<
io_hdf5_node
<
Frame
>>
([
&
]
{
BOOST_CONTRACT_ASSERT
(
start_number
>=
0
);
BOOST_CONTRACT_ASSERT
(
params
.
start_number
>=
0
);
BOOST_CONTRACT_ASSERT
(
params
.
nb_frames_per_file
>
0
);
BOOST_CONTRACT_ASSERT
(
params
.
nb_frames_per_chunk
>
0
);
BOOST_CONTRACT_ASSERT
(
nb_frames
>
0
);
BOOST_CONTRACT_ASSERT
(
nb_frames_per_file
>
0
);
BOOST_CONTRACT_ASSERT
(
nb_frames_per_chunk
>
0
);
}),
parent_t
(
g
),
aggregate_node
(
g
,
tbb
::
flow
::
serial
,
aggregate_body
(
nb_frames_per_chunk
)),
write_chunk_node
(
g
,
tbb
::
flow
::
serial
,
write_chunk_body
(
base_path
,
filename_format
,
filename_prefix
,
filename_suffix
,
start_number
,
rank
,
file_exists_policy
,
nb_frames
,
nb_frames_per_file
,
nb_frames_per_chunk
,
dimensions
,
px
,
comp
))
aggregate_node
(
g
,
tbb
::
flow
::
serial
,
aggregate_body
(
params
.
nb_frames_per_chunk
)),
write_chunk_node
(
g
,
tbb
::
flow
::
serial
,
write_chunk_body
(
params
,
rank
,
nb_frames
,
dimensions
,
px
))
{
// Construct compression node
switch
(
comp
)
{
switch
(
params
.
compression
)
{
case
io
::
h5
::
compression_enum
::
none
:
comp_node
.
emplace
(
g
,
tbb
::
flow
::
unlimited
,
comp_none_body
());
break
;
...
...
@@ -285,7 +272,7 @@ namespace processing
}
// Check for generated case where aggregation is not needed
if
(
nb_frames_per_chunk
==
1
)
{
if
(
params
.
nb_frames_per_chunk
==
1
)
{
tbb
::
flow
::
make_edge
(
*
comp_node
,
write_chunk_node
);
typename
parent_t
::
input_ports_type
input_tuple
(
*
comp_node
);
typename
parent_t
::
output_ports_type
output_tuple
(
write_chunk_node
);
...
...
processings/legacy/src/pipeline_legacy.cpp
View file @
78651737
...
...
@@ -76,12 +76,8 @@ namespace processing::pipelines
frames_buffer
.
emplace
(
m_graph
,
tbb
::
flow
::
unlimited
,
m_frames_buffer
,
m_frames_buffer_mutex
);
// Saving node
io_hdf5
.
emplace
(
m_graph
,
proc_params
.
saving
.
base_path
,
proc_params
.
saving
.
filename_format
,
proc_params
.
saving
.
filename_prefix
,
proc_params
.
saving
.
filename_suffix
,
proc_params
.
saving
.
start_number
,
rank
,
proc_params
.
saving
.
file_exists_policy
,
acq_params
.
xfer
.
time_slice
.
count
,
proc_params
.
saving
.
nb_frames_per_file
,
proc_params
.
saving
.
nb_frames_per_chunk
,
frame_info
.
dimensions
(),
frame_info
.
pixel_type
(),
proc_params
.
saving
.
compression
);
io_hdf5
.
emplace
(
m_graph
,
proc_params
.
saving
,
rank
,
acq_params
.
xfer
.
time_slice
.
count
,
frame_info
.
dimensions
(),
frame_info
.
pixel_type
());
checkpoint_hdf5
.
emplace
(
m_graph
,
tbb
::
flow
::
unlimited
,
m_nb_frames_saved
,
proc_params
.
saving
.
nb_frames_per_chunk
);
...
...
test/io/test_hdf5_writer.cpp
View file @
78651737
...
...
@@ -22,7 +22,9 @@ namespace h5 = lima::io::h5;
BOOST_FIXTURE_TEST_CASE
(
test_hdf5_writer_raw
,
frame_fixture
)
{
h5
::
writer
m
(
"test_writer_raw.h5"
,
3
,
1
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
());
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
3
;
h5
::
writer
m
(
"test_writer_raw.h5"
,
params
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
());
m
.
write_view
(
lima
::
const_view
(
input_frame
),
0
);
m
.
write_view
(
lima
::
const_view
(
input_frame
),
1
);
m
.
write_view
(
lima
::
const_view
(
input_frame
),
2
);
...
...
@@ -30,8 +32,10 @@ BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_raw, frame_fixture)
BOOST_FIXTURE_TEST_CASE
(
test_hdf5_writer_zip
,
frame_fixture
)
{
h5
::
writer
m
(
"test_writer_zip.h5"
,
3
,
1
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
(),
h5
::
compression_enum
::
zip
);
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
3
;
params
.
compression
=
h5
::
compression_enum
::
zip
;
h5
::
writer
m
(
"test_writer_zip.h5"
,
params
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
());
m
.
write_view
(
lima
::
const_view
(
input_frame
),
0
);
m
.
write_view
(
lima
::
const_view
(
input_frame
),
1
);
m
.
write_view
(
lima
::
const_view
(
input_frame
),
2
);
...
...
@@ -39,8 +43,10 @@ BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_zip, frame_fixture)
BOOST_FIXTURE_TEST_CASE
(
test_hdf5_writer_bshuf_lz4
,
frame_fixture
)
{
h5
::
writer
m
(
"test_writer_bshuf_lz4.h5"
,
3
,
1
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
(),
h5
::
compression_enum
::
bshuf_lz4
);
h5
::
saving_params
params
;
params
.
nb_frames_per_file
=
3
;
params
.
compression
=
h5
::
compression_enum
::
bshuf_lz4
;
h5
::
writer
m
(
"test_writer_bshuf_lz4.h5"
,
params
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
());
m
.
write_view
(
lima
::
const_view
(
input_frame
),
0
);
m
.
write_view
(
lima
::
const_view
(
input_frame
),
1
);
m
.
write_view
(
lima
::
const_view
(
input_frame
),
2
);
...
...
test/io/test_multi.cpp
View file @
78651737
...
...
@@ -29,13 +29,13 @@ struct frame_view
struct
writer
{
using
params_t
=
io
::
saving_params
;
using
file_exists_policy_t
=
io
::
write_file_exists_policy
;
inline
static
const
std
::
size_t
max_nb_frames_per_file
=
1000
;
template
<
typename
String
,
typename
...
Settings
>
writer
(
String
const
&
filepath
,
int
nb_frames
,
int
nb_frames_per_chunk
=
1
,
Settings
const
&
...
settings
)
:
m_filepath
(
filepath
)
writer
(
String
const
&
filepath
,
params_t
const
&
params
,
Settings
const
&
...
settings
)
:
m_filepath
(
filepath
)
{
std
::
cout
<<
"opening file "
<<
m_filepath
<<
std
::
endl
;
}
...
...
@@ -56,7 +56,7 @@ struct writer
}
// namespace mock
BOOST_AUTO_TEST_CASE
(
test_task_io_multi_frame_per_frame
)
BOOST_AUTO_TEST_CASE
(
test_task_io_multi_
writer_
frame_per_frame
)
{
//saving settings{base_path_setting("/tmp"),
// filename_prefix_setting("test"),
...
...
@@ -68,8 +68,10 @@ BOOST_AUTO_TEST_CASE(test_task_io_multi_frame_per_frame)
// compression_setting(compression_enum::none),
// nb_frames_per_file_setting(3)};
io
::
multi
<
mock
::
writer
>
multi
{
"/tmp"
,
io
::
default_filename_format
,
"test"
,
".h5"
,
1
,
0
,
io
::
file_exists_policy_enum
::
overwrite
,
20
,
3
,
1
};
io
::
saving_params
settings
{
"/tmp"
,
io
::
default_filename_format
,
"test"
,
".h5"
,
1
,
io
::
file_exists_policy_enum
::
overwrite
,
3
};
io
::
multi
<
mock
::
writer
>
multi
{
settings
,
0
,
20
};
boost
::
gil
::
gray8_image_t
img
;
...
...
@@ -81,21 +83,12 @@ BOOST_AUTO_TEST_CASE(test_task_io_multi_frame_per_frame)
multi
.
write_view
(
mock
::
frame_view
{
19
});
}
BOOST_AUTO_TEST_CASE
(
test_task_io_multi_sequence
)
BOOST_AUTO_TEST_CASE
(
test_task_io_multi_
writer_
sequence
)
{
//saving settings{base_path_setting("/tmp"),
// filename_prefix_setting("test"),
// filename_suffix_setting(".h5"),
// start_number_setting(1),
// file_exists_policy_setting(file_exists_policy_enum::overwrite),
// nb_frames_setting(20),
// nb_frames_per_chunk_setting(1),
// compression_setting(compression_enum::none),
// nb_frames_per_file_setting(3)};
io
::
saving_params
settings
{
"/tmp"
,
io
::
default_filename_format
,
"test"
,
".h5"
,
1
,
io
::
file_exists_policy_enum
::
overwrite
,
100
};
io
::
multi
<
mock
::
writer
>
multi
{
"/tmp"
,
io
::
default_filename_format
,
"test"
,
".h5"
,
1
,
0
,
io
::
file_exists_policy_enum
::
overwrite
,
1000
,
100
,
10
};
io
::
multi
<
mock
::
writer
>
multi
{
settings
,
0
,
1000
};
boost
::
gil
::
gray8_image_t
img
;
...
...
test/process/nodes/test_io_hdf5_node.cpp
View file @
78651737
...
...
@@ -22,20 +22,19 @@ BOOST_FIXTURE_TEST_CASE(test_io_hdf5_node, frame_fixture)
const
int
nb_frames
=
500
;
auto
src
=
input_node
(
g
,
nb_frames
);
std
::
filesystem
::
path
base_path
=
"."
;
std
::
string
filename_format
=
lima
::
io
::
default_filename_format
;
std
::
string
filename_prefix
=
"test"
;
std
::
string
filename_suffix
=
".h5"
;
int
start_number
=
0
;
lima
::
io
::
h5
::
saving_params
params
{
"."
,
lima
::
io
::
default_filename_format
,
"test"
,
".h5"
,
0
,
lima
::
io
::
file_exists_policy_enum
::
overwrite
,
100
,
10
,
lima
::
io
::
h5
::
compression_enum
::
bshuf_lz4
};
int
rank
=
0
;
lima
::
io
::
file_exists_policy_enum
file_exists
=
lima
::
io
::
file_exists_policy_enum
::
overwrite
;
int
nb_frames_per_file
=
100
;
int
nb_frames_per_chunk
=
10
;
lima
::
io
::
h5
::
compression_enum
comp
=
lima
::
io
::
h5
::
compression_enum
::
bshuf_lz4
;
lima
::
processing
::
io_hdf5_node
<
lima
::
frame
>
saving
(
g
,
base_path
,
filename_format
,
filename_prefix
,
filename_suffix
,
start_number
,
rank
,
file_exists
,
nb_frames
,
nb_frames_per_file
,
nb_frames_per_chunk
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
(),
comp
);
lima
::
processing
::
io_hdf5_node
<
lima
::
frame
>
saving
(
g
,
params
,
rank
,
nb_frames
,
input_frame
.
dimensions
(),
input_frame
.
pixel_type
());