Skip to content
GitLab
Commit 78651737 authored by Alejandro Homs Puron's avatar Alejandro Homs Puron
Browse files

[IO] Use h5::saving_params to build writer/multi & io_hdf5_node

parent 328f746d
Hide whitespace changes
Inline Side-by-side
bench/io/bench_io_hdf5.cpp
View file @ 78651737
......@@ -105,7 +105,9 @@ static void graph_io_hdf5_raw(benchmark::State& state)
const int nb_frames = 10000;
auto src = input_node(g, nb_frames);
lima::io::h5::writer writer("test_raw.h5", nb_frames, 1, dimensions_t{512, 512}, lima::pixel_enum::gray8);
lima::io::h5::saving_params params;
params.nb_frames_per_file = nb_frames;
lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
flow::function_node<frame_t> write_view(
g, flow::serial, [&writer](frame_t in) { writer.write_view(lima::view(in), in.metadata.idx); });
......@@ -157,8 +159,10 @@ static void graph_io_hdf5_chunk_raw(benchmark::State& state)
}
});
lima::io::h5::writer writer("test_raw.h5", nb_frames, nb_frames_per_chunk, dimensions_t{512, 512},
lima::pixel_enum::gray8);
lima::io::h5::saving_params params;
params.nb_frames_per_file = nb_frames;
params.nb_frames_per_chunk = nb_frames_per_chunk;
lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
flow::function_node<chunk_ptr_t> write_chunk(
g, flow::serial, [&writer](chunk_ptr_t in) { writer.write_chunk(in->data.data(), in->data.size(), in->id); });
......@@ -210,8 +214,10 @@ static void graph_io_hdf5_comp(benchmark::State& state)
return res;
});
lima::io::h5::writer writer("test_comp.h5", nb_frames, 1, dimensions_t{512, 512}, lima::pixel_enum::gray8,
lima::io::h5::compression_enum::bshuf_lz4);
lima::io::h5::saving_params params;
params.nb_frames_per_file = nb_frames;
params.compression = lima::io::h5::compression_enum::bshuf_lz4;
lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
flow::function_node<chunk_ptr_t> write_chunk(
g, flow::serial, [&writer](chunk_ptr_t in) { writer.write_chunk(in->data.data(), in->data.size(), in->id); });
......@@ -290,8 +296,11 @@ static void graph_io_hdf5_chunk_comp(benchmark::State& state)
return res;
});
lima::io::h5::writer writer("test_chunk_comp.h5", nb_frames, nb_frames_per_chunk, dimensions_t{512, 512},
lima::pixel_enum::gray8, lima::io::h5::compression_enum::bshuf_lz4);
lima::io::h5::saving_params params;
params.nb_frames_per_file = nb_frames;
params.nb_frames_per_chunk = nb_frames_per_chunk;
params.compression = lima::io::h5::compression_enum::bshuf_lz4;
lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
flow::function_node<chunk_ptr_t> write_chunk(g, flow::serial, [&writer, &nb_frames_per_chunk](chunk_ptr_t in) {
writer.write_chunk(in->data.data(), in->data.size(), in->id);
......
include/lima/io/file_exists_policy.hpp
View file @ 78651737
......@@ -22,7 +22,8 @@ namespace io
/// Policy defining the action when the writer finds an existing file
struct write_file_exists_policy
{
write_file_exists_policy(file_exists_policy_enum file_exists_policy) : m_file_exists_policy(file_exists_policy)
template <typename Params>
write_file_exists_policy(Params const& params) : m_file_exists_policy(params.file_exists_policy)
{
}
......
include/lima/io/h5/writer.hpp
View file @ 78651737
......@@ -36,6 +36,7 @@
#include <lima/io/file_exists_policy.hpp>
#include <lima/io/h5/enums.hpp>
#include <lima/io/h5/nexus.hpp>
#include <lima/io/h5/params.hpp>
namespace boost
{
......@@ -126,31 +127,31 @@ namespace io
{
public:
using dimensions_t = lima::point<hsize_t>;
using params_t = saving_params;
using file_exists_policy_t = write_file_exists_policy;
writer(std::filesystem::path const& filename, // Common
int nb_frames, int nb_frames_per_chunk, // Data
point_t const& dim, pixel_enum const& px, // HDF5
compression_enum comp = compression_enum::none, //
h5::path entry_name = "/entry_0000", // Nexus
h5::path detector_name = "Simulator", //
double exposure_time = 0.1, //
double latency_time = 1.e-8, //
std::string acq_mode = "Single", //
std::string trigger_mode = "IntTrig", //
std::string pixel_type = "Bpp32", // Detector
int image_xsize = 1024, //
int image_ysize = 1024, //
std::string model = "Generator", //
std::string name = "Simulator", //
std::string type = "Simulator", //
double pixel_xsize = 1e-6, //
double pixel_ysize = 1e-6 //
writer(std::filesystem::path const& filename, // Common
saving_params const& params, // Saving params
point_t const& dim, pixel_enum const& px, // HDF5
h5::path entry_name = "/entry_0000", // Nexus
h5::path detector_name = "Simulator", //
double exposure_time = 0.1, //
double latency_time = 1.e-8, //
std::string acq_mode = "Single", //
std::string trigger_mode = "IntTrig", //
std::string pixel_type = "Bpp32", // Detector
int image_xsize = 1024, //
int image_ysize = 1024, //
std::string model = "Generator", //
std::string name = "Simulator", //
std::string type = "Simulator", //
double pixel_xsize = 1e-6, //
double pixel_ysize = 1e-6 //
) :
m_dimensions(cast_dimensions<dimensions_t>(dim)),
m_dtype(h5::datatype(px)),
m_nb_frames(nb_frames),
m_compression(comp)
m_nb_frames(params.nb_frames_per_file),
m_compression(params.compression)
{
using namespace detail;
using namespace std::string_literals;
......@@ -180,6 +181,9 @@ namespace io
LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Fcreate error")
<< boost::errinfo_filesystem_path(filename));
auto nb_frames = params.nb_frames_per_file;
auto nb_frames_per_chunk = params.nb_frames_per_chunk;
// Create NEXUS entry
nx::group entry = f.create_entry(entry_name, detector_name, nb_frames, exposure_time, latency_time,
acq_mode, trigger_mode, pixel_type, image_xsize, image_ysize, model,
......@@ -397,37 +401,35 @@ namespace io
{
public:
using dimensions_t = lima::point<hsize_t>;
using params_t = saving_params;
using file_exists_policy_t = write_file_exists_policy;
writer_sparse(std::filesystem::path const& filename, // Common
int nb_frames, int nb_frames_per_chunk, //
int nb_bins, // Data
point_t const& dim, // HDF5
lima::frame radius1d, //
lima::frame radius2d_mask, //
compression_enum comp = compression_enum::none, //
h5::path entry_name = "/entry_0000", // Nexus
h5::path detector_name = "Simulator", //
double exposure_time = 0.1, //
double latency_time = 1.e-8, //
std::string acq_mode = "Single", //
std::string trigger_mode = "IntTrig", //
std::string pixel_type = "Bpp32", // Detector
int image_xsize = 1024, //
int image_ysize = 1024, //
std::string model = "Generator", //
std::string name = "Simulator", //
std::string type = "Simulator", //
double pixel_xsize = 1e-6, //
double pixel_ysize = 1e-6 //
writer_sparse(std::filesystem::path const& filename, // Common
saving_params const& params, // Saving params
int nb_bins, // Data
point_t const& dim, // HDF5
h5::path entry_name = "/entry_0000", // Nexus
h5::path detector_name = "Simulator", //
double exposure_time = 0.1, //
double latency_time = 1.e-8, //
std::string acq_mode = "Single", //
std::string trigger_mode = "IntTrig", //
std::string pixel_type = "Bpp32", // Detector
int image_xsize = 1024, //
int image_ysize = 1024, //
std::string model = "Generator", //
std::string name = "Simulator", //
std::string type = "Simulator", //
double pixel_xsize = 1e-6, //
double pixel_ysize = 1e-6 //
) :
m_dimensions(cast_dimensions<dimensions_t>(dim)),
m_bckg_dtype(H5T_NATIVE_FLOAT),
m_peak_value_dtype(H5T_NATIVE_FLOAT),
m_index_dtype(H5T_NATIVE_UINT32),
m_nb_frames(nb_frames),
m_nb_frames(params.nb_frames_per_file),
m_nb_bins(nb_bins),
m_compression(comp)
m_compression(params.compression)
{
using namespace detail;
using namespace std::string_literals;
......@@ -439,6 +441,9 @@ namespace io
LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Fcreate error")
<< boost::errinfo_filesystem_path(filename));
auto nb_frames = params.nb_frames_per_file;
auto nb_frames_per_chunk = params.nb_frames_per_chunk;
// Create NEXUS entry
nx::group entry = f.create_entry(entry_name, detector_name, nb_frames, exposure_time, latency_time,
acq_mode, trigger_mode, pixel_type, image_xsize, image_ysize, model,
......
include/lima/io/hdf5.hpp
View file @ 78651737
......@@ -29,7 +29,8 @@ namespace io
/// Saves the currently instantiated frame to an HDF5 file.
inline void h5_write_frame(std::filesystem::path const& filename, frame const& frm)
{
h5::writer m(filename, 1, 1, frm.dimensions(), frm.pixel_type());
h5::saving_params params{}; // default constructor: 1 frame/file, 1 frame/chunk, no comp
h5::writer m(filename, params, frm.dimensions(), frm.pixel_type());
m.write_view(const_view(frm));
}
......@@ -37,7 +38,9 @@ namespace io
/// Saves the currently instantiated frame to an HDF5 file with zip compression.
inline void h5_zip_write_frame(std::filesystem::path const& filename, frame const& frm)
{
h5::writer m(filename, 1, 1, frm.dimensions(), frm.pixel_type(), h5::compression_enum::zip);
h5::saving_params params{}; // default constructor: 1 frame/file, 1 frame/chunk
params.compression = h5::compression_enum::zip;
h5::writer m(filename, params, frm.dimensions(), frm.pixel_type());
m.write_view(const_view(frm));
}
......@@ -45,7 +48,9 @@ namespace io
/// Saves the currently instantiated frame to an HDF5 file with bshuf/LZ4 compression.
inline void h5_bshuf_lz4_write_frame(std::filesystem::path const& filename, frame const& frm)
{
h5::writer m(filename, 1, 1, frm.dimensions(), frm.pixel_type(), h5::compression_enum::bshuf_lz4);
h5::saving_params params{}; // default constructor: 1 frame/file, 1 frame/chunk
params.compression = h5::compression_enum::bshuf_lz4;
h5::writer m(filename, params, frm.dimensions(), frm.pixel_type());
m.write_view(const_view(frm));
}
......
include/lima/io/multi.hpp
View file @ 78651737
......@@ -24,8 +24,7 @@
#include <lima/exceptions.hpp>
#include <lima/logging.hpp>
#include <lima/core/frame.hpp>
#include <lima/io/const.hpp>
#include <lima/io/enums.hpp>
#include <lima/io/params.hpp>
namespace lima
{
......@@ -47,36 +46,29 @@ namespace io
public:
using driver_t = Driver;
using params_t = typename driver_t::params_t;
using file_exists_policy_t = typename driver_t::file_exists_policy_t;
/// Construct a multi file driver
/// \tparams Args Parameters that are forwarded to the underlying driver
template <typename... Args>
multi(std::filesystem::path base_path, std::string filename_format, std::string filename_prefix,
std::string filename_suffix, int start_number, int rank, file_exists_policy_enum file_exists_policy,
int nb_frames, int nb_frames_per_file, int nb_frames_per_chunk, Args... args) :
m_nb_frames(nb_frames),
m_nb_frames_per_file(nb_frames_per_file),
m_nb_frames_per_chunk(nb_frames_per_chunk),
m_start_number(start_number),
m_rank(rank),
m_base_path(base_path),
m_filename_format(filename_format),
m_filename_prefix(filename_prefix),
m_filename_suffix(filename_suffix),
m_file_exists_policy(file_exists_policy)
multi(params_t const& params, int rank, int nb_frames, Args... args) :
m_params(params), m_rank(rank), m_file_exists_policy(params), m_nb_frames(nb_frames)
{
if (m_params.nb_frames_per_file < 1)
LIMA_THROW_EXCEPTION(lima::invalid_argument("Invalid nb_frames_per_file"));
auto int_ceiling = [](int x, int y) { return 1 + ((x - 1) / y); };
int number_of_files = int_ceiling(m_nb_frames, m_nb_frames_per_file);
int number_of_files = int_ceiling(m_nb_frames, m_params.nb_frames_per_file);
LIMA_LOG(trace, io) << "Expecting " << number_of_files << " files";
// Reserve drivers
m_drivers.resize(number_of_files);
// Create the factory (bind the driver specific arguments)
m_driver_factory = [nb_frames_per_file, nb_frames_per_chunk, args...](std::string const& filename) {
return driver_t{filename, nb_frames_per_file, nb_frames_per_chunk, args...};
m_driver_factory = [params, args...](std::string const& filename) {
return driver_t{filename, params, args...};
};
// Create the initial driver
......@@ -88,7 +80,7 @@ namespace io
void apply(int frame_idx, int nb_frames, Callable func)
{
// Check if the driver is available
auto driver_idx = frame_idx / m_nb_frames_per_file;
auto driver_idx = frame_idx / m_params.nb_frames_per_file;
if (!is_driver_available(driver_idx))
// Create the next driver
open(driver_idx);
......@@ -97,13 +89,13 @@ namespace io
auto& dcounter = m_drivers[driver_idx];
// Write the data
func(dcounter->driver, frame_idx % m_nb_frames_per_file);
func(dcounter->driver, frame_idx % m_params.nb_frames_per_file);
// Increment counter
dcounter->nb_frames_xferred += nb_frames;
// Release the driver if we are done with it
if (dcounter->nb_frames_xferred >= m_nb_frames_per_file)
if (dcounter->nb_frames_xferred >= m_params.nb_frames_per_file)
dcounter.reset();
}
......@@ -121,7 +113,7 @@ namespace io
/// \param chunk_idx is the frame idx of the first frame in the chunk
void write_chunk(const void* chunk_data, std::size_t chunk_size, int chunk_idx = 0)
{
apply(chunk_idx, m_nb_frames_per_chunk, [&](auto& writer, int frame_idx) {
apply(chunk_idx, m_params.nb_frames_per_chunk, [&](auto& writer, int frame_idx) {
// Write the data
writer.write_chunk(chunk_data, chunk_size, frame_idx);
});
......@@ -132,7 +124,7 @@ namespace io
void close(int frame_idx)
{
// Check if the driver is available
auto driver_idx = frame_idx / m_nb_frames_per_file;
auto driver_idx = frame_idx / m_params.nb_frames_per_file;
if (is_driver_available(driver_idx)) {
LIMA_LOG(trace, io) << "Closing I/O driver " << driver_idx;
// Close driver
......@@ -142,16 +134,10 @@ namespace io
protected:
// Settings
int m_nb_frames; //!< The number of frames expected to be read/saved in the file
int m_nb_frames_per_file; //!< The number of frames per file
int m_nb_frames_per_chunk; //!< The number of frames per chunk
int m_start_number; //!< The start number of the
params_t m_params; //!< The driver parameters
int m_rank; //!< The MPI rank of the process
std::filesystem::path m_base_path; //!< The directory where to save the files
std::string m_filename_format; //!< The format of the
std::string m_filename_prefix; //!< The file prefix
std::string m_filename_suffix; //!< The file suffix
file_exists_policy_t m_file_exists_policy; //!< Behavior when the file already exists
int m_nb_frames; //!< The number of frames expected to be read/saved in the file
file_exists_policy_t m_file_exists_policy; //!< The policy verifying before opening the file
private:
std::vector<std::shared_ptr<driver_counter>> m_drivers; // Vector elements must be copy-constructible
......@@ -162,8 +148,9 @@ namespace io
{
using namespace fmt::literals;
return fmt::format(m_filename_format, "filename_prefix"_a = m_filename_prefix, "rank"_a = m_rank,
"file_number"_a = m_start_number + driver_idx, "filename_suffix"_a = m_filename_suffix);
return fmt::format(m_params.filename_format, "filename_prefix"_a = m_params.filename_prefix,
"rank"_a = m_rank, "file_number"_a = m_params.start_number + driver_idx,
"filename_suffix"_a = m_params.filename_suffix);
}
bool is_driver_available(std::size_t driver_idx) { return (bool) m_drivers[driver_idx]; }
......@@ -174,7 +161,7 @@ namespace io
if (driver_idx >= m_drivers.size())
LIMA_THROW_EXCEPTION(io_error("Unexpected I/O driver index"));
std::filesystem::path filepath = m_base_path / filename(driver_idx);
std::filesystem::path filepath = m_params.base_path / filename(driver_idx);
LIMA_LOG(trace, io) << "Opening file " << filepath;
//Check whether the file exists, etc
......
include/lima/io/params.hpp
View file @ 78651737
......@@ -11,7 +11,6 @@
#include <lima/io/const.hpp>
#include <lima/io/enums.hpp>
#include <lima/io/h5/enums.hpp>
namespace lima
{
......
include/lima/processing/nodes/io_hdf5.hpp
View file @ 78651737
......@@ -197,17 +197,10 @@ namespace processing
struct write_chunk_body
{
using writer_t = io::multi<io::h5::writer>;
write_chunk_body(std::filesystem::path base_path, std::string filename_format, std::string filename_prefix,
std::string filename_suffix, int start_number, int rank,
io::file_exists_policy_enum file_exists_policy, int nb_frames, int nb_frames_per_file,
int nb_frames_per_chunk, dimensions_t const& dimensions, pixel_enum pixel,
io::h5::compression_enum compression) :
m_writer(std::make_shared<writer_t>(base_path, filename_format, filename_prefix, filename_suffix,
start_number, rank, file_exists_policy, nb_frames,
nb_frames_per_file, nb_frames_per_chunk, dimensions, pixel,
compression)),
m_nb_frames_per_chunk(nb_frames_per_chunk)
write_chunk_body(io::h5::saving_params const& params, int rank, int nb_frames,
dimensions_t const& dimensions, pixel_enum pixel) :
m_writer(std::make_shared<writer_t>(params, rank, nb_frames, dimensions, pixel)),
m_nb_frames_per_chunk(params.nb_frames_per_chunk)
{
}
......@@ -250,27 +243,21 @@ namespace processing
};
public:
io_hdf5_node(tbb::flow::graph& g, std::filesystem::path base_path, std::string filename_format,
std::string filename_prefix, std::string filename_suffix, int start_number, int rank,
io::file_exists_policy_enum file_exists_policy, int nb_frames, int nb_frames_per_file,
int nb_frames_per_chunk, dimensions_t const& dimensions, pixel_enum px,
io::h5::compression_enum comp) :
io_hdf5_node(tbb::flow::graph& g, io::h5::saving_params const& params, int rank, int nb_frames,
dimensions_t const& dimensions, pixel_enum px) :
// Contract
boost::contract::constructor_precondition<io_hdf5_node<Frame>>([&] {
BOOST_CONTRACT_ASSERT(start_number >= 0);
BOOST_CONTRACT_ASSERT(params.start_number >= 0);
BOOST_CONTRACT_ASSERT(params.nb_frames_per_file > 0);
BOOST_CONTRACT_ASSERT(params.nb_frames_per_chunk > 0);
BOOST_CONTRACT_ASSERT(nb_frames > 0);
BOOST_CONTRACT_ASSERT(nb_frames_per_file > 0);
BOOST_CONTRACT_ASSERT(nb_frames_per_chunk > 0);
}),
parent_t(g),
aggregate_node(g, tbb::flow::serial, aggregate_body(nb_frames_per_chunk)),
write_chunk_node(g, tbb::flow::serial,
write_chunk_body(base_path, filename_format, filename_prefix, filename_suffix,
start_number, rank, file_exists_policy, nb_frames, nb_frames_per_file,
nb_frames_per_chunk, dimensions, px, comp))
aggregate_node(g, tbb::flow::serial, aggregate_body(params.nb_frames_per_chunk)),
write_chunk_node(g, tbb::flow::serial, write_chunk_body(params, rank, nb_frames, dimensions, px))
{
// Construct compression node
switch (comp) {
switch (params.compression) {
case io::h5::compression_enum::none:
comp_node.emplace(g, tbb::flow::unlimited, comp_none_body());
break;
......@@ -285,7 +272,7 @@ namespace processing
}
// Check for generated case where aggregation is not needed
if (nb_frames_per_chunk == 1) {
if (params.nb_frames_per_chunk == 1) {
tbb::flow::make_edge(*comp_node, write_chunk_node);
typename parent_t::input_ports_type input_tuple(*comp_node);
typename parent_t::output_ports_type output_tuple(write_chunk_node);
......
processings/legacy/src/pipeline_legacy.cpp
View file @ 78651737
......@@ -76,12 +76,8 @@ namespace processing::pipelines
frames_buffer.emplace(m_graph, tbb::flow::unlimited, m_frames_buffer, m_frames_buffer_mutex);
// Saving node
io_hdf5.emplace(m_graph, proc_params.saving.base_path, proc_params.saving.filename_format,
proc_params.saving.filename_prefix, proc_params.saving.filename_suffix,
proc_params.saving.start_number, rank, proc_params.saving.file_exists_policy,
acq_params.xfer.time_slice.count, proc_params.saving.nb_frames_per_file,
proc_params.saving.nb_frames_per_chunk, frame_info.dimensions(),
frame_info.pixel_type(), proc_params.saving.compression);
io_hdf5.emplace(m_graph, proc_params.saving, rank, acq_params.xfer.time_slice.count,
frame_info.dimensions(), frame_info.pixel_type());
checkpoint_hdf5.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_saved,
proc_params.saving.nb_frames_per_chunk);
......
test/io/test_hdf5_writer.cpp
View file @ 78651737
......@@ -22,7 +22,9 @@ namespace h5 = lima::io::h5;
BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_raw, frame_fixture)
{
h5::writer m("test_writer_raw.h5", 3, 1, input_frame.dimensions(), input_frame.pixel_type());
h5::saving_params params;
params.nb_frames_per_file = 3;
h5::writer m("test_writer_raw.h5", params, input_frame.dimensions(), input_frame.pixel_type());
m.write_view(lima::const_view(input_frame), 0);
m.write_view(lima::const_view(input_frame), 1);
m.write_view(lima::const_view(input_frame), 2);
......@@ -30,8 +32,10 @@ BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_raw, frame_fixture)
BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_zip, frame_fixture)
{
h5::writer m("test_writer_zip.h5", 3, 1, input_frame.dimensions(), input_frame.pixel_type(),
h5::compression_enum::zip);
h5::saving_params params;
params.nb_frames_per_file = 3;
params.compression = h5::compression_enum::zip;
h5::writer m("test_writer_zip.h5", params, input_frame.dimensions(), input_frame.pixel_type());
m.write_view(lima::const_view(input_frame), 0);
m.write_view(lima::const_view(input_frame), 1);
m.write_view(lima::const_view(input_frame), 2);
......@@ -39,8 +43,10 @@ BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_zip, frame_fixture)
BOOST_FIXTURE_TEST_CASE(test_hdf5_writer_bshuf_lz4, frame_fixture)
{
h5::writer m("test_writer_bshuf_lz4.h5", 3, 1, input_frame.dimensions(), input_frame.pixel_type(),
h5::compression_enum::bshuf_lz4);
h5::saving_params params;
params.nb_frames_per_file = 3;
params.compression = h5::compression_enum::bshuf_lz4;
h5::writer m("test_writer_bshuf_lz4.h5", params, input_frame.dimensions(), input_frame.pixel_type());
m.write_view(lima::const_view(input_frame), 0);
m.write_view(lima::const_view(input_frame), 1);
m.write_view(lima::const_view(input_frame), 2);
......
test/io/test_multi.cpp
View file @ 78651737
......@@ -29,13 +29,13 @@ struct frame_view
struct writer
{
using params_t = io::saving_params;
using file_exists_policy_t = io::write_file_exists_policy;
inline static const std::size_t max_nb_frames_per_file = 1000;
template <typename String, typename... Settings>
writer(String const& filepath, int nb_frames, int nb_frames_per_chunk = 1, Settings const&... settings) :
m_filepath(filepath)
writer(String const& filepath, params_t const& params, Settings const&... settings) : m_filepath(filepath)
{
std::cout << "opening file " << m_filepath << std::endl;
}
......@@ -56,7 +56,7 @@ struct writer
} // namespace mock
BOOST_AUTO_TEST_CASE(test_task_io_multi_frame_per_frame)
BOOST_AUTO_TEST_CASE(test_task_io_multi_writer_frame_per_frame)
{
//saving settings{base_path_setting("/tmp"),
// filename_prefix_setting("test"),
......@@ -68,8 +68,10 @@ BOOST_AUTO_TEST_CASE(test_task_io_multi_frame_per_frame)
// compression_setting(compression_enum::none),
// nb_frames_per_file_setting(3)};
io::multi<mock::writer> multi{
"/tmp", io::default_filename_format, "test", ".h5", 1, 0, io::file_exists_policy_enum::overwrite, 20, 3, 1};
io::saving_params settings{
"/tmp", io::default_filename_format, "test", ".h5", 1, io::file_exists_policy_enum::overwrite, 3};
io::multi<mock::writer> multi{settings, 0, 20};
boost::gil::gray8_image_t img;
......@@ -81,21 +83,12 @@ BOOST_AUTO_TEST_CASE(test_task_io_multi_frame_per_frame)
multi.write_view(mock::frame_view{19});
}
BOOST_AUTO_TEST_CASE(test_task_io_multi_sequence)
BOOST_AUTO_TEST_CASE(test_task_io_multi_writer_sequence)
{
//saving settings{base_path_setting("/tmp"),
// filename_prefix_setting("test"),
// filename_suffix_setting(".h5"),
// start_number_setting(1),
// file_exists_policy_setting(file_exists_policy_enum::overwrite),
// nb_frames_setting(20),
// nb_frames_per_chunk_setting(1),
// compression_setting(compression_enum::none),
// nb_frames_per_file_setting(3)};
io::saving_params settings{
"/tmp", io::default_filename_format, "test", ".h5", 1, io::file_exists_policy_enum::overwrite, 100};
io::multi<mock::writer> multi{
"/tmp", io::default_filename_format, "test", ".h5", 1, 0, io::file_exists_policy_enum::overwrite, 1000, 100,
10};
io::multi<mock::writer> multi{settings, 0, 1000};
boost::gil::gray8_image_t img;
......
test/process/nodes/test_io_hdf5_node.cpp
View file @ 78651737
......@@ -22,20 +22,19 @@ BOOST_FIXTURE_TEST_CASE(test_io_hdf5_node, frame_fixture)
const int nb_frames = 500;
auto src = input_node(g, nb_frames);
std::filesystem::path base_path = ".";
std::string filename_format = lima::io::default_filename_format;
std::string filename_prefix = "test";
std::string filename_suffix = ".h5";
int start_number = 0;
lima::io::h5::saving_params params{".",
lima::io::default_filename_format,
"test",
".h5",
0,
lima::io::file_exists_policy_enum::overwrite,
100,
10,
lima::io::h5::compression_enum::bshuf_lz4};
int rank = 0;
lima::io::file_exists_policy_enum file_exists = lima::io::file_exists_policy_enum::overwrite;
int nb_frames_per_file = 100;
int nb_frames_per_chunk = 10;
lima::io::h5::compression_enum comp = lima::io::h5::compression_enum::bshuf_lz4;
lima::processing::io_hdf5_node<lima::frame> saving(
g, base_path, filename_format, filename_prefix, filename_suffix, start_number, rank, file_exists, nb_frames,
nb_frames_per_file, nb_frames_per_chunk, input_frame.dimensions(), input_frame.pixel_type(), comp);
lima::processing::io_hdf5_node<lima::frame> saving(g, params, rank, nb_frames, input_frame.dimensions(),
input_frame.pixel_type());