Commit 9e1c9852 authored by Alejandro Homs Puron's avatar Alejandro Homs Puron
Browse files

[IO] Add H5 reader

parent 1685d941
......@@ -37,5 +37,20 @@ namespace io
file_exists_policy_enum m_file_exists_policy;
};
/// Policy defining the action when the reader does not find a file
struct read_file_exists_policy
{
template <typename Params>
read_file_exists_policy(Params const& /*params*/)
{
}
void operator()(std::filesystem::path filepath)
{
if (!std::filesystem::exists(filepath))
LIMA_THROW_EXCEPTION(io_error("File does not exist") << boost::errinfo_file_name(filepath.string()));
}
};
} //namespace io
} //namespace lima
......@@ -19,6 +19,8 @@ namespace io
{
BOOST_DESCRIBE_ENUM(compression_enum, none, zip, bshuf_lz4)
BOOST_DESCRIBE_ENUM(file_type_enum, hdf5, nexus)
using boost::describe::operator<<;
using boost::describe::operator>>;
......
......@@ -19,6 +19,13 @@ namespace io
bshuf_lz4 //!< bitshuffle filter and lz4 compression
};
/// Used by file reader
enum class file_type_enum
{
hdf5, //!< Bare HDF5 file
nexus //!< Nexus HDF5 file
};
} //namespace h5
} //namespace io
} //namespace lima
......@@ -19,7 +19,7 @@ namespace io::h5
BOOST_DESCRIBE_STRUCT(saving_params, (lima::io::saving_params), (nb_frames_per_chunk, compression))
// clang-format off
BOOST_ANNOTATE_MEMBER(h5::saving_params, nb_frames_per_chunk,
BOOST_ANNOTATE_MEMBER(saving_params, nb_frames_per_chunk,
(desc, "nb frames per chunk"),
(doc, "The number of frames per chunk"))
......@@ -28,5 +28,17 @@ namespace io::h5
(doc, "The compression filter [none, zip, bshuf_lz4]"))
// clang-format on
BOOST_DESCRIBE_STRUCT(loading_params, (lima::io::loading_params), (file_type, dataset_path))
// clang-format off
BOOST_ANNOTATE_MEMBER(loading_params, file_type,
(desc, "file type [hdf5, nexus]"),
(doc, "The expected type format to load"))
BOOST_ANNOTATE_MEMBER(loading_params, dataset_path,
(desc, "dataset_path"),
(doc, "The HDF5 path of the dataset"))
// clang-format on
} //namespace io::h5
} //namespace lima
......@@ -19,5 +19,11 @@ namespace io::h5
compression_enum compression = compression_enum::none;
};
struct loading_params : io::loading_params
{
file_type_enum file_type = file_type_enum::nexus;
std::string dataset_path = "/entry_0000/measurement/data"s;
};
} //namespace io::h5
} //namespace lima
// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#pragma once
#include <filesystem>
#include <boost/exception/errinfo_errno.hpp>
#include <boost/exception/errinfo_file_name.hpp>
#include <boost/gil/extension/dynamic_image/dynamic_image_all.hpp>
#include <hdf5.h>
#include <lima/exceptions.hpp>
#include <lima/logging.hpp>
#include <lima/core/frame.hpp>
#include <lima/core/image/view.hpp>
#include <lima/io/file_exists_policy.hpp>
#include <lima/io/h5/params.hpp>
#include <lima/io/h5/nexus.hpp>
namespace boost
{
template <class Tag, class T>
class error_info;
typedef error_info<struct errinfo_filesystem_path_, std::filesystem::path> errinfo_filesystem_path;
typedef error_info<struct errinfo_h5_path_, lima::io::h5::path> errinfo_h5_path;
typedef error_info<struct errinfo_frame_idx_, hsize_t> errinfo_frame_idx;
} // namespace boost
namespace lima
{
namespace io::h5
{
class reader
{
public:
using dimensions_t = lima::point_t;
using params_t = loading_params;
using file_exists_policy_t = read_file_exists_policy;
reader(std::filesystem::path const& filename, params_t const& params = {}) :
m_nb_frames(params.nb_frames_per_file)
{
bool is_nexus = (params.file_type == file_type_enum::nexus);
const char* type_str = is_nexus ? "Nexus" : "HDF5";
LIMA_LOG(trace, io) << "Constructing " << type_str << " reader for file " << filename;
// Open the HDF5 file
const char* nx_class = is_nexus ? "NXroot" : nullptr;
auto f = nx::file::open(filename, nx_class);
if (!f.is_open())
LIMA_THROW_EXCEPTION(lima::hdf5_error("Error opening H5 file")
<< boost::errinfo_filesystem_path(filename));
// Get Dataset
auto&& dataset_path = params.dataset_path;
m_dset = dataset::open(f, dataset_path);
if (!m_dset.is_open())
LIMA_THROW_EXCEPTION(lima::hdf5_error("Error opening H5 dataset")
<< boost::errinfo_h5_path(dataset_path));
// Query dataspace dimensions
auto dspace = m_dset.space();
m_nb_dims = dspace.ndims();
if ((m_nb_dims < 2) || (m_nb_dims > 3))
LIMA_THROW_EXCEPTION(lima::hdf5_error("Invalid H5 dataset dimensions")
<< boost::errinfo_h5_path(dataset_path));
hsize_t dims[] = {1, 0, 0};
hsize_t* pdim = (m_nb_dims == 2) ? &dims[1] : &dims[0];
if (dspace.dims(dims, nullptr) < 0)
LIMA_THROW_EXCEPTION(lima::hdf5_error("Error querying H5 dataset dimensions")
<< boost::errinfo_h5_path(dataset_path));
if (m_nb_frames == 0)
m_nb_frames = dims[0];
else if (dims[0] < m_nb_frames)
LIMA_THROW_EXCEPTION(lima::hdf5_error("Mismatch in number of frames in dataset")
<< boost::errinfo_h5_path(dataset_path));
m_frame_dims = dimensions_t(dims[2], dims[1]);
// Query dataset type
m_pixel = m_dset.datatype().pixel_type();
}
reader(reader const&) = default;
reader& operator=(reader const&) = default;
reader(reader&& w) = default;
dimensions_t dimensions() const { return m_frame_dims; }
pixel_enum pixel_type() const { return m_pixel; }
// Read a frame
frame read_frame(hsize_t frame_idx = 0)
{
frame out(m_frame_dims, m_pixel);
read_view(view(out), frame_idx);
return out;
}
template <typename... Views>
void read_view(boost::gil::any_image_view<Views...> const& v, hsize_t frame_idx = 0)
{
boost::gil::apply_operation(v, [&](auto&& view) { this->read_view(view, frame_idx); });
}
template <typename View>
void read_view(View const& v, hsize_t frame_idx = 0)
{
if (frame_idx >= m_nb_frames)
LIMA_THROW_EXCEPTION(lima::hdf5_error("Frame number out of bound")
<< boost::errinfo_frame_idx(frame_idx));
else if (v.dimensions() != m_frame_dims)
LIMA_THROW_EXCEPTION(lima::hdf5_error("Frame dimensions mismatch"));
dataspace mem_space = m_dset.space();
dataspace file_space = m_dset.space();
if (m_nb_dims == 3) {
hsize_t mem_offset[] = {0, 0, 0};
hsize_t file_offset[] = {frame_idx, 0, 0};
hsize_t count[] = {1, 1, 1};
hsize_t block[] = {1, hsize_t(m_frame_dims.y), hsize_t(m_frame_dims.x)};
mem_space.select_hyperslab(H5S_SELECT_SET, mem_offset, nullptr, count, block);
file_space.select_hyperslab(H5S_SELECT_SET, file_offset, nullptr, count, block);
}
using data_type = typename boost::gil::channel_type<View>::type;
auto dtype = predef_datatype::create<data_type>();
m_dset.read(&v(0, 0), dtype, mem_space, file_space);
}
protected:
dataset m_dset; //!< The dataset
dimensions_t m_frame_dims; //!< The dimensions of data frame
hsize_t m_nb_dims; //!< The number of dimensions of the dataset
hsize_t m_nb_frames; //!< The number of frames in the dataset
pixel_enum m_pixel; //!< The pixel type
};
} //namespace io::h5
} //namespace lima
......@@ -20,6 +20,7 @@
#include <lima/logging.hpp>
#include <lima/core/image/view.hpp>
#include <lima/io/h5/writer.hpp>
#include <lima/io/h5/reader.hpp>
namespace lima
{
......
......@@ -117,6 +117,42 @@ namespace io
});
}
// Factorize code used in reader dimensions & pixel_type
template <typename Callable>
auto apply(int frame_idx, Callable func)
{
// Check if the driver is available
auto driver_idx = frame_idx / m_params.nb_frames_per_file;
if (!is_driver_available(driver_idx))
// Create the next driver
open(driver_idx);
// Get the driver
auto& driver = m_drivers[driver_idx]->driver;
// And call the function
return func(driver);
}
/// Reader dimensions
auto dimensions(int frame_idx = 0)
{
return apply(frame_idx, [](auto& reader) { return reader.dimensions(); });
}
/// Reader pixel type
auto pixel_type(int frame_idx = 0)
{
return apply(frame_idx, [](auto& reader) { return reader.pixel_type(); });
}
/// Read the next_frame
lima::frame read_frame(int frame_idx = 0)
{
lima::frame frame;
apply(frame_idx, 1, [&frame](auto& reader, int frame_idx) { frame = reader.read_frame(frame_idx); });
return frame;
}
/// Close the driver for the given frame idx
/// \param frame_idx is the frame idx (or idx of the first frame in the chunk)
void close(int frame_idx)
......
......@@ -66,5 +66,19 @@ namespace io
(doc, "Number of frames per file"))
// clang-format on
LIMA_IO_DESCRIBE_FILENAME(filename_loading_params)
BOOST_DESCRIBE_STRUCT(loading_params, (filename_loading_params), (start_number, nb_frames_per_file))
// clang-format off
BOOST_ANNOTATE_MEMBER(loading_params, start_number,
(desc, "start number"),
(doc, "Start number in the file sequence"))
BOOST_ANNOTATE_MEMBER(loading_params, nb_frames_per_file,
(desc, "nb frames per file"),
(doc, "Number of frames per file [0 = autodetect]"))
// clang-format on
} //namespace io
} //namespace lima
......@@ -64,5 +64,17 @@ namespace io
int nb_frames_per_file = 1;
};
struct DefaultLoadingValues : DefaultFilenameValues
{
static constexpr char prefix[] = "input";
};
using filename_loading_params = filename_params_base<DefaultLoadingValues>;
struct loading_params : filename_loading_params
{
int start_number = 0;
int nb_frames_per_file = 0; //!< Automatic detection
};
} //namespace io
} //namespace lima
......@@ -56,6 +56,7 @@ add_executable(test_io_hdf5
test_hdf5_wrapper.cpp
test_hdf5_hl.cpp
test_hdf5_writer.cpp
test_hdf5_reader.cpp
)
target_precompile_headers(test_io_hdf5
......
// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#include "frame_fixture.hpp"
#include <lima/io/hdf5.hpp>
// A fixture that generates a hdf5 file with data
struct hdf5_file_fixture : frame_fixture
{
hdf5_file_fixture(std::string n = "test_hdf5_data.h5",
lima::io::h5::compression_enum c = lima::io::h5::compression_enum::none) :
file_name(n), comp(c)
{
BOOST_TEST_MESSAGE("setup hdf5_file_fixture");
lima::io::h5::saving_params params; // default constructor: 1 frame/file, 1 frame/chunk
params.compression = comp;
lima::io::h5::writer m(file_name, params, input_frame.dimensions(), input_frame.pixel_type());
m.write_view(lima::const_view(input_frame));
}
std::string file_name;
lima::io::h5::compression_enum comp;
};
// Copyright (C) 2018 Alejandro Homs Puron, ESRF.
// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
#include <type_traits>
#include <boost/test/unit_test.hpp>
#include <boost/gil/algorithm.hpp>
#include "hdf5_file_fixture.hpp"
#include <lima/io/hdf5.hpp>
namespace io = lima::io;
namespace h5 = lima::io::h5;
namespace gil = boost::gil;
bool frame_equal(lima::frame const& f1, lima::frame const& f2)
{
if ((f1.pixel_type() != f2.pixel_type()) || (f1.dimensions() != f2.dimensions()))
return false;
auto v1 = const_view(f1);
auto v2 = const_view(f2);
return gil::apply_operation(v1, [&v2](auto&& rv1) {
using V = std::decay_t<decltype(rv1)>;
BOOST_ASSERT(boost::variant2::holds_alternative<V>(v2));
auto&& rv2 = boost::variant2::get<V>(v2);
BOOST_ASSERT(rv1.size() == rv2.size());
return std::equal(rv1.begin(), rv1.end(), rv2.begin(),
[](auto&& p1, auto&& p2) { return gil::static_equal(p1, p2); });
});
}
BOOST_FIXTURE_TEST_CASE(test_frame_equal, frame_fixture)
{
BOOST_ASSERT(frame_equal(input_frame, input_frame));
}
BOOST_FIXTURE_TEST_CASE(test_hdf5_reader_raw, hdf5_file_fixture)
{
h5::reader r(file_name);
auto file_frame = r.read_frame(0);
BOOST_ASSERT(frame_equal(file_frame, input_frame));
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment