[IO] Add H5 reader

include/lima/io/file_exists_policy.hpp

@@ -37,5 +37,20 @@ namespace io
         file_exists_policy_enum m_file_exists_policy;
     };
 
+    /// Policy defining the action when the reader does not find a file
+    struct read_file_exists_policy
+    {
+        template <typename Params>
+        read_file_exists_policy(Params const& /*params*/)
+        {
+        }
+
+        void operator()(std::filesystem::path filepath)
+        {
+            if (!std::filesystem::exists(filepath))
+                LIMA_THROW_EXCEPTION(io_error("File does not exist") << boost::errinfo_file_name(filepath.string()));
+        }
+    };
+
 } //namespace io
 } //namespace lima

include/lima/io/h5/enums.describe.hpp

@@ -19,6 +19,8 @@ namespace io
     {
         BOOST_DESCRIBE_ENUM(compression_enum, none, zip, bshuf_lz4)
 
+        BOOST_DESCRIBE_ENUM(file_type_enum, hdf5, nexus)
+
         using boost::describe::operator<<;
         using boost::describe::operator>>;
 

include/lima/io/h5/enums.hpp

@@ -19,6 +19,13 @@ namespace io
             bshuf_lz4 //!< bitshuffle filter and lz4 compression
         };
 
+        /// Used by file reader
+        enum class file_type_enum
+        {
+            hdf5, //!< Bare HDF5 file
+            nexus //!< Nexus HDF5 file
+        };
+
     } //namespace h5
 } //namespace io
 } //namespace lima

include/lima/io/h5/params.describe.hpp

@@ -19,7 +19,7 @@ namespace io::h5
     BOOST_DESCRIBE_STRUCT(saving_params, (lima::io::saving_params), (nb_frames_per_chunk, compression))
 
     // clang-format off
-    BOOST_ANNOTATE_MEMBER(h5::saving_params, nb_frames_per_chunk,
+    BOOST_ANNOTATE_MEMBER(saving_params, nb_frames_per_chunk,
         (desc, "nb frames per chunk"),
         (doc, "The number of frames per chunk"))
 
@@ -28,5 +28,17 @@ namespace io::h5
         (doc, "The compression filter [none, zip, bshuf_lz4]"))
     // clang-format on
 
+    BOOST_DESCRIBE_STRUCT(loading_params, (lima::io::loading_params), (file_type, dataset_path))
+
+    // clang-format off
+    BOOST_ANNOTATE_MEMBER(loading_params, file_type,
+        (desc, "file type [hdf5, nexus]"),
+        (doc, "The expected type format to load"))
+
+    BOOST_ANNOTATE_MEMBER(loading_params, dataset_path,
+        (desc, "dataset_path"),
+        (doc, "The HDF5 path of the dataset"))
+    // clang-format on
+
 } //namespace io::h5
 } //namespace lima

include/lima/io/h5/params.hpp

@@ -19,5 +19,11 @@ namespace io::h5
         compression_enum compression = compression_enum::none;
     };
 
+    struct loading_params : io::loading_params
+    {
+        file_type_enum file_type = file_type_enum::nexus;
+        std::string dataset_path = "/entry_0000/measurement/data"s;
+    };
+
 } //namespace io::h5
 } //namespace lima

include/lima/io/h5/reader.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <filesystem>
+
+#include <boost/exception/errinfo_errno.hpp>
+#include <boost/exception/errinfo_file_name.hpp>
+#include <boost/gil/extension/dynamic_image/dynamic_image_all.hpp>
+
+#include <hdf5.h>
+
+#include <lima/exceptions.hpp>
+#include <lima/logging.hpp>
+#include <lima/core/frame.hpp>
+#include <lima/core/image/view.hpp>
+#include <lima/io/file_exists_policy.hpp>
+#include <lima/io/h5/params.hpp>
+#include <lima/io/h5/nexus.hpp>
+
+namespace boost
+{
+template <class Tag, class T>
+class error_info;
+
+typedef error_info<struct errinfo_filesystem_path_, std::filesystem::path> errinfo_filesystem_path;
+typedef error_info<struct errinfo_h5_path_, lima::io::h5::path> errinfo_h5_path;
+typedef error_info<struct errinfo_frame_idx_, hsize_t> errinfo_frame_idx;
+} // namespace boost
+
+namespace lima
+{
+namespace io::h5
+{
+    class reader
+    {
+      public:
+        using dimensions_t = lima::point_t;
+        using params_t = loading_params;
+        using file_exists_policy_t = read_file_exists_policy;
+
+        reader(std::filesystem::path const& filename, params_t const& params = {}) :
+            m_nb_frames(params.nb_frames_per_file)
+        {
+            bool is_nexus = (params.file_type == file_type_enum::nexus);
+            const char* type_str = is_nexus ? "Nexus" : "HDF5";
+            LIMA_LOG(trace, io) << "Constructing " << type_str << " reader for file " << filename;
+
+            // Open the HDF5 file
+            const char* nx_class = is_nexus ? "NXroot" : nullptr;
+            auto f = nx::file::open(filename, nx_class);
+            if (!f.is_open())
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Error opening H5 file")
+                                     << boost::errinfo_filesystem_path(filename));
+
+            // Get Dataset
+            auto&& dataset_path = params.dataset_path;
+            m_dset = dataset::open(f, dataset_path);
+            if (!m_dset.is_open())
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Error opening H5 dataset")
+                                     << boost::errinfo_h5_path(dataset_path));
+
+            // Query dataspace dimensions
+            auto dspace = m_dset.space();
+            m_nb_dims = dspace.ndims();
+            if ((m_nb_dims < 2) || (m_nb_dims > 3))
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Invalid H5 dataset dimensions")
+                                     << boost::errinfo_h5_path(dataset_path));
+
+            hsize_t dims[] = {1, 0, 0};
+            hsize_t* pdim = (m_nb_dims == 2) ? &dims[1] : &dims[0];
+            if (dspace.dims(dims, nullptr) < 0)
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Error querying H5 dataset dimensions")
+                                     << boost::errinfo_h5_path(dataset_path));
+
+            if (m_nb_frames == 0)
+                m_nb_frames = dims[0];
+            else if (dims[0] < m_nb_frames)
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Mismatch in number of frames in dataset")
+                                     << boost::errinfo_h5_path(dataset_path));
+
+            m_frame_dims = dimensions_t(dims[2], dims[1]);
+
+            // Query dataset type
+            m_pixel = m_dset.datatype().pixel_type();
+        }
+
+        reader(reader const&) = default;
+        reader& operator=(reader const&) = default;
+
+        reader(reader&& w) = default;
+
+        dimensions_t dimensions() const { return m_frame_dims; }
+
+        pixel_enum pixel_type() const { return m_pixel; }
+
+        // Read a frame
+        frame read_frame(hsize_t frame_idx = 0)
+        {
+            frame out(m_frame_dims, m_pixel);
+            read_view(view(out), frame_idx);
+            return out;
+        }
+
+        template <typename... Views>
+        void read_view(boost::gil::any_image_view<Views...> const& v, hsize_t frame_idx = 0)
+        {
+            boost::gil::apply_operation(v, [&](auto&& view) { this->read_view(view, frame_idx); });
+        }
+
+        template <typename View>
+        void read_view(View const& v, hsize_t frame_idx = 0)
+        {
+            if (frame_idx >= m_nb_frames)
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Frame number out of bound")
+                                     << boost::errinfo_frame_idx(frame_idx));
+            else if (v.dimensions() != m_frame_dims)
+                LIMA_THROW_EXCEPTION(lima::hdf5_error("Frame dimensions mismatch"));
+
+            dataspace mem_space = m_dset.space();
+            dataspace file_space = m_dset.space();
+            if (m_nb_dims == 3) {
+                hsize_t mem_offset[] = {0, 0, 0};
+                hsize_t file_offset[] = {frame_idx, 0, 0};
+                hsize_t count[] = {1, 1, 1};
+                hsize_t block[] = {1, hsize_t(m_frame_dims.y), hsize_t(m_frame_dims.x)};
+                mem_space.select_hyperslab(H5S_SELECT_SET, mem_offset, nullptr, count, block);
+                file_space.select_hyperslab(H5S_SELECT_SET, file_offset, nullptr, count, block);
+            }
+
+            using data_type = typename boost::gil::channel_type<View>::type;
+            auto dtype = predef_datatype::create<data_type>();
+            m_dset.read(&v(0, 0), dtype, mem_space, file_space);
+        }
+
+      protected:
+        dataset m_dset; //!< The dataset
+
+        dimensions_t m_frame_dims; //!< The dimensions of data frame
+        hsize_t m_nb_dims;         //!< The number of dimensions of the dataset
+        hsize_t m_nb_frames;       //!< The number of frames in the dataset
+        pixel_enum m_pixel;        //!< The pixel type
+    };
+
+} //namespace io::h5
+} //namespace lima

include/lima/io/hdf5.hpp

@@ -20,6 +20,7 @@
 #include <lima/logging.hpp>
 #include <lima/core/image/view.hpp>
 #include <lima/io/h5/writer.hpp>
+#include <lima/io/h5/reader.hpp>
 
 namespace lima
 {

include/lima/io/multi.hpp

@@ -117,6 +117,42 @@ namespace io
             });
         }
 
+        // Factorize code used in reader dimensions & pixel_type
+        template <typename Callable>
+        auto apply(int frame_idx, Callable func)
+        {
+            // Check if the driver is available
+            auto driver_idx = frame_idx / m_params.nb_frames_per_file;
+            if (!is_driver_available(driver_idx))
+                // Create the next driver
+                open(driver_idx);
+
+            // Get the driver
+            auto& driver = m_drivers[driver_idx]->driver;
+            // And call the function
+            return func(driver);
+        }
+
+        /// Reader dimensions
+        auto dimensions(int frame_idx = 0)
+        {
+            return apply(frame_idx, [](auto& reader) { return reader.dimensions(); });
+        }
+
+        /// Reader pixel type
+        auto pixel_type(int frame_idx = 0)
+        {
+            return apply(frame_idx, [](auto& reader) { return reader.pixel_type(); });
+        }
+
+        /// Read the next_frame
+        lima::frame read_frame(int frame_idx = 0)
+        {
+            lima::frame frame;
+            apply(frame_idx, 1, [&frame](auto& reader, int frame_idx) { frame = reader.read_frame(frame_idx); });
+            return frame;
+        }
+
         /// Close the driver for the given frame idx
         /// \param frame_idx is the frame idx (or idx of the first frame in the chunk)
         void close(int frame_idx)

include/lima/io/params.describe.hpp

@@ -66,5 +66,19 @@ namespace io
         (doc, "Number of frames per file"))
     // clang-format on
 
+    LIMA_IO_DESCRIBE_FILENAME(filename_loading_params)
+
+    BOOST_DESCRIBE_STRUCT(loading_params, (filename_loading_params), (start_number, nb_frames_per_file))
+
+    // clang-format off
+    BOOST_ANNOTATE_MEMBER(loading_params, start_number,
+        (desc, "start number"),
+        (doc, "Start number in the file sequence"))
+
+    BOOST_ANNOTATE_MEMBER(loading_params, nb_frames_per_file,
+        (desc, "nb frames per file"),
+        (doc, "Number of frames per file [0 = autodetect]"))
+    // clang-format on
+
 } //namespace io
 } //namespace lima

include/lima/io/params.hpp

@@ -64,5 +64,17 @@ namespace io
         int nb_frames_per_file = 1;
     };
 
+    struct DefaultLoadingValues : DefaultFilenameValues
+    {
+        static constexpr char prefix[] = "input";
+    };
+    using filename_loading_params = filename_params_base<DefaultLoadingValues>;
+
+    struct loading_params : filename_loading_params
+    {
+        int start_number = 0;
+        int nb_frames_per_file = 0; //!< Automatic detection
+    };
+
 } //namespace io
 } //namespace lima

test/io/CMakeLists.txt

@@ -56,6 +56,7 @@ add_executable(test_io_hdf5
     test_hdf5_wrapper.cpp
     test_hdf5_hl.cpp
     test_hdf5_writer.cpp
+    test_hdf5_reader.cpp
 )
 
 target_precompile_headers(test_io_hdf5

test/io/hdf5_file_fixture.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#include "frame_fixture.hpp"
+
+#include <lima/io/hdf5.hpp>
+
+// A fixture that generates a hdf5 file with data
+struct hdf5_file_fixture : frame_fixture
+{
+    hdf5_file_fixture(std::string n = "test_hdf5_data.h5",
+                      lima::io::h5::compression_enum c = lima::io::h5::compression_enum::none) :
+        file_name(n), comp(c)
+    {
+        BOOST_TEST_MESSAGE("setup hdf5_file_fixture");
+        lima::io::h5::saving_params params; // default constructor: 1 frame/file, 1 frame/chunk
+        params.compression = comp;
+        lima::io::h5::writer m(file_name, params, input_frame.dimensions(), input_frame.pixel_type());
+        m.write_view(lima::const_view(input_frame));
+    }
+
+    std::string file_name;
+    lima::io::h5::compression_enum comp;
+};

test/io/test_hdf5_reader.cpp

+// Copyright (C) 2018 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#include <type_traits>
+
+#include <boost/test/unit_test.hpp>
+
+#include <boost/gil/algorithm.hpp>
+
+#include "hdf5_file_fixture.hpp"
+
+#include <lima/io/hdf5.hpp>
+
+namespace io = lima::io;
+namespace h5 = lima::io::h5;
+namespace gil = boost::gil;
+
+bool frame_equal(lima::frame const& f1, lima::frame const& f2)
+{
+    if ((f1.pixel_type() != f2.pixel_type()) || (f1.dimensions() != f2.dimensions()))
+        return false;
+    auto v1 = const_view(f1);
+    auto v2 = const_view(f2);
+    return gil::apply_operation(v1, [&v2](auto&& rv1) {
+        using V = std::decay_t<decltype(rv1)>;
+        BOOST_ASSERT(boost::variant2::holds_alternative<V>(v2));
+        auto&& rv2 = boost::variant2::get<V>(v2);
+        BOOST_ASSERT(rv1.size() == rv2.size());
+        return std::equal(rv1.begin(), rv1.end(), rv2.begin(),
+                          [](auto&& p1, auto&& p2) { return gil::static_equal(p1, p2); });
+    });
+}
+
+BOOST_FIXTURE_TEST_CASE(test_frame_equal, frame_fixture)
+{
+    BOOST_ASSERT(frame_equal(input_frame, input_frame));
+}
+
+BOOST_FIXTURE_TEST_CASE(test_hdf5_reader_raw, hdf5_file_fixture)
+{
+    h5::reader r(file_name);
+    auto file_frame = r.read_frame(0);
+    BOOST_ASSERT(frame_equal(file_frame, input_frame));
+}