Merge branch '111-add-reader-mode' into 'develop'

Resolve "Add Reader Mode"

Closes #111

See merge request !110

bench/io/bench_io_hdf5.cpp

@@ -105,7 +105,9 @@ static void graph_io_hdf5_raw(benchmark::State& state)
     const int nb_frames = 10000;
     auto src = input_node(g, nb_frames);
 
-    lima::io::h5::writer writer("test_raw.h5", nb_frames, 1, dimensions_t{512, 512}, lima::pixel_enum::gray8);
+    lima::io::h5::saving_params params;
+    params.nb_frames_per_file = nb_frames;
+    lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
 
     flow::function_node<frame_t> write_view(
         g, flow::serial, [&writer](frame_t in) { writer.write_view(lima::view(in), in.metadata.idx); });
@@ -157,8 +159,10 @@ static void graph_io_hdf5_chunk_raw(benchmark::State& state)
             }
         });
 
-    lima::io::h5::writer writer("test_raw.h5", nb_frames, nb_frames_per_chunk, dimensions_t{512, 512},
-                                lima::pixel_enum::gray8);
+    lima::io::h5::saving_params params;
+    params.nb_frames_per_file = nb_frames;
+    params.nb_frames_per_chunk = nb_frames_per_chunk;
+    lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
 
     flow::function_node<chunk_ptr_t> write_chunk(
         g, flow::serial, [&writer](chunk_ptr_t in) { writer.write_chunk(in->data.data(), in->data.size(), in->id); });
@@ -210,8 +214,10 @@ static void graph_io_hdf5_comp(benchmark::State& state)
         return res;
     });
 
-    lima::io::h5::writer writer("test_comp.h5", nb_frames, 1, dimensions_t{512, 512}, lima::pixel_enum::gray8,
-                                lima::io::h5::compression_enum::bshuf_lz4);
+    lima::io::h5::saving_params params;
+    params.nb_frames_per_file = nb_frames;
+    params.compression = lima::io::h5::compression_enum::bshuf_lz4;
+    lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
 
     flow::function_node<chunk_ptr_t> write_chunk(
         g, flow::serial, [&writer](chunk_ptr_t in) { writer.write_chunk(in->data.data(), in->data.size(), in->id); });
@@ -290,8 +296,11 @@ static void graph_io_hdf5_chunk_comp(benchmark::State& state)
             return res;
         });
 
-    lima::io::h5::writer writer("test_chunk_comp.h5", nb_frames, nb_frames_per_chunk, dimensions_t{512, 512},
-                                lima::pixel_enum::gray8, lima::io::h5::compression_enum::bshuf_lz4);
+    lima::io::h5::saving_params params;
+    params.nb_frames_per_file = nb_frames;
+    params.nb_frames_per_chunk = nb_frames_per_chunk;
+    params.compression = lima::io::h5::compression_enum::bshuf_lz4;
+    lima::io::h5::writer writer("test_writer_raw.h5", params, dimensions_t{512, 512}, lima::pixel_enum::gray8);
 
     flow::function_node<chunk_ptr_t> write_chunk(g, flow::serial, [&writer, &nb_frames_per_chunk](chunk_ptr_t in) {
         writer.write_chunk(in->data.data(), in->data.size(), in->id);

detectors/simulator/include/lima/detectors/simulator/hw/enums.describe.hpp

@@ -11,13 +11,13 @@
 #include <boost/describe/io_enums.hpp>
 
 #include <lima/detectors/simulator/hw/enums.hpp>
+#include <lima/detectors/simulator/hw/generator/enums.describe.hpp>
 
 namespace lima
 {
 namespace detectors::simulator::hw
 {
-    BOOST_DESCRIBE_ENUM(generator_type_enum, gauss, diffraction)
-    BOOST_DESCRIBE_ENUM(channel_depth_enum, uint8, uint16, uint32, float32)
+    BOOST_DESCRIBE_ENUM(image_source_enum, generator, loader)
 
     using boost::describe::operator<<;
     using boost::describe::operator>>;

detectors/simulator/include/lima/detectors/simulator/hw/enums.hpp

@@ -6,25 +6,17 @@
 
 #pragma once
 
+#include <lima/detectors/simulator/hw/generator/enums.hpp>
+
 namespace lima
 {
 namespace detectors::simulator::hw
 {
-    /// Generator type
-    enum class generator_type_enum : int
-    {
-        gauss,
-        diffraction
-    };
-
-    /// Pixel bit depth
-    enum class channel_depth_enum : int
+    /// Image source type
+    enum class image_source_enum : int
     {
-        unknown,
-        uint8,
-        uint16,
-        uint32,
-        float32
+        generator,
+        loader
     };
 
 } // namespace detectors::simulator::hw

detectors/simulator/include/lima/detectors/simulator/hw/generator.hpp

@@ -6,147 +6,75 @@
 
 #pragma once
 
-#include <cmath>
-#include <vector>
+#include <lima/core/pixel.hpp>
 
-#include <lima/core/image/typedefs.hpp>
+#include <lima/processing/serial/generator.hpp>
 
-#include <lima/detectors/simulator/hw/params.hpp>
+#include <lima/detectors/simulator/hw/generator/params.hpp>
 
 namespace lima
 {
-namespace detectors::simulator::hw
+namespace detectors::simulator::hw::generator
 {
-    /// Calculates Gauss(x,y) for a given peak
-    ///
-    /// \param[in] x     X-coord
-    /// \param[in] y     Y-coord
-    /// \param[in] x0    X-coord of the center
-    /// \param[in] y0    Y-coord of the center
-    /// \param[in] fwhm  Full Width at Half Maximum
-    /// \param[in] max   the central maximum value
-    /// \return Gauss(x,y)
-    inline double gauss_2d(std::ptrdiff_t x, std::ptrdiff_t y, double x0, double y0, double fwhm, double max)
+    // The default channel converter maps the range of the value of the source to the destination,
+    // this is not what we want here
+    struct channel_converter_round
     {
-        const double SGM_FWHM = 0.42466090014400952136075141705144; // 1/(2*sqrt(2*ln(2)))
-
-        double sigma = SGM_FWHM * std::fabs(fwhm);
-        return max * std::exp(-((x - x0) * (x - x0) + (y - y0) * (y - y0)) / (2 * sigma * sigma));
-    }
-
-    /// Calculates the summary intensity at certain point
-    ///
-    /// \param[in] x  X-coord
-    /// \param[in] y  Y-coord
-    /// \return intensity
-    ///
-    inline double diffract(boost::gil::point<std::ptrdiff_t> dim, double x, double y)
-    {
-        const double pi = 3.14159265358979323846;
-
-        x -= dim.x / 2;
-        y -= dim.y / 2;
-        double r = std::sqrt(x * x + y * y);
-        double w = (2 * pi / 100 * (0.5 + r / 500));
-        double ar = (r >= 300) ? (r - 300) : 0;
-        double a = std::exp(-std::pow(ar / 500, 4.0)) / (r / 5000 + 0.1);
-        return a * std::pow(std::cos(r * w), 20.0);
-    }
-
-    struct generate_gauss
-    {
-        using point_t = boost::gil::point<std::ptrdiff_t>;
-        using const_t = generate_gauss;
-        using value_type = boost::gil::pixel<float, boost::gil::gray_layout_t>;
-        using reference = value_type;
-        using const_reference = value_type;
-        using argument_type = point_t;
-        using result_type = reference;
-
-        static const bool is_mutable = false;
-
-        generate_gauss(int frame_nr, double grow_factor) : m_frame_nr(frame_nr), m_grow_factor(grow_factor) {}
-
-        result_type operator()(const point_t& p) const
+        template <typename Ch1, typename Ch2>
+        void operator()(const Ch1& src, Ch2& dst) const
         {
-            double res = 0.0;
-
-            // For each peaks
-            for (auto&& peak : m_peaks)
-                // Add gaussian
-                res += gauss_2d(p.x, p.y, peak.x0, peak.y0, peak.fwhm, peak.max);
-
-            // Apply grow factor
-            res *= (1.0 + m_grow_factor * m_frame_nr);
-
-            return value_type(res);
+            dst = std::round(src);
         }
-
-        std::vector<gauss_peak> m_peaks;
-        int m_frame_nr = 0;
-        double m_grow_factor = 0.0;
     };
 
-    typedef generate_gauss::point_t point_t;
-    typedef boost::gil::virtual_2d_locator<generate_gauss, false> generate_gauss_locator_t;
-    typedef boost::gil::image_view<generate_gauss_locator_t> generate_gauss_view_t;
-
-    struct generate_diffraction
+    struct generator
     {
-        using point_t = boost::gil::point<std::ptrdiff_t>;
-        using const_t = generate_diffraction;
-        using value_type = boost::gil::pixel<float, boost::gil::gray_layout_t>;
-        using reference = value_type;
-        using const_reference = value_type;
-        using argument_type = point_t;
-        using result_type = reference;
-
-        static const bool is_mutable = false;
-
-        generate_diffraction(int frame_nr, double grow_factor, double diffract_x, double diffract_y,
-                             double diffract_sx = 0.0, double diffract_sy = 0.0) :
-            m_frame_nr(frame_nr),
-            m_grow_factor(grow_factor),
-            m_diffract_x(diffract_x),
-            m_diffract_y(diffract_y),
-            m_diffract_sx(diffract_sx),
-            m_diffract_sy(diffract_sy)
+        generator(init_params const& init_pars, exec_params const& exec_pars) :
+            m_init_params(init_pars), m_exec_params(exec_pars)
         {
         }
 
-        result_type operator()(const point_t& p) const
-        {
-            double res = 0.0;
-
-            double gx = m_diffract_x + m_frame_nr * m_diffract_sx;
-            double gy = m_diffract_y + m_frame_nr * m_diffract_sy;
+        point_t dimensions() { return {m_init_params.width, m_init_params.height}; }
 
-            // For each peaks
-            for (auto&& peak : m_peaks)
-                // Add gaussian
-                res += gauss_2d(gx, gy, peak.x0, peak.y0, peak.fwhm, peak.max);
+        pixel_enum pixel_type() { return m_exec_params.pixel_type; }
 
-            // Apply grow factor
-            res *= (1 + m_grow_factor * m_frame_nr);
-
-            res *= diffract(m_dim, p.x, p.y);
-
-            return value_type(res);
+        // visitor will be called with generated image view
+        template <typename Visitor>
+        void frame_getter_visit(int frame_nr, Visitor&& visitor)
+        {
+            auto pixel_visitor = [&](auto pixel) {
+                auto image_dims = dimensions();
+
+                auto converter = [image_dims, pixel, chan_conv = channel_converter_round()](auto&& gen) {
+                    auto src = make_generated_view(image_dims, gen);
+                    using Channel = typename std::decay_t<decltype(pixel)>::type;
+                    return boost::gil::color_converted_view<Channel>(src, chan_conv);
+                };
+
+                switch (m_exec_params.generator_type) {
+                case generator_type_enum::gauss: {
+                    generate_gauss func(frame_nr, m_exec_params.gauss);
+                    visitor(converter(func));
+                    break;
+                }
+
+                case generator_type_enum::diffraction: {
+                    generate_diffraction func(frame_nr, m_exec_params.diffraction);
+                    visitor(converter(func));
+                    break;
+                }
+
+                default:
+                    //TODO: Normalize exceptions
+                    throw std::runtime_error("Invalid generator mode");
+                }
+            };
+            auto pixel_type = typeof_pixel(m_exec_params.pixel_type);
+            std::visit(pixel_visitor, pixel_type);
         }
 
-        point_t m_dim;
-        std::vector<gauss_peak> m_peaks;
-        int m_frame_nr = 0;
-        double m_grow_factor = 0.0;
-
-        double m_diffract_x;
-        double m_diffract_y;
-        double m_diffract_sx;
-        double m_diffract_sy;
+        init_params m_init_params;
+        exec_params m_exec_params;
     };
-
-    using point_t = generate_gauss::point_t;
-    using generate_diffraction_locator_t = boost::gil::virtual_2d_locator<generate_diffraction, false>;
-    using generate_diffraction_view_t = boost::gil::image_view<generate_diffraction_locator_t>;
-} // namespace detectors::simulator::hw
+} // namespace detectors::simulator::hw::generator
 } // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/generator/enums.describe.hpp

+// Copyright (C) 2018 Samuel Debionne, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <boost/describe.hpp>
+
+#include <boost/describe/io_enums.hpp>
+
+#include <lima/detectors/simulator/hw/generator/enums.hpp>
+
+namespace lima
+{
+namespace detectors::simulator::hw::generator
+{
+    BOOST_DESCRIBE_ENUM(generator_type_enum, gauss, diffraction)
+
+} // namespace detectors::simulator::hw::generator
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/generator/enums.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+namespace lima
+{
+namespace detectors::simulator::hw::generator
+{
+    /// Generator type
+    enum class generator_type_enum : int
+    {
+        gauss,
+        diffraction
+    };
+
+} // namespace detectors::simulator::hw::generator
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/generator/params.describe.hpp

+// Copyright (C) 2018 Samuel Debionne, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <boost/describe/annotations.hpp>
+#include <boost/describe/io_structs.hpp>
+
+#include <lima/processing/serial/generator/params.describe.hpp>
+
+#include <lima/detectors/simulator/hw/generator/enums.describe.hpp>
+#include <lima/detectors/simulator/hw/generator/params.hpp>
+
+namespace lima
+{
+namespace detectors::simulator::hw::generator
+{
+    BOOST_DESCRIBE_STRUCT(init_params, (), (width, height))
+
+    // clang-format off
+    BOOST_ANNOTATE_MEMBER(init_params, width,
+        (desc, "frame width"),
+        (doc, "The width of the frame in pixel"))
+
+    BOOST_ANNOTATE_MEMBER(init_params, height,
+        (desc, "frame height"),
+        (doc, "The height of the frame in pixel"))
+    // clang-format on
+
+    BOOST_DESCRIBE_STRUCT(exec_params, (), (generator_type, gauss, diffraction, pixel_type))
+
+    // clang-format off
+    BOOST_ANNOTATE_MEMBER(exec_params, generator_type,
+        (desc, "type of generator"),
+        (doc, "The type of generator [gauss, diffraction]"))
+
+    BOOST_ANNOTATE_MEMBER(exec_params, gauss,
+        (desc, "Gauss generator params"),
+        (doc, "The configuration of the Gauss generator"))
+
+    BOOST_ANNOTATE_MEMBER(exec_params, diffraction,
+        (desc, "Diffraction generator params"),
+        (doc, "The configuration of the Diffraction generator"))
+
+    BOOST_ANNOTATE_MEMBER(exec_params, pixel_type,
+        (desc, "pixel type"),
+        (doc, "The pixel type (channel, color space and layout) [gray8, gray16, gray32, gray32f]"))
+    // clang-format on
+
+    using boost::describe::operator<<;
+
+} // namespace detectors::simulator::hw::generator
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/generator/params.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <vector>
+
+#include <lima/core/enums.hpp>
+
+#include <lima/processing/serial/generator/params.hpp>
+
+#include <lima/detectors/simulator/hw/generator/enums.hpp>
+
+namespace lima
+{
+namespace detectors::simulator::hw::generator
+{
+    using namespace lima::processing::generator;
+
+    /// Initialization parameters: defined once in the application
+    struct init_params
+    {
+        int width = 2048;
+        int height = 2048;
+    };
+
+    /// Execution parameters: defined each time the generator is instantiated
+    struct exec_params
+    {
+        generator_type_enum generator_type = generator_type_enum::gauss; //!< The type of generator
+        gauss_params gauss = {2048, 2048};                               //!< The Gauss generator parameters
+        diffraction_params diffraction = {2048, 2048};                   //!< The Diffraction generator parameters
+        pixel_enum pixel_type = pixel_enum::gray8;                       //!< The pixel type (layout, bitdepth)
+    };
+
+} // namespace detectors::simulator::hw::generator
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/loader.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <boost/gil/extension/dynamic_image/apply_operation.hpp>
+
+#include <lima/core/pixel.hpp>
+
+#include <lima/io/multi.hpp>
+#include <lima/io/hdf5.hpp>
+
+#include <lima/detectors/simulator/hw/loader/params.hpp>
+
+namespace lima
+{
+namespace detectors::simulator::hw::loader
+{
+    struct loader
+    {
+        using reader_t = io::multi<io::h5::reader>;
+
+        loader(exec_params const& exec_pars, int rank, int nb_frames) :
+            m_reader(exec_pars, rank, nb_frames),
+            m_dimensions(m_reader.dimensions()),
+            m_pixel_type(m_reader.pixel_type())
+        {
+        }
+
+        point_t dimensions() { return m_dimensions; }
+
+        pixel_enum pixel_type() { return m_pixel_type; }
+
+        // visitor will be called with generated image view
+        template <typename Visitor>
+        void frame_getter_visit(int frame_nr, Visitor&& visitor)
+        {
+            auto frame = m_reader.read_frame(frame_nr);
+            boost::gil::apply_operation(const_view(frame), visitor);
+        }
+
+        reader_t m_reader;
+        point_t m_dimensions;
+        pixel_enum m_pixel_type;
+    };
+} // namespace detectors::simulator::hw::loader
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/loader/params.describe.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <boost/describe/annotations.hpp>
+#include <boost/describe/io_structs.hpp>
+
+#include <lima/io/h5/params.describe.hpp>
+
+#include <lima/detectors/simulator/hw/loader/params.hpp>
+
+namespace lima
+{
+namespace detectors::simulator::hw::loader
+{
+    BOOST_DESCRIBE_STRUCT(exec_params, (io::h5::reader::params_t), ())
+
+    using boost::describe::operator<<;
+
+} // namespace detectors::simulator::hw::loader
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/loader/params.hpp

+// Copyright (C) 2020 Alejandro Homs Puron, ESRF.
+
+// Use, modification and distribution is subject to the Boost Software
+// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
+// http://www.boost.org/LICENSE_1_0.txt)
+
+#pragma once
+
+#include <vector>
+
+#include <lima/io/h5/reader.hpp>
+
+namespace lima
+{
+namespace detectors::simulator::hw::loader
+{
+    using namespace std::string_literals;
+
+    /// Execution parameters: defined each time the loader is instantiated
+    struct exec_params : io::h5::reader::params_t // loading_params
+    {
+    };
+
+} // namespace detectors::simulator::hw::loader
+} // namespace lima

detectors/simulator/include/lima/detectors/simulator/hw/params.describe.hpp

@@ -11,6 +11,8 @@
 
 #include <lima/hw/params.describe.hpp>
 
+#include <lima/detectors/simulator/hw/generator/params.describe.hpp>
+#include <lima/detectors/simulator/hw/loader/params.describe.hpp>
 #include <lima/detectors/simulator/hw/enums.describe.hpp>
 #include <lima/detectors/simulator/hw/params.hpp>
 
@@ -18,60 +20,32 @@ namespace lima
 {
 namespace detectors::simulator::hw
 {
-    BOOST_DESCRIBE_STRUCT(init_params, (), (width, height))
+    BOOST_DESCRIBE_STRUCT(init_params, (), (generator))
 
     // clang-format off
-    BOOST_ANNOTATE_MEMBER(init_params, width,
-        (desc, "frame width"),
-        (doc, "The width of the frame in pixel"))
-
-    BOOST_ANNOTATE_MEMBER(init_params, height,
-        (desc, "frame height"),
-        (doc, "The height of the frame in pixel"))
+    BOOST_ANNOTATE_MEMBER(init_params, generator,
+        (desc, "generator init params"),