From 96f86dc6cafceb08dbb3ef5d2acf74e7153bead1 Mon Sep 17 00:00:00 2001 From: Alejandro Homs Puron Date: Thu, 10 Mar 2022 11:08:53 +0100 Subject: [PATCH 1/9] [IO] Complete HDF5 support for signed integer pixel types --- include/lima/io/h5/wrapper/datatype.hpp | 1 + include/lima/io/h5/writer.hpp | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/include/lima/io/h5/wrapper/datatype.hpp b/include/lima/io/h5/wrapper/datatype.hpp index 21448610..1ac2c029 100644 --- a/include/lima/io/h5/wrapper/datatype.hpp +++ b/include/lima/io/h5/wrapper/datatype.hpp @@ -24,6 +24,7 @@ namespace io // clang-format off template struct deduct; template <> struct deduct { static const hid_t type() { return H5T_NATIVE_UINT8; } }; + template <> struct deduct { static const hid_t type() { return H5T_NATIVE_INT8; } }; template <> struct deduct { static const hid_t type() { return H5T_NATIVE_INT8; } }; template <> struct deduct { static const hid_t type() { return H5T_NATIVE_UINT16; } }; template <> struct deduct { static const hid_t type() { return H5T_NATIVE_INT16; } }; diff --git a/include/lima/io/h5/writer.hpp b/include/lima/io/h5/writer.hpp index 3acb750d..4d089606 100644 --- a/include/lima/io/h5/writer.hpp +++ b/include/lima/io/h5/writer.hpp @@ -68,6 +68,11 @@ namespace io static const bool is_supported = true; static const hid_t type() { return H5T_NATIVE_UINT8; } }; + template <> + struct write_support_private + : write_support_private + { + }; template <> struct write_support_private @@ -75,6 +80,11 @@ namespace io static const bool is_supported = true; static const hid_t type() { return H5T_NATIVE_UINT16; } }; + template <> + struct write_support_private + : write_support_private + { + }; template <> struct write_support_private @@ -82,6 +92,11 @@ namespace io static const bool is_supported = true; static const hid_t type() { return H5T_NATIVE_UINT32; } }; + template <> + struct write_support_private + : write_support_private + { + }; template <> struct write_support_private -- GitLab From f4e8bdc6e5352b0138136596214803f2ebdd9bd6 Mon Sep 17 00:00:00 2001 From: Alejandro Homs Puron Date: Fri, 4 Feb 2022 20:28:29 +0100 Subject: [PATCH 2/9] [IO] Add static data to sparse files: bin radius (1d) & masked pixel radius (2d) --- include/lima/io/h5/writer.hpp | 69 +++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 3 deletions(-) diff --git a/include/lima/io/h5/writer.hpp b/include/lima/io/h5/writer.hpp index 4d089606..5932ba87 100644 --- a/include/lima/io/h5/writer.hpp +++ b/include/lima/io/h5/writer.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -458,6 +459,8 @@ namespace io int nb_frames, int nb_frames_per_chunk, // int nb_bins, // Data point_t const& dim, // HDF5 + lima::frame radius1d, // + lima::frame radius2d_mask, // compression_enum comp = compression_enum::none, // h5::path entry_name = "/entry_0000", // Nexus h5::path detector_name = "Simulator", // @@ -543,6 +546,69 @@ namespace io nx::group data(entry, data_path, "NXdata"); + // Radius1d + { + // Validation + if (radius1d.pixel_type() != pixel_enum::gray32f) { + LIMA_THROW_EXCEPTION(lima::invalid_argument("Invalid radius1d type: must be gray32f")); + } else if (radius1d.dimensions() != point_t(m_nb_bins, 1)) { + std::ostringstream error; + auto&& [x, y] = radius1d.dimensions(); + error << "Invalid radius1d size: " << x << "x" << y << ", " + << "expected " << m_nb_bins << "x" << 1; + LIMA_THROW_EXCEPTION(lima::invalid_argument(error.str())); + } + auto v = boost::variant2::get(const_view(radius1d)); + + // Set dimensions + hsize_t dims[] = {m_nb_bins}; + dataspace file_space(lengthof(dims), dims); + dataspace mem_space(lengthof(dims), dims); + predef_datatype dtype = datatype(v); + + // Create dataset + dataset mask(data, "radius"_p, dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + + // Add attributes + mask.attrs.create("interpretation", "spectrum"s); + mask.attrs.create("long_name", "Bin radius"s); + + // Write mask + H5Dwrite(mask, dtype, mem_space, file_space, H5P_DEFAULT, &v(0, 0)); + } + + // Radius2d-Mask + { + // Validation + if (radius2d_mask.pixel_type() != pixel_enum::gray32f) { + LIMA_THROW_EXCEPTION(lima::invalid_argument("Invalid radius2d-mask type: must be gray32f")); + } else if (radius2d_mask.dimensions() != dim) { + std::ostringstream error; + auto&& [x, y] = radius2d_mask.dimensions(); + auto&& [dim_x, dim_y] = dim; + error << "Invalid radius2d-mask size: " << x << "x" << y << ", " + << "expected " << dim_x << "x" << dim_y; + LIMA_THROW_EXCEPTION(lima::invalid_argument(error.str())); + } + auto v = boost::variant2::get(const_view(radius2d_mask)); + + // Set dimensions + hsize_t dims[] = {m_dimensions.y, m_dimensions.x}; + dataspace file_space(lengthof(dims), dims); + dataspace mem_space(lengthof(dims), dims); + predef_datatype dtype = datatype(v); + + // Create dataset + dataset mask(data, "mask"_p, dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT); + + // Add attributes + mask.attrs.create("interpretation", "image"s); + mask.attrs.create("long_name", "Detector masked radius2d"s); + + // Write mask + H5Dwrite(mask, dtype, mem_space, file_space, H5P_DEFAULT, &v(0, 0)); + } + // Background { // Set dimensions @@ -605,9 +671,6 @@ namespace io dataset(data, "intensity"_p, m_peak_value_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT); } - // Add attributes - //m_mask.attrs.create("interpretation", "image"s); - // Create NEXUS plot link link_hard(entry, data_path, entry, "instrument"_p / detector_name / "plot"_p); data.attrs.create("signal", "background_avg"_p); -- GitLab From dab2779beb3403ad9530a3d982a39c12c2b82486 Mon Sep 17 00:00:00 2001 From: Samuel Debionne Date: Wed, 9 Feb 2022 21:33:13 +0100 Subject: [PATCH 3/9] [PROC][SMX] Add SMX processing pipeline --- processings/smx/CMakeLists.txt | 26 ++ .../processing/pipelines/smx.describe.hpp | 10 + .../include/lima/processing/pipelines/smx.hpp | 9 + .../pipelines/smx/io_hdf5_sparse_node.hpp | 94 +++++ .../pipelines/smx/params.describe.hpp | 201 ++++++++++ .../lima/processing/pipelines/smx/params.hpp | 92 +++++ .../pipelines/smx/peak_finder_node.hpp | 140 +++++++ .../pipelines/smx/pipeline.describe.hpp | 55 +++ .../processing/pipelines/smx/pipeline.hpp | 93 +++++ .../processing/pipelines/smx/sparse_frame.hpp | 72 ++++ processings/smx/src/CMakeLists.txt | 52 +++ processings/smx/src/pipeline_smx.cpp | 367 ++++++++++++++++++ processings/smx/tango/CMakeLists.txt | 7 + processings/smx/tango/cpp/CMakeLists.txt | 21 + .../smx/tango/cpp/conda/CMakeLists.txt | 31 ++ processings/smx/tango/cpp/conda/build.sh | 3 + processings/smx/tango/cpp/conda/meta.yaml | 46 +++ processings/smx/tango/cpp/processing.cpp | 105 +++++ processings/smx/tango/cpp/processing.hpp | 85 ++++ .../smx/tango/cpp/processing_class.cpp | 331 ++++++++++++++++ .../smx/tango/cpp/processing_class.hpp | 113 ++++++ 21 files changed, 1953 insertions(+) create mode 100644 processings/smx/CMakeLists.txt create mode 100644 processings/smx/include/lima/processing/pipelines/smx.describe.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/params.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/pipeline.describe.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/pipeline.hpp create mode 100644 processings/smx/include/lima/processing/pipelines/smx/sparse_frame.hpp create mode 100644 processings/smx/src/CMakeLists.txt create mode 100644 processings/smx/src/pipeline_smx.cpp create mode 100644 processings/smx/tango/CMakeLists.txt create mode 100644 processings/smx/tango/cpp/CMakeLists.txt create mode 100644 processings/smx/tango/cpp/conda/CMakeLists.txt create mode 100644 processings/smx/tango/cpp/conda/build.sh create mode 100644 processings/smx/tango/cpp/conda/meta.yaml create mode 100644 processings/smx/tango/cpp/processing.cpp create mode 100644 processings/smx/tango/cpp/processing.hpp create mode 100644 processings/smx/tango/cpp/processing_class.cpp create mode 100644 processings/smx/tango/cpp/processing_class.hpp diff --git a/processings/smx/CMakeLists.txt b/processings/smx/CMakeLists.txt new file mode 100644 index 00000000..b2d76467 --- /dev/null +++ b/processings/smx/CMakeLists.txt @@ -0,0 +1,26 @@ +# Copyright (C) 2018 Samuel Debionne, ESRF. + +# Use, modification and distribution is subject to the Boost Software +# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +project(smx VERSION 0.1.0) + +string(TOUPPER "${PROJECT_NAME}" PROJECT_NAME_UPPER) +set(_option_name LIMA_PIPELINE_${PROJECT_NAME_UPPER}) + +option(${_option_name} "Add Support for Pipeline ${PROJECT_NAME_UPPER}" ON) + +add_subdirectory(src) +#add_subdirectory(test) + +# Tango +if(LIMA_ENABLE_TANGO) + add_subdirectory(tango) +endif() + +# Python +if(LIMA_ENABLE_PYTHON) + #add_subdirectory(python) +endif() + diff --git a/processings/smx/include/lima/processing/pipelines/smx.describe.hpp b/processings/smx/include/lima/processing/pipelines/smx.describe.hpp new file mode 100644 index 00000000..742da2cf --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx.describe.hpp @@ -0,0 +1,10 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include diff --git a/processings/smx/include/lima/processing/pipelines/smx.hpp b/processings/smx/include/lima/processing/pipelines/smx.hpp new file mode 100644 index 00000000..08bdb589 --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx.hpp @@ -0,0 +1,9 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include diff --git a/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp b/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp new file mode 100644 index 00000000..d1fe0039 --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp @@ -0,0 +1,94 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +#include + +#include +#include +#include +#include + +#include +#include + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + struct io_hdf5_sparse_node : public tbb::flow::composite_node>, + std::tuple> + { + using parent_t = tbb::flow::composite_node>, + std::tuple>; + using dimensions_t = typename sparse_frame::point_t; + + using sequencer_node_t = tbb::flow::sequencer_node>; + using writer_node_t = tbb::flow::function_node, tbb::flow::continue_msg>; + + sequencer_node_t sequencer_node; + writer_node_t writer_node; + + struct writer_body + { + using writer_t = io::multi; + + writer_body(std::filesystem::path base_path, std::string filename_format, std::string filename_prefix, + std::string filename_suffix, int start_number, int rank, + io::file_exists_policy_enum file_exists, int nb_frames, int nb_frames_per_file, + int nb_frames_per_chunk, int nb_bins, dimensions_t const& dimensions, + io::h5::compression_enum comp = io::h5::compression_enum::bshuf_lz4) : + m_writer(std::make_shared(base_path, filename_format, filename_prefix, filename_suffix, + start_number, rank, file_exists, nb_frames, nb_frames_per_file, + nb_frames_per_chunk, nb_bins, dimensions, comp)), + m_nb_frames_per_chunk(nb_frames_per_chunk) + { + } + + tbb::flow::continue_msg operator()(std::shared_ptr const& in) + { + m_writer->apply(in->metadata.idx, 1, [&](auto& writer, int frame_idx) { + writer.write_frame_peaks(frame_idx, in->peak_indices, in->peak_values); + writer.write_background_avg(frame_idx, in->background_avg); + writer.write_background_std(frame_idx, in->background_std); + }); + + return tbb::flow::continue_msg{}; + } + + std::shared_ptr m_writer; + + int m_nb_frames_per_chunk = 0; + }; + + io_hdf5_sparse_node(tbb::flow::graph& g, std::filesystem::path base_path, std::string filename_format, + std::string filename_prefix, std::string filename_suffix, int start_number, int rank, + io::file_exists_policy_enum file_exists, int nb_frames, int nb_frames_per_file, + int nb_frames_per_chunk, int nb_bins, dimensions_t const& dimensions, + io::h5::compression_enum comp = io::h5::compression_enum::bshuf_lz4) : + parent_t(g), + sequencer_node(g, [](auto&& frame) -> int { return frame->metadata.idx; }), + writer_node(g, 1 /*serial for now*/, + writer_body(base_path, filename_format, filename_prefix, filename_suffix, start_number, + rank, file_exists, nb_frames, nb_frames_per_file, nb_frames_per_chunk, nb_bins, + dimensions, comp)) + { + tbb::flow::make_edge(sequencer_node, writer_node); + typename parent_t::input_ports_type input_tuple(sequencer_node); + typename parent_t::output_ports_type output_tuple(writer_node); + parent_t::set_external_ports(input_tuple, output_tuple); + } + }; + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp b/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp new file mode 100644 index 00000000..c9b4eefd --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp @@ -0,0 +1,201 @@ +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +#include + +#include + +// This file is part of the user interface of the library and should not include any private dependencies +// or other thing that expose implementation details + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + BOOST_DESCRIBE_STRUCT(fifo_params, (), (nb_fifo_frames)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(fifo_params, nb_fifo_frames, + (desc, "number of frames in FIFO"), + (doc, "The number of frames in the Processing FIFO")) + // clang-format on + + BOOST_DESCRIBE_STRUCT(buffer_params, (), (nb_roi_counters_buffer, nb_frames_buffer)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(buffer_params, nb_roi_counters_buffer, + (desc, "number of roi counters in buffer"), + (doc, "The number of roi counters in the Circular buffer")) + + BOOST_ANNOTATE_MEMBER(buffer_params, nb_peak_counters_buffer, + (desc, "number of peak counters in buffer"), + (doc, "The number of peak counters in the Circular buffer")) + + BOOST_ANNOTATE_MEMBER(buffer_params, nb_frames_buffer, + (desc, "number of frames in buffer"), + (doc, "The number of frames in the Circular buffer")) + // clang-format on + + BOOST_DESCRIBE_STRUCT(roi_counters_params, (), (rois)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(roi_counters_params, rois, + (desc, "roi counters"), + (doc, "A vector of ROIs for the counters")) + // clang-format on + + BOOST_DESCRIBE_STRUCT(gpu_params, (), (device_idx)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(gpu_params, device_idx, + (desc, "device idx"), + (doc, "The index of the GPU device in the list of OpenCL 1.2 devices")) + // clang-format on + + BOOST_DESCRIBE_STRUCT(gain_pedestal_params, (), (gain_path, pedestal_path)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(gain_pedestal_params, gain_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(gain_pedestal_params, pedestal_path, + (desc, "todo"), + (doc, "TODO")) + // clang-format on + + BOOST_DESCRIBE_STRUCT(fai_params, (), + (error_model, variance_path, mask_path, dark_path, dark_variance_path, flat_path, + solid_angle_path, polarization_path, absorption_path, dummy, delta_dummy, + normalization_factor, csr_path, cutoff_clip, cycle, radius2d_path, radius1d_path, noise, + cutoff_pick, cl_source_path)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(fai_params, error_model, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, variance_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, mask_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, dark_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, dark_variance_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, flat_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, solid_angle_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, polarization_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, absorption_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, dummy, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, delta_dummy, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, normalization_factor, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, csr_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, cutoff_clip, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, cycle, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, radius2d_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, radius1d_path, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, noise, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, cutoff_pick, + (desc, "todo"), + (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(fai_params, cl_source_path, + (desc, "todo"), + (doc, "TODO")) + // clang-format on + + BOOST_DESCRIBE_STRUCT(proc_params, (), (fifo, buffers, saving_dense, saving_sparse, gpu, jfrau, fai, counters)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(proc_params, fifo, + (desc, "fifo parameters"), + (doc, "The processing FIFO parameters")) + + BOOST_ANNOTATE_MEMBER(proc_params, buffers, + (desc, "buffers parameters"), + (doc, "The circulatr buffer parameters")) + + BOOST_ANNOTATE_MEMBER(proc_params, saving_dense, + (desc, "saving parameters raw"), + (doc, "The HDF5 saving parameters for the raw frame")) + + BOOST_ANNOTATE_MEMBER(proc_params, saving_sparse, + (desc, "saving parameters assembled"), + (doc, "The HDF5 saving parameters for the assembled frame")) + + BOOST_ANNOTATE_MEMBER(proc_params, gpu, + (desc, "gpu params"), + (doc, "The GPU (OpenCL) parameters")) + + BOOST_ANNOTATE_MEMBER(proc_params, jfrau, + (desc, "jungfrau gain pedestal params"), + (doc, "The Gain / Pesdetal correction parameters")) + + BOOST_ANNOTATE_MEMBER(proc_params, fai, + (desc, "peak finder params"), + (doc, "The peak finder (Fast Azimuthal Integrations) parameters")) + + BOOST_ANNOTATE_MEMBER(proc_params, counters, + (desc, "roi counters parameters"), + (doc, "The ROI counters parameters")) + // clang-format on + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/include/lima/processing/pipelines/smx/params.hpp b/processings/smx/include/lima/processing/pipelines/smx/params.hpp new file mode 100644 index 00000000..475a3418 --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/params.hpp @@ -0,0 +1,92 @@ +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +#include +#include + +#include + +// This file is part of the user interface of the library and should not include any private dependencies +// or other thing that expose implementation details + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + struct fifo_params + { + int nb_fifo_frames = 100; + }; + + struct buffer_params + { + int nb_roi_counters_buffer = 100; + int nb_peak_counters_buffer = 100; + int nb_frames_buffer = 100; + }; + + struct roi_counters_params + { + using roi_t = rectangle; + std::vector rois = {}; + }; + + struct gpu_params + { + std::size_t device_idx = 0; + }; + + struct gain_pedestal_params + { + std::filesystem::path gain_path; // JungFrau Gain/Pedestal + std::filesystem::path pedestal_path; // + }; + + struct fai_params + { + fai::error_model_enum error_model = fai::error_model_enum::poissonian; // FAI + std::filesystem::path variance_path; // Preprocessing + std::filesystem::path mask_path; // + std::filesystem::path dark_path; // + std::filesystem::path dark_variance_path; // + std::filesystem::path flat_path; // + std::filesystem::path solid_angle_path; // + std::filesystem::path polarization_path; // + std::filesystem::path absorption_path; // + float dummy = 0.0f; // + float delta_dummy = 0.0f; // + float normalization_factor = 0.0f; // + std::filesystem::path csr_path; // + float cutoff_clip = 3.0f; // Sigma clipping + int cycle = 4; // + float empty = 0.0f; // + std::filesystem::path radius2d_path; // Peak finding + std::filesystem::path radius1d_path; // + float noise = 0.0f; // + float cutoff_pick = 0.0f; // + std::filesystem::path cl_source_path = "detectors/processing/psi/src"; // CL source file path + }; + + struct proc_params + { + fifo_params fifo; + buffer_params buffers; + io::h5::saving_params saving_dense; + io::h5::saving_params saving_sparse; + gpu_params gpu; + gain_pedestal_params jfrau; + fai_params fai; + roi_counters_params counters; + }; + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp b/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp new file mode 100644 index 00000000..b282140c --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp @@ -0,0 +1,140 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#include + +#include +#include + +#include + +#include +#include + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + namespace bcl = boost::compute; + + struct peak_finder_node : public tbb::flow::function_node> + { + using parent_t = tbb::flow::function_node>; + + struct peak_finder_body + { + peak_finder_body(std::size_t width, std::size_t height, fai::kernel_params const& params, + std::filesystem::path const& cl_source_path, bcl::context& context, + bcl::command_queue& queue) : + context(context), + queue(queue), + nb_pixels(width * height), + nb_bins(params.csr_indptr.size() - 1), + raw_d(nb_pixels, context), // Input + peak_indices_d(nb_pixels, context), // Outputs Peaks + peak_values_d(nb_pixels, context), // + background_avg_d(nb_bins, context), // Background + background_std_d(nb_bins, context), // + debug(false), + corrected_d(debug ? nb_pixels : 0), // Corrected + preprocessed_d(debug ? nb_pixels : 0), // Preprocessed + cliped_d(debug ? nb_bins : 0), // Cliped (after sigma clip) + found_d(debug ? nb_pixels : 0) // Found (after peak find) + { + LIMA_LOG(trace, proc) << "nb_pixels:" << nb_pixels; + LIMA_LOG(trace, proc) << "nb_bins:" << nb_bins; + + // Create the peakfinder for the kernel params + pf = fai::make_peak_finder( + context, // + width, height, // + params.gain, params.pedestal, params.error_model, params.variance, params.mask, params.dark, + params.dark_variance, params.flat, params.solid_angle, params.polarization, params.absorption, + params.dummy, params.delta_dummy, params.normalization_factor, params.csr_data, + params.csr_indices, params.csr_indptr, params.cutoff_clip, params.cycle, params.empty, + params.radius2d, params.radius1d, params.noise, params.cutoff_pick, // + cl_source_path, queue); + + LIMA_LOG(trace, proc) << "Peak finder created"; + } + + std::shared_ptr operator()(lima::frame const& in) const + { + assert(nb_pixels == in.size()); + + // Write input to the device (non-blocking) + //copy_async(in.m_data, raw_d, queue); + auto v = boost::variant2::get(lima::const_view(in)); + bcl::copy(v.begin(), v.end(), raw_d.begin(), queue); + + // Run the kernels + std::size_t nb_peaks = pf(queue, raw_d, // + corrected_d, preprocessed_d, cliped_d, found_d, // + peak_indices_d, peak_values_d, background_avg_d, background_std_d); + //{ + // std::ofstream of("/tmp/preprocessed_" + std::to_string(in.metadata.idx) + ".bin"); + // const char* d = reinterpret_cast(preprocessed_dbg.data()); + // of.write(d, preprocessed_dbg.size() * sizeof(preprocessed_dbg[0])); + //} + + LIMA_LOG(trace, proc) << nb_peaks << " peaks found"; + + std::shared_ptr res = + std::make_shared(in.dimensions(), nb_peaks, nb_bins); + + // Copy metadata + res->metadata = in.metadata; + res->attributes = in.attributes; + + // Read result from the device to array (non-blocking) + bcl::copy_n(peak_indices_d.begin(), nb_peaks, res->peak_indices.begin(), queue); + bcl::copy_n(peak_values_d.begin(), nb_peaks, res->peak_values.begin(), queue); + bcl::copy(background_avg_d.begin(), background_avg_d.end(), res->background_avg.begin(), queue); + bcl::copy(background_std_d.begin(), background_std_d.end(), res->background_std.begin(), queue); + + return res; + } + + bcl::context context; + mutable bcl::command_queue queue; + + std::size_t nb_pixels; + std::size_t nb_bins; + + fai::peak_finder pf; + + mutable fai::vector raw_d; // Input + mutable fai::vector peak_indices_d; // Outputs Peaks + mutable fai::vector peak_values_d; // + mutable fai::vector background_avg_d; // Background + mutable fai::vector background_std_d; // + + // Following outputs are usually used only for debugging + bool debug; + mutable fai::vector corrected_d; // Corrected + mutable fai::vector preprocessed_d; // Preprocessed + mutable fai::vector cliped_d; // Cliped (after sigma clip) + mutable fai::vector found_d; // Found (after peak find) + }; + + peak_finder_node(tbb::flow::graph& g, std::size_t width, std::size_t height, + fai::kernel_params const& params, std::filesystem::path const& cl_source_path, + bcl::context& context, bcl::command_queue& queue) : + parent_t(g, tbb::flow::serial /*serial fo now*/, + peak_finder_body(width, height, params, cl_source_path, context, queue)) + { + } + }; + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/include/lima/processing/pipelines/smx/pipeline.describe.hpp b/processings/smx/include/lima/processing/pipelines/smx/pipeline.describe.hpp new file mode 100644 index 00000000..bf0ad21d --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/pipeline.describe.hpp @@ -0,0 +1,55 @@ +// Copyright (C) 2018 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +#include + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + BOOST_DESCRIBE_STRUCT(counters, (), + (nb_frames_source, nb_frames_reconstructed, nb_frames_counters, nb_frames_sparsified, + nb_frames_sparse_saved, nb_frames_dense_saved)) + + // clang-format off + BOOST_ANNOTATE_MEMBER(counters, nb_frames_source, + (desc, "nb frames source"), + (doc, "The number of frame poped from the FIFO")) + + BOOST_ANNOTATE_MEMBER(counters, nb_frames_reconstructed, + (desc, "nb frames roi counters"), + (doc, "The number of frame reconstructed (with gap)")) + + BOOST_ANNOTATE_MEMBER(counters, nb_frames_counters, + (desc, "nb frames roi counters"), + (doc, "The number of frame processed with ROI counters")) + + BOOST_ANNOTATE_MEMBER(counters, nb_frames_sparsified, + (desc, "nb frames roi counters"), + (doc, "The number of frame sparsified with FAI")) + + BOOST_ANNOTATE_MEMBER(counters, nb_frames_sparse_saved, + (desc, "nb frames sparse saved"), + (doc, "The number of sparse frame saved to file")) + + BOOST_ANNOTATE_MEMBER(counters, nb_frames_dense_saved, + (desc, "nb frames dense saved"), + (doc, "The number of dense frame saved to file")) + // clang-format on + + using boost::json::tag_invoke; + + } // namespace smx +} // namespace processing::pipelines + +} // namespace lima diff --git a/processings/smx/include/lima/processing/pipelines/smx/pipeline.hpp b/processings/smx/include/lima/processing/pipelines/smx/pipeline.hpp new file mode 100644 index 00000000..bda7d4cf --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/pipeline.hpp @@ -0,0 +1,93 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +#include +#include + +#include +#include + +#include + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + struct counters + { + int nb_frames_source = 0; + int nb_frames_reconstructed = 0; + int nb_frames_counters = 0; + int nb_frames_sparsified = 0; + int nb_frames_sparse_saved = 0; + int nb_frames_dense_saved = 0; + }; + + class PIPELINE_SMX_EXPORT pipeline + { + public: + using proc_params_t = proc_params; + + using input_t = lima::frame; + using frame_t = lima::frame; + + pipeline(int rank, hw::frame_info const& acq_info, hw::acq_params const& acq_params, + proc_params_t const& proc_params); + ~pipeline(); // defined in the implementation file, where impl is a complete type + pipeline(pipeline&&); // defined in the implementation file + pipeline(const pipeline&) = delete; + pipeline& operator=(pipeline&&); // defined in the implementation file + pipeline& operator=(const pipeline&) = delete; + + /// Activate the pipeline (start poping data from the queue) + void activate(); + + /// Returns when the processing has finished + void wait_for_all(); + + /// Abort the processing + void abort(); + + /// Process a frame + void process(input_t const& frm); + + /// Returns the progress counters + counters get_counters() const; + + /// Pop a given number of ROI counters + std::vector pop_roi_counters(std::size_t nb_frames = 1); + + /// Pop a given number of peak counters + std::vector pop_peak_counters(std::size_t nb_frames = 1); + + /// Returns frame for the given index + frame_t get_frame(std::size_t frame_idx = -1) const; + + private: + class impl; + +#if defined(LIMA_HAS_PROPAGATE_CONST) + std::experimental::propagate_const< // const-forwarding pointer wrapper + std::unique_ptr< // unique-ownership opaque pointer + impl>> + m_pimpl; // to the forward-declared implementation class +#else + std::unique_ptr< // unique-ownership opaque pointer + impl> + m_pimpl; // to the forward-declared implementation class +#endif + }; + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/include/lima/processing/pipelines/smx/sparse_frame.hpp b/processings/smx/include/lima/processing/pipelines/smx/sparse_frame.hpp new file mode 100644 index 00000000..1626cb92 --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/sparse_frame.hpp @@ -0,0 +1,72 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + struct sparse_frame + { + static const std::size_t num_dimensions = 2; + using coord_t = std::ptrdiff_t; + using point_t = point; + + using attributes_t = std::pmr::unordered_map; + using metadata_t = frame_metadata; + + // Construct empty + explicit sparse_frame() = default; + + sparse_frame(const point_t& dims, std::size_t nb_peaks, std::size_t nb_bins) : + m_dimensions(dims), + peak_indices(nb_peaks), + peak_values(nb_peaks), + background_avg(nb_bins), + background_std(nb_bins) + { + } + + sparse_frame(coord_t width, coord_t height, std::size_t nb_peaks, std::size_t nb_bins) : + m_dimensions(width, height), + peak_indices(nb_peaks), + peak_values(nb_peaks), + background_avg(nb_bins), + background_std(nb_bins) + { + } + + const point_t& dimensions() const { return m_dimensions; } + coord_t width() const { return m_dimensions.x; } + coord_t height() const { return m_dimensions.y; } + std::size_t size() const { return m_dimensions.x * m_dimensions.y; } + std::size_t nb_peaks() const { return peak_indices.size(); } + std::size_t nb_bins() const { return background_avg.size(); } + + std::vector peak_indices; + std::vector peak_values; + std::vector background_avg; + std::vector background_std; + + point_t m_dimensions; //" + PUBLIC "$" # For export header + PUBLIC "$" +) + +target_link_libraries(pipeline_smx + PUBLIC lima_core processing_fai +) + +# Set version and output name +set_target_properties(pipeline_smx PROPERTIES + OUTPUT_NAME "lima_${PROJECT_NAME}" + VERSION "${PROJECT_VERSION}" + SOVERSION "${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}") + + +install( + TARGETS pipeline_smx processing_fai + EXPORT "${TARGETS_EXPORT_NAME}" + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} # import library + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} # .so files are libraries + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} # .dll files are binaries + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} # this does not actually install anything (but used by downstream projects) +) + +set (PROJECT_LIBRARIES lima_core pipeline_smx) +include(PackageConfig) \ No newline at end of file diff --git a/processings/smx/src/pipeline_smx.cpp b/processings/smx/src/pipeline_smx.cpp new file mode 100644 index 00000000..0299641e --- /dev/null +++ b/processings/smx/src/pipeline_smx.cpp @@ -0,0 +1,367 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +// Map Boost.GIL type to its OpenCL counter parts +BOOST_COMPUTE_TYPE_NAME(lima::gray8_pixel_t, uchar) +BOOST_COMPUTE_TYPE_NAME(lima::gray16_pixel_t, ushort) +BOOST_COMPUTE_TYPE_NAME(lima::gray32_pixel_t, uint) + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + class pipeline::impl + { + using input_t = lima::frame; + using frame_t = lima::frame; + + public: + using proc_params_t = proc_params; + + impl(int rank, hw::frame_info const& frame_info, hw::acq_params const& acq_params, + proc_params_t const& proc_params) : + m_fifo(proc_params.fifo.nb_fifo_frames), + m_graph(m_task_group_context), + m_roi_counters_buffer(proc_params.buffers.nb_roi_counters_buffer), + m_peak_counters_buffer(proc_params.buffers.nb_peak_counters_buffer), + m_frames_buffer(proc_params.buffers.nb_frames_buffer) + { + // Filter for a 1.2 platform and set it as the default + auto platforms = bcl::system::platforms(); + if (platforms.empty()) { + LIMA_THROW_EXCEPTION(lima::runtime_error("No platform found. Check OpenCL installation!")); + } + + std::optional plat; + for (auto&& p : platforms) { + LIMA_LOG(trace, proc) << "Plat: " << p.name(); + if (p.check_version(1, 2)) { + plat = p; + } + } + if (!plat) { + LIMA_THROW_EXCEPTION(lima::runtime_error("No OpenCL 1.2 platform found.")); + } + + // Get the list of GPU devices of the platform + auto devices = plat->devices(CL_DEVICE_TYPE_GPU); + if (devices.empty()) { + LIMA_THROW_EXCEPTION(lima::runtime_error("No device found. Check OpenCL installation!")); + } + bcl::device device = devices[proc_params.gpu.device_idx]; + + // Getting information about used queue and device + const size_t max_compute_units = device.get_info(); + const size_t max_work_group_size = device.get_info(); + + LIMA_LOG(trace, proc) << "Dev: " << device.name(); + + // Create OCL context + bcl::context context(device); + + // Create OCL queue + bcl::command_queue queue(context, context.get_device()); + + // Limit the number of log traces given the order of magnitude of m_nb_frames + const unsigned int log_frame_limiter = order_of_magnitude_base10(acq_params.xfer.time_slice.count, 1); + + src.emplace(m_graph, [this, log_frame_limiter, stop = false](tbb::flow_control& fc) mutable { + frame_t res; + + // If input is flagged to stop, return early + if (stop) { + fc.stop(); + return res; + } + + // Poll for available frames + while (!m_fifo.pop(res)) + ; + + if (!(res.metadata.idx % log_frame_limiter)) + LIMA_LOG(trace, proc) << "Processing thread got frame " << res.metadata.idx; + + // If the frame is final, flag the input to stop next run + if (res.metadata.is_final) { + LIMA_LOG(trace, proc) << "Processing thread got final frame"; + stop = true; + } + + m_nb_frames_source++; + + return res; + }); + + // Peak finder node + fai::kernel_params params = fai::read_kernel_params( + proc_params.jfrau.gain_path, proc_params.jfrau.pedestal_path, proc_params.fai.error_model, + proc_params.fai.variance_path, proc_params.fai.mask_path, proc_params.fai.dark_path, + proc_params.fai.dark_variance_path, proc_params.fai.flat_path, proc_params.fai.solid_angle_path, + proc_params.fai.polarization_path, proc_params.fai.absorption_path, proc_params.fai.dummy, + proc_params.fai.delta_dummy, proc_params.fai.normalization_factor, proc_params.fai.csr_path, + proc_params.fai.cutoff_clip, proc_params.fai.cycle, proc_params.fai.empty, + proc_params.fai.radius2d_path, proc_params.fai.radius1d_path, proc_params.fai.noise, + proc_params.fai.cutoff_pick, context, queue); + + LIMA_LOG(trace, proc) << "Peak finder kernel parameters parsed"; + + auto&& [dense_x, dense_y] = frame_info.dimensions(); + peak_finder.emplace(m_graph, dense_x, dense_y, params, proc_params.fai.cl_source_path, context, queue); + checkpoint_peak_finder.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_sparsified); + peak_counter.emplace(m_graph, tbb::flow::unlimited, [](auto ptr) { return ptr->peak_indices.size(); }); + peak_counters_buffer.emplace(m_graph, tbb::flow::unlimited, m_peak_counters_buffer, + m_peak_counters_buffer_mutex); + + int nb_bins = params.csr_indptr.size() - 1; + + // Dense HDF5 node + io_hdf5_dense.emplace( + m_graph, proc_params.saving_dense.base_path, proc_params.saving_dense.filename_format, + proc_params.saving_dense.filename_prefix, proc_params.saving_dense.filename_suffix, + proc_params.saving_dense.start_number, rank, proc_params.saving_dense.file_exists_policy, + acq_params.xfer.time_slice.count, proc_params.saving_dense.nb_frames_per_file, + proc_params.saving_dense.nb_frames_per_chunk, frame_info.dimensions(), frame_info.pixel_type(), + proc_params.saving_dense.compression); + checkpoint_hdf5_dense.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_dense_saved); + + // Sparse HDF5 node + io_hdf5_sparse.emplace( + m_graph, proc_params.saving_sparse.base_path, proc_params.saving_sparse.filename_format, + proc_params.saving_sparse.filename_prefix, proc_params.saving_sparse.filename_suffix, + proc_params.saving_sparse.start_number, rank, proc_params.saving_sparse.file_exists_policy, + acq_params.xfer.time_slice.count, proc_params.saving_sparse.nb_frames_per_file, + proc_params.saving_sparse.nb_frames_per_chunk, nb_bins, frame_info.dimensions(), + proc_params.saving_sparse.compression); + checkpoint_hdf5_sparse.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_sparse_saved); + + // ROI counters node (optional) + if (!proc_params.counters.rois.empty()) { + roi_counters.emplace(m_graph, tbb::flow::unlimited, proc_params.counters.rois); + roi_counters_buffer.emplace(m_graph, tbb::flow::unlimited, m_roi_counters_buffer, + m_roi_counters_buffer_mutex); + roi_counters_checkpoint.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_counters); + } + + // Graph topology + tbb::flow::make_edge(*src, *peak_finder); + tbb::flow::make_edge(*peak_finder, *io_hdf5_sparse); + tbb::flow::make_edge(*peak_finder, *peak_counter); + tbb::flow::make_edge(*peak_counter, *peak_counters_buffer); + tbb::flow::make_edge(*peak_finder, *checkpoint_peak_finder); + tbb::flow::make_edge(*io_hdf5_sparse, *checkpoint_hdf5_sparse); + + tbb::flow::make_edge(*src, *io_hdf5_dense); + tbb::flow::make_edge(*io_hdf5_dense, *checkpoint_hdf5_dense); + + if (roi_counters) { + tbb::flow::make_edge(*src, *roi_counters); + tbb::flow::make_edge(*roi_counters, *roi_counters_buffer); + tbb::flow::make_edge(*roi_counters_buffer, *roi_counters_checkpoint); + } + + LIMA_LOG(trace, proc) << "Processing constructed"; + } + + /// Activate the processing (start poping data from the queue) + void activate() { src->activate(); } + + /// Returns when the processing has finished + void wait_for_all() { m_graph.wait_for_all(); } + + /// Abort the pipeline + void abort() { m_task_group_context.cancel_group_execution(); } + + /// Push the given frame to the FIFO of frames to process + void process(input_t const& frm) + { + if (frm.metadata.is_final) + while (!m_fifo.push(frm)) + ; + else if (!m_fifo.push(frm)) + LIMA_LOG(error, det) << "Buffer overrun: frame #" << frm.metadata.idx; + } + + /// Returns the progress counters + counters get_counters() const + { + return {(int) m_nb_frames_source, (int) m_nb_frames_reconstructed, (int) m_nb_frames_counters, + (int) m_nb_frames_sparsified, (int) m_nb_frames_sparse_saved, (int) m_nb_frames_dense_saved}; + } + + /// Pop a given number of ROI counters + std::vector pop_roi_counters(std::size_t nb_frames = 1) + { + std::vector res; + + // If ROI counters buffer is empty, returns early + if (m_roi_counters_buffer.empty()) + return res; + + const std::lock_guard lock(m_roi_counters_buffer_mutex); + + std::size_t nb_elements = std::min(nb_frames, m_roi_counters_buffer.size()); + res.resize(nb_elements); + + // Copy to results and erase from buffer + std::copy_n(m_roi_counters_buffer.begin(), nb_elements, res.begin()); + m_roi_counters_buffer.erase_begin(nb_elements); + + return res; + } + + /// Pop a given number of peak counters + std::vector pop_peak_counters(std::size_t nb_frames = 1) + { + std::vector res; + + // If peak counters buffer is empty, returns early + if (m_peak_counters_buffer.empty()) + return res; + + const std::lock_guard lock(m_peak_counters_buffer_mutex); + + std::size_t nb_elements = std::min(nb_frames, m_peak_counters_buffer.size()); + res.resize(nb_elements); + + // Copy to results and erase from buffer + std::copy_n(m_peak_counters_buffer.begin(), nb_elements, res.begin()); + m_peak_counters_buffer.erase_begin(nb_elements); + + return res; + } + + /// Returns frame for the given index + frame_t get_frame(std::size_t frame_idx = -1) const + { + frame_t res; + + // If frames buffer is empty, returns early + if (m_frames_buffer.empty()) + return res; + + const std::lock_guard lock(m_frames_buffer_mutex); + + // If frame_idx == -1 returns the latest frame in the buffer + if (frame_idx == -1) + res = m_frames_buffer.back(); + else { + auto it = std::find_if(m_frames_buffer.begin(), m_frames_buffer.end(), + [frame_idx](frame_t const& frm) { return frame_idx == frm.metadata.idx; }); + if (it != m_frames_buffer.end()) + res = *it; + } + + return res; + } + + private: + // Queue of frame (runtime sized) + boost::lockfree::spsc_queue m_fifo; + + // TBB Flow Graph + tbb::task_arena m_arena; + tbb::task_group_context m_task_group_context; + + tbb::flow::graph m_graph; + std::optional> src; + std::optional> frames_buffer; + + std::optional peak_finder; + std::optional>> checkpoint_peak_finder; + std::optional, int, tbb::flow::lightweight>> + peak_counter; + std::optional> peak_counters_buffer; + + std::optional> io_hdf5_dense; + std::optional> checkpoint_hdf5_dense; + std::optional io_hdf5_sparse; + std::optional> checkpoint_hdf5_sparse; + + std::optional> roi_counters; + std::optional> roi_counters_buffer; + std::optional> roi_counters_checkpoint; + + // Counters + std::atomic_int m_nb_frames_source = 0; + std::atomic_int m_nb_frames_reconstructed = 0; + std::atomic_int m_nb_frames_counters = 0; + std::atomic_int m_nb_frames_sparsified = 0; + std::atomic_int m_nb_frames_sparse_saved = 0; + std::atomic_int m_nb_frames_dense_saved = 0; + + // ROI counters + boost::circular_buffer m_roi_counters_buffer; + std::mutex m_roi_counters_buffer_mutex; + + // Peak counters + boost::circular_buffer m_peak_counters_buffer; + std::mutex m_peak_counters_buffer_mutex; + + // Frames + boost::circular_buffer m_frames_buffer; + mutable std::mutex m_frames_buffer_mutex; + }; + + // Pimpl boilerplate + pipeline::pipeline(int rank, hw::frame_info const& acq_info, hw::acq_params const& acq_params, + proc_params_t const& proc_params) : + m_pimpl(std::make_unique(rank, acq_info, acq_params, proc_params)) + { + } + + pipeline::pipeline(pipeline&&) = default; + + pipeline::~pipeline() = default; + + pipeline& pipeline::operator=(pipeline&&) = default; + + void pipeline::activate() { m_pimpl->activate(); } + + void pipeline::wait_for_all() { m_pimpl->wait_for_all(); } + + void pipeline::abort() { m_pimpl->abort(); } + + void pipeline::process(input_t const& frm) { m_pimpl->process(frm); } + + counters pipeline::get_counters() const { return m_pimpl->get_counters(); } + + std::vector pipeline::pop_roi_counters(std::size_t nb_frames) + { + return m_pimpl->pop_roi_counters(nb_frames); + } + + std::vector pipeline::pop_peak_counters(std::size_t nb_frames) + { + return m_pimpl->pop_peak_counters(nb_frames); + } + + pipeline::frame_t pipeline::get_frame(std::size_t frame_idx) const { return m_pimpl->get_frame(frame_idx); } + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/tango/CMakeLists.txt b/processings/smx/tango/CMakeLists.txt new file mode 100644 index 00000000..86835cc0 --- /dev/null +++ b/processings/smx/tango/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright (C) 2021 Alejandro Homs Puron, ESRF. + +# Use, modification and distribution is subject to the Boost Software +# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +add_subdirectory(cpp) diff --git a/processings/smx/tango/cpp/CMakeLists.txt b/processings/smx/tango/cpp/CMakeLists.txt new file mode 100644 index 00000000..fc5bb65a --- /dev/null +++ b/processings/smx/tango/cpp/CMakeLists.txt @@ -0,0 +1,21 @@ +# Copyright (C) 2018 Samuel Debionne, ESRF. + +# Use, modification and distribution is subject to the Boost Software +# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +add_library(pipeline_smx_tango STATIC + processing.cpp + processing_class.cpp +) + +target_include_directories(pipeline_smx_tango + PUBLIC "$" + PUBLIC "$" +) + +target_link_libraries(pipeline_smx_tango PUBLIC + lima_core + lima_tango + pipeline_smx +) diff --git a/processings/smx/tango/cpp/conda/CMakeLists.txt b/processings/smx/tango/cpp/conda/CMakeLists.txt new file mode 100644 index 00000000..98fbd62e --- /dev/null +++ b/processings/smx/tango/cpp/conda/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright (C) 2021 Samuel Debionne, ESRF. + +# Use, modification and distribution is subject to the Boost Software +# License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +# http://www.boost.org/LICENSE_1_0.txt) + +cmake_minimum_required(VERSION 3.1) + +# Set path to additional cmake scripts +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) + +find_package(Threads REQUIRED) +find_package(Tango REQUIRED) +find_package(OmniORB REQUIRED) +find_package(ZeroMQ REQUIRED) + +add_library(pipeline_legacy_tango SHARED + processing.cpp + processing_class.cpp +) + +target_link_libraries(pipeline_legacy_tango PUBLIC + Threads::Threads + libzmq + ${OMNIORB4_LIBRARIES_sh} + Tango::Tango + Boost::program_options + Boost::json + simulator_ctrl + simulator_recv +) diff --git a/processings/smx/tango/cpp/conda/build.sh b/processings/smx/tango/cpp/conda/build.sh new file mode 100644 index 00000000..75e7fe33 --- /dev/null +++ b/processings/smx/tango/cpp/conda/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cmake -Bbuild -H. -G "Ninja" -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_FIND_ROOT_PATH=$PREFIX -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=ONLY -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY +cmake --build build --target install diff --git a/processings/smx/tango/cpp/conda/meta.yaml b/processings/smx/tango/cpp/conda/meta.yaml new file mode 100644 index 00000000..67e99998 --- /dev/null +++ b/processings/smx/tango/cpp/conda/meta.yaml @@ -0,0 +1,46 @@ +# Strip the 'v' from the version tag +{% if GIT_DESCRIBE_TAG is defined %} +{% set version = GIT_DESCRIBE_TAG[1:] %} +{% else %} +{% set version = "0.0.0" %} +{% endif %} + +package: + name: lima2-tango-servers + version: {{ version }} + +source: + path: .. + +build: + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} + noarch: python + script: "{{ PYTHON }} -m pip install . -vv" + +requirements: + host: + - lima2 + - cpptango + build: + - cmake >=3.16 + - ninja + - {{ compiler('cxx') }} + run: + - lima2 + - cpptango + +test: + source_files: + - tests + requires: + - pytest + imports: + - Lima2.Ds + commands: + - pytest tests + +about: + home: https://github.com/esrf-bliss/Lima2 + license: GPL + license_file: LICENCE + summary: A library for the unified control of 2D detectors diff --git a/processings/smx/tango/cpp/processing.cpp b/processings/smx/tango/cpp/processing.cpp new file mode 100644 index 00000000..1849aa9f --- /dev/null +++ b/processings/smx/tango/cpp/processing.cpp @@ -0,0 +1,105 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include "processing.hpp" + +namespace lima::tango +{ +void processing::init_device() +{ + DEBUG_STREAM << "processing::init_device() create device " << device_name << std::endl; +} + +void processing::always_executed_hook() +{ + DEBUG_STREAM << "processing::always_executed_hook() " << device_name << std::endl; + + // code always executed before all requests +} + +void processing::read_attr_hardware(TANGO_UNUSED(vector& attr_list)) +{ + DEBUG_STREAM << "processing::read_attr_hardware(vector &attr_list) entering... " << std::endl; + + // Add your own code +} + +void processing::write_attr_hardware(TANGO_UNUSED(vector& attr_list)) +{ + DEBUG_STREAM << "control::write_attr_hardware(vector &attr_list) entering... " << std::endl; + + // Add your own code +} + +void processing::add_dynamic_attributes() +{ + // Add your own code to create and add dynamic attributes if any +} + +struct roi_counter +{ + std::size_t frame_idx; + float min; + float max; + float avg; + float std; +}; + +Tango::DevEncoded processing::pop_roi_counters(int nb_frames) const +{ + auto counters = m_proc->pop_roi_counters(nb_frames); + + // Denormalize ROI counters data structure + std::vector buffer; + buffer.reserve(counters.size() * 5); + for (auto&& counter : counters) + for (auto&& roi : counter.counters) + buffer.emplace_back(roi_counter{counter.frame_idx, roi.m_min, roi.m_max, roi.m_avg, roi.m_std}); + + return make_devencoded(buffer); +} + +Tango::DevVarLongArray processing::pop_peak_counters(int nb_frames) const +{ + auto counters = m_proc->pop_peak_counters(nb_frames); + + Tango::DevVarLongArray res; + res.length(counters.size()); + + for (std::size_t i = 0; i < counters.size(); i++) + res[i] = counters[i]; + + return res; +} + +Tango::DevEncoded processing::get_frame(std::size_t frame_idx) const +{ + auto frm = m_proc->get_frame(frame_idx); + + return make_devencoded(frm); +} + +void processing::add_dynamic_commands() +{ + // Add your own code to create and add dynamic commands if any +} + +// Additional Methods + +} // namespace lima::tango diff --git a/processings/smx/tango/cpp/processing.hpp b/processings/smx/tango/cpp/processing.hpp new file mode 100644 index 00000000..cd2845fa --- /dev/null +++ b/processings/smx/tango/cpp/processing.hpp @@ -0,0 +1,85 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#include + +namespace lima::tango +{ +class processing : public TANGO_BASE_CLASS +{ + public: + using processing_t = lima::processing::pipelines::smx::pipeline; + using proc_params_t = typename processing_t::proc_params_t; + + // Constructors and destructors + processing(Tango::DeviceClass* cl, const char* name, std::shared_ptr proc) : + TANGO_BASE_CLASS(cl, name), m_proc(proc) + { + init_device(); + } + processing(Tango::DeviceClass* cl, const char* name, const char* description, std::shared_ptr proc) : + TANGO_BASE_CLASS(cl, name, description), m_proc(proc) + { + init_device(); + } + ~processing() { delete_device(); } + + /// Miscellaneous methods + ///{ + /// Will be called at device destruction or at init command + void delete_device() { DEBUG_STREAM << "processing::delete_device() " << device_name << std::endl; } + + /// Initialize the device + void init_device() override; + + /// Always executed method before execution command method + void always_executed_hook(); + ///} + + /// Attribute related methods + ///{ + + /// Hardware acquisition for attributes + virtual void read_attr_hardware(vector& attr_list); + + /// Hardware writing for attributes + virtual void write_attr_hardware(vector& attr_list); + + /// Add dynamic attributes if any + void add_dynamic_attributes(); + ///} + + /// Command related methods + ///{ + + /// Returns the progress counters + auto counters() const { return m_proc->get_counters(); } + + /// Pop ROI counters from the buffer + Tango::DevEncoded pop_roi_counters(int nb_frames) const; + + /// Pop peak counters from the buffer + Tango::DevVarLongArray pop_peak_counters(int nb_frames) const; + + /// Returns a frame for a given index + Tango::DevEncoded get_frame(std::size_t frame_idx) const; + + /// Add dynamic commands if any + void add_dynamic_commands(); + + ///} + + private: + std::shared_ptr m_proc; +}; + +// Additional Classes Definitions + +} // namespace lima::tango diff --git a/processings/smx/tango/cpp/processing_class.cpp b/processings/smx/tango/cpp/processing_class.cpp new file mode 100644 index 00000000..3667c0e4 --- /dev/null +++ b/processings/smx/tango/cpp/processing_class.cpp @@ -0,0 +1,331 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "processing_class.hpp" + +namespace lima::tango +{ +processing_class* processing_class::_instance = nullptr; + +processing_class::processing_class(std::string name) : device_class(name) +{ + cout2 << "Entering processing_class constructor" << std::endl; + + set_default_property(); + write_class_property(); + + cout2 << "Leaving processing_class constructor" << std::endl; +} + +processing_class* processing_class::init(std::string const& name) +{ + if (_instance == nullptr) { + try { + _instance = new processing_class(name); + } catch (std::bad_alloc&) { + throw; + } + } + return _instance; +} + +processing_class* processing_class::instance() +{ + if (_instance == nullptr) { + std::cerr << "Class is not initialised !" << std::endl; + exit(-1); + } + return _instance; +} + +//========================================= +// Define classes for attributes +//========================================= + +//========================================= +// Define classes for commands +//========================================= + +class pop_roi_counters_command : public Tango::Command +{ + public: + pop_roi_counters_command() : + Tango::Command("popRoiCounters", Tango::DEV_LONG, Tango::DEV_ENCODED, "Max number of frame", + "A DevEncoded structured array of ROI statistics [frame_idx, min, max, avg, std]", + Tango::OPERATOR) + { + } + + CORBA::Any* execute(Tango::DeviceImpl* dev, const CORBA::Any& in_any) override + { + cout2 << "pop_roi_counters_command::execute(): arrived" << std::endl; + + Tango::DevLong nb_frames; + extract(in_any, nb_frames); + + Tango::DevEncoded res = static_cast(dev)->pop_roi_counters(nb_frames); + + return insert(new Tango::DevEncoded(res)); + } +}; + +class pop_peak_counters_command : public Tango::Command +{ + public: + pop_peak_counters_command() : + Tango::Command("popPeakCounters", Tango::DEV_LONG, Tango::DEVVAR_LONGARRAY, "Max number of frame", + "An integer array of peaks", Tango::OPERATOR) + { + } + + CORBA::Any* execute(Tango::DeviceImpl* dev, const CORBA::Any& in_any) override + { + cout2 << "pop_peak_counters_command::execute(): arrived" << std::endl; + + Tango::DevLong nb_frames; + extract(in_any, nb_frames); + + Tango::DevVarLongArray res = static_cast(dev)->pop_peak_counters(nb_frames); + + return insert(new Tango::DevVarLongArray(res)); + } +}; + +class get_frame_command : public Tango::Command +{ + public: + get_frame_command() : + Tango::Command("getFrame", Tango::DEV_LONG, Tango::DEV_ENCODED, "Frame Index", "A DevEncoded image", + Tango::OPERATOR) + { + } + + CORBA::Any* execute(Tango::DeviceImpl* dev, const CORBA::Any& in_any) override + { + cout2 << "get_frame_command::execute(): arrived" << std::endl; + + Tango::DevLong frame_idx; + extract(in_any, frame_idx); + + Tango::DevEncoded res = static_cast(dev)->get_frame(frame_idx); + + return insert(new Tango::DevEncoded(res)); + } +}; + +//=================================================================== +// Properties management +//=================================================================== + +Tango::DbDatum processing_class::get_class_property(std::string& prop_name) +{ + for (unsigned int i = 0; i < cl_prop.size(); i++) + if (cl_prop[i].name == prop_name) + return cl_prop[i]; + // if not found, returns an empty DbDatum + return Tango::DbDatum(prop_name); +} + +Tango::DbDatum processing_class::get_default_device_property(string& prop_name) +{ + for (unsigned int i = 0; i < dev_def_prop.size(); i++) + if (dev_def_prop[i].name == prop_name) + return dev_def_prop[i]; + // if not found, return an empty DbDatum + return Tango::DbDatum(prop_name); +} + +Tango::DbDatum processing_class::get_default_class_property(std::string& prop_name) +{ + for (unsigned int i = 0; i < cl_def_prop.size(); i++) + if (cl_def_prop[i].name == prop_name) + return cl_def_prop[i]; + // if not found, return an empty DbDatum + return Tango::DbDatum(prop_name); +} + +void processing_class::set_default_property() +{ + // Set Default Class Properties + + // Set Default device Properties +} + +void processing_class::write_class_property() +{ + // First time, check if database used + if (Tango::Util::_UseDb == false) + return; + + Tango::DbData data; + std::string classname = get_name(); + std::string header; + std::string::size_type start, end; + + // Put title + Tango::DbDatum title("ProjectTitle"); + std::string str_title("LIMA2"); + title << str_title; + data.push_back(title); + + // Put Description + Tango::DbDatum description("Description"); + std::vector str_desc; + str_desc.push_back("LIMA2 Processing Class"); + description << str_desc; + data.push_back(description); + + // Put inheritance + Tango::DbDatum inher_datum("InheritedFrom"); + std::vector inheritance; + inheritance.push_back("TANGO_BASE_CLASS"); + inher_datum << inheritance; + data.push_back(inher_datum); + + // Call database and and values + get_db_class()->put_property(data); +} + +//=================================================================== +// Factory methods +//=================================================================== + +void processing_class::device_factory(const char* name, std::shared_ptr proc) +{ + processing* dev = new processing(this, name, proc); + + // Add it into the device list + device_list.push_back(dev); + + // Export devices to the outside world + if ((Tango::Util::_UseDb == true) && (Tango::Util::_FileDb == false)) + export_device(dev); + else + export_device(dev, dev->get_name().c_str()); +} + +//void processing_class::device_factory(const Tango::DevVarStringArray* devlist_ptr) +//{ +// // create devices and add it into the device list +// for (unsigned long i = 0; i < devlist_ptr->length(); i++) { +// cout4 << "device name : " << (*devlist_ptr)[i].in() << std::endl; +// device_list.push_back(new device_t(this, (*devlist_ptr)[i])); +// } +// +// // manage dynamic attributes if any +// erase_dynamic_attributes(devlist_ptr, get_class_attr()->get_attr_list()); +// +// // export devices to the outside world +// for (unsigned long i = 1; i <= devlist_ptr->length(); i++) { +// // add dynamic attributes if any +// device_t* dev = static_cast(device_list[device_list.size() - i]); +// dev->add_dynamic_attributes(); +// +// // check before if database used. +// if ((tango::util::_usedb == true) && (tango::util::_filedb == false)) +// export_device(dev); +// else +// export_device(dev, dev->get_name().c_str()); +// } +//} + +void processing_class::attribute_factory(std::vector& att_list) +{ + // Add counters JSON attribute + { + using counters_attr_t = JsonReadAttr; + counters_attr_t* attr = new counters_attr_t("counters", Tango::DEV_STRING, Tango::READ); + + Tango::UserDefaultAttrProp attr_prop; + + attr_prop.description = "Processing counters"; + attr_prop.label = "counters"; + + attr->set_default_properties(attr_prop); + + att_list.push_back(attr); + } + + // Create a list of static attributes + create_static_attribute_list(get_class_attr()->get_attr_list()); +} + +void processing_class::pipe_factory() {} + +void processing_class::command_factory() +{ + command_list.emplace_back(new pop_roi_counters_command()); + command_list.emplace_back(new pop_peak_counters_command()); + command_list.emplace_back(new get_frame_command()); + + // TODO Add get_frame +} + +//=================================================================== +// Dynamic attributes related methods +//=================================================================== + +void processing_class::create_static_attribute_list(std::vector& att_list) +{ + for (unsigned long i = 0; i < att_list.size(); i++) { + std::string att_name(att_list[i]->get_name()); + transform(att_name.begin(), att_name.end(), att_name.begin(), ::tolower); + defaultAttList.push_back(att_name); + } + + cout2 << defaultAttList.size() << " attributes in default list" << std::endl; +} + +void processing_class::erase_dynamic_attributes(const Tango::DevVarStringArray* devlist_ptr, + std::vector& att_list) +{ + Tango::Util* tg = Tango::Util::instance(); + + for (unsigned long i = 0; i < devlist_ptr->length(); i++) { + Tango::DeviceImpl* dev_impl = tg->get_device_by_name(((std::string)(*devlist_ptr)[i]).c_str()); + device_t* dev = static_cast(dev_impl); + + std::vector& dev_att_list = dev->get_device_attr()->get_attribute_list(); + for (auto ite_att = dev_att_list.begin(); ite_att != dev_att_list.end(); ++ite_att) { + std::string att_name((*ite_att)->get_name_lower()); + if ((att_name == "state") || (att_name == "status")) + continue; + auto ite_str = find(defaultAttList.begin(), defaultAttList.end(), att_name); + if (ite_str == defaultAttList.end()) { + cout2 << att_name << " is a UNWANTED dynamic attribute for device " << (*devlist_ptr)[i] << std::endl; + Tango::Attribute& att = dev->get_device_attr()->get_attr_by_name(att_name.c_str()); + dev->remove_attribute(att_list[att.get_attr_idx()], true, false); + --ite_att; + } + } + } +} + +Tango::Attr* processing_class::get_attr_object_by_name(std::vector& att_list, std::string attname) +{ + for (auto&& att : att_list) + if (att->get_name() == attname) + return att; + + // Attr does not exist + return NULL; +} + +} // namespace lima::tango diff --git a/processings/smx/tango/cpp/processing_class.hpp b/processings/smx/tango/cpp/processing_class.hpp new file mode 100644 index 00000000..dddc33aa --- /dev/null +++ b/processings/smx/tango/cpp/processing_class.hpp @@ -0,0 +1,113 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include + +#include + +#include "processing.hpp" + +namespace lima::tango +{ +/// The processing_class singleton definition +class processing_class : public device_class +{ + public: + using processing_t = lima::processing::pipelines::smx::pipeline; + using device_t = processing; + + // write class properties data members + Tango::DbData cl_prop; + Tango::DbData cl_def_prop; + Tango::DbData dev_def_prop; + + // Non-copiable + processing_class(const processing_class&) = delete; + processing_class& operator=(const processing_class&) = delete; + processing_class(processing_class&&) = delete; + processing_class& operator=(processing_class&&) = delete; + + /// Create the object if not already done.Otherwise, just return a pointer to the object + /// + /// \param name The class name + static processing_class* init(std::string const& name); + + /// Singleton access, check if object already created, and return a pointer to the object + static processing_class* instance(); + + /// Clean up + ~processing_class() { _instance = nullptr; } + + /// Get the class property for specified name + Tango::DbDatum get_class_property(std::string&) override; + + /// Return the default value for device property + Tango::DbDatum get_default_device_property(std::string&) override; + + /// Return the default value for class property + Tango::DbDatum get_default_class_property(std::string&) override; + + /// Create the device object(s) and store them in the device list + void device_factory(const char* name, std::shared_ptr proc); + + protected: + processing_class(std::string name); + + /// Create the command object(s) and store them in the command list + void command_factory(); + + /// Create the attribute object(s) and store them in the attribute list + void attribute_factory(vector&); + + /// Create the pipe object(s) and store them in the pipe list + void pipe_factory(); + + /// Properties management + ///{ + + /// Set class description fields as property in database + void write_class_property(); + + /// Set default property (class and device) for wizard. + /// For each property, add to wizard property name and description. + /// If default value has been set, add it to wizard property and + /// store it in a DbDatum. + void set_default_property(); + + ///} + + std::string get_cvstag(); + std::string get_cvsroot(); + + static processing_class* _instance; //!< Singleton instance + + private: + /// Factory methods + + /// Create the device object(s) and store them in the device list + void device_factory(const Tango::DevVarStringArray*) override {} + + /// Create the a list of static attributes + /// + /// \param att_list the ceated attribute list + void create_static_attribute_list(std::vector&); + + /// Delete the dynamic attributes if any + /// + /// \param devlist_ptr the device list pointer + /// /// \param list of all attributes + void erase_dynamic_attributes(const Tango::DevVarStringArray*, std::vector&); + + /// Returns Tango::Attr * object found by name + Tango::Attr* get_attr_object_by_name(std::vector& att_list, std::string attname); + + std::vector defaultAttList; //!< Default attibute list +}; + +} // namespace lima::tango -- GitLab From 0d75e64fff72fc495dff39c1c65a474f063015ed Mon Sep 17 00:00:00 2001 From: Alejandro Homs Puron Date: Fri, 4 Feb 2022 20:28:29 +0100 Subject: [PATCH 4/9] [PROC][SMX] Add static data to sparse files: bin radius (1d) & masked pixel radius (2d) --- .../pipelines/smx/io_hdf5_sparse_node.hpp | 9 +- .../pipelines/smx/sparse_static_data.hpp | 121 ++++++++++++++++++ processings/smx/src/pipeline_smx.cpp | 6 +- 3 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 processings/smx/include/lima/processing/pipelines/smx/sparse_static_data.hpp diff --git a/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp b/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp index d1fe0039..fd29169b 100644 --- a/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp +++ b/processings/smx/include/lima/processing/pipelines/smx/io_hdf5_sparse_node.hpp @@ -45,11 +45,13 @@ namespace processing::pipelines writer_body(std::filesystem::path base_path, std::string filename_format, std::string filename_prefix, std::string filename_suffix, int start_number, int rank, io::file_exists_policy_enum file_exists, int nb_frames, int nb_frames_per_file, - int nb_frames_per_chunk, int nb_bins, dimensions_t const& dimensions, + int nb_frames_per_chunk, int nb_bins, dimensions_t const& dimensions, lima::frame radius1d, + lima::frame radius2d_mask, io::h5::compression_enum comp = io::h5::compression_enum::bshuf_lz4) : m_writer(std::make_shared(base_path, filename_format, filename_prefix, filename_suffix, start_number, rank, file_exists, nb_frames, nb_frames_per_file, - nb_frames_per_chunk, nb_bins, dimensions, comp)), + nb_frames_per_chunk, nb_bins, dimensions, radius1d, + radius2d_mask, comp)), m_nb_frames_per_chunk(nb_frames_per_chunk) { } @@ -74,13 +76,14 @@ namespace processing::pipelines std::string filename_prefix, std::string filename_suffix, int start_number, int rank, io::file_exists_policy_enum file_exists, int nb_frames, int nb_frames_per_file, int nb_frames_per_chunk, int nb_bins, dimensions_t const& dimensions, + lima::frame radius1d, lima::frame radius2d_mask, io::h5::compression_enum comp = io::h5::compression_enum::bshuf_lz4) : parent_t(g), sequencer_node(g, [](auto&& frame) -> int { return frame->metadata.idx; }), writer_node(g, 1 /*serial for now*/, writer_body(base_path, filename_format, filename_prefix, filename_suffix, start_number, rank, file_exists, nb_frames, nb_frames_per_file, nb_frames_per_chunk, nb_bins, - dimensions, comp)) + dimensions, radius1d, radius2d_mask, comp)) { tbb::flow::make_edge(sequencer_node, writer_node); typename parent_t::input_ports_type input_tuple(sequencer_node); diff --git a/processings/smx/include/lima/processing/pipelines/smx/sparse_static_data.hpp b/processings/smx/include/lima/processing/pipelines/smx/sparse_static_data.hpp new file mode 100644 index 00000000..c467ef6b --- /dev/null +++ b/processings/smx/include/lima/processing/pipelines/smx/sparse_static_data.hpp @@ -0,0 +1,121 @@ +// Copyright (C) 2018 Alejandro Homs, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include +#include +#include +#include + +#include + +namespace lima +{ +namespace processing::pipelines +{ + namespace smx + { + namespace detail + { + lima::frame read_h5_file(lima::pixel_enum pixel, int dataset_dims, std::filesystem::path file_path, + lima::io::h5::path const& data_path = "/data") + { + LIMA_LOG(trace, proc) << "Loading file: " << file_path << " [" << data_path.c_str() << "] ..."; + + // load data from H5 file + hid_t file_id = H5Fopen(file_path.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) + LIMA_THROW_EXCEPTION(lima::runtime_error("Failed to open HDF5 file")); + + // Open an existing dataset + hid_t dataset_id = H5Dopen2(file_id, data_path.c_str(), H5P_DEFAULT); + if (dataset_id < 0) + LIMA_THROW_EXCEPTION(lima::runtime_error("Failed to open HDF5 dataset")); + + // Check the dimensions + assert((dataset_dims > 0) && (dataset_dims <= 3)); + hid_t dataspace_id = H5Dget_space(dataset_id); + const int ndims = H5Sget_simple_extent_ndims(dataspace_id); + if (ndims != dataset_dims) + LIMA_THROW_EXCEPTION( + lima::invalid_argument("Invalid dataset number of dimensions: " + std::to_string(ndims))); + + hsize_t dims[ndims]; + H5Sget_simple_extent_dims(dataspace_id, dims, NULL); + + // Last dimension defines the nb of colums + const int last_dim = dataset_dims - 1; + auto cols = dims[last_dim]; + auto rows = + std::accumulate(dims, dims + last_dim, 1, [](hsize_t lhs, hsize_t rhs) { return lhs * rhs; }); + + // Allocate the image memory + lima::frame out(point_t(cols, rows), pixel); + auto&& view = lima::view(out); + auto ptr = boost::gil::apply_operation(view, [&](auto&& v) { return static_cast(&v(0, 0)); }); + + // Read the dataset + hid_t dtype_id = lima::io::h5::datatype(view); + herr_t res; + res = H5Dread(dataset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, ptr); + if (res < 0) + LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Dread error") << boost::errinfo_errno(res)); + + // Close the dataspace + res = H5Sclose(dataspace_id); + if (res < 0) + LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Sclose error") << boost::errinfo_errno(res)); + + // Close the dataset + res = H5Dclose(dataset_id); + if (res < 0) + LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Dclose error") << boost::errinfo_errno(res)); + + // Close the file + res = H5Fclose(file_id); + if (res < 0) + LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Fclose error") << boost::errinfo_errno(res)); + + LIMA_LOG(trace, proc) << "File loaded with " << dims[1] << "x" << dims[0] << " pixels"; + + return out; + } + } // namespace detail + + auto read_radius1d(std::filesystem::path radius1d_file_path, + lima::io::h5::path const& radius1d_data_path = "/data") + { + return detail::read_h5_file(pixel_enum::gray32f, 1, radius1d_file_path, radius1d_data_path); + } + + auto read_radius2d_mask(std::filesystem::path radius2d_file_path, std::filesystem::path mask_file_path, + lima::io::h5::path const& radius2d_data_path = "/data", + lima::io::h5::path const& mask_data_path = "/data") + { + auto mask_frame = detail::read_h5_file(pixel_enum::gray8, 2, mask_file_path, mask_data_path); + auto mask = boost::variant2::get(const_view(mask_frame)); + + auto out_frame = detail::read_h5_file(pixel_enum::gray32f, 2, radius2d_file_path, radius2d_data_path); + + auto&& mask_dims = mask.dimensions(); + auto&& out_dims = out_frame.dimensions(); + LIMA_LOG(trace, proc) << "Radius-2d loaded with " << out_dims.x << "x" << out_dims.y << " pixels"; + if (out_dims != mask_dims) + LIMA_THROW_EXCEPTION(std::runtime_error("Dimension mismatch between mask & radius-2d datas")); + + auto view = boost::variant2::get(lima::view(out_frame)); + auto mask_it = mask.begin(); + for (auto image_it = view.begin(); image_it != view.end(); ++image_it, ++mask_it) + if (*mask_it) + *image_it = NAN; + + return out_frame; + } + + } // namespace smx +} // namespace processing::pipelines +} // namespace lima diff --git a/processings/smx/src/pipeline_smx.cpp b/processings/smx/src/pipeline_smx.cpp index 0299641e..6025d3ec 100644 --- a/processings/smx/src/pipeline_smx.cpp +++ b/processings/smx/src/pipeline_smx.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -149,13 +150,16 @@ namespace processing::pipelines proc_params.saving_dense.compression); checkpoint_hdf5_dense.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_dense_saved); + auto radius = read_radius1d(proc_params.fai.radius1d_path); + auto mask = read_radius2d_mask(proc_params.fai.radius2d_path, proc_params.fai.mask_path); + // Sparse HDF5 node io_hdf5_sparse.emplace( m_graph, proc_params.saving_sparse.base_path, proc_params.saving_sparse.filename_format, proc_params.saving_sparse.filename_prefix, proc_params.saving_sparse.filename_suffix, proc_params.saving_sparse.start_number, rank, proc_params.saving_sparse.file_exists_policy, acq_params.xfer.time_slice.count, proc_params.saving_sparse.nb_frames_per_file, - proc_params.saving_sparse.nb_frames_per_chunk, nb_bins, frame_info.dimensions(), + proc_params.saving_sparse.nb_frames_per_chunk, nb_bins, frame_info.dimensions(), radius, mask, proc_params.saving_sparse.compression); checkpoint_hdf5_sparse.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_sparse_saved); -- GitLab From 055f98b55711d3e44fccbd709943645d63471dea Mon Sep 17 00:00:00 2001 From: Samuel Debionne Date: Thu, 10 Feb 2022 09:48:27 +0100 Subject: [PATCH 5/9] [CMAKE] Change library prefix to lima_pipeline_ for processing pipelines --- processings/cuda/src/CMakeLists.txt | 2 +- processings/legacy/src/CMakeLists.txt | 2 +- processings/smx/src/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/processings/cuda/src/CMakeLists.txt b/processings/cuda/src/CMakeLists.txt index a74eae5c..fca9e6d8 100644 --- a/processings/cuda/src/CMakeLists.txt +++ b/processings/cuda/src/CMakeLists.txt @@ -34,7 +34,7 @@ target_link_libraries(pipeline_cuda # Set version and output name set_target_properties(pipeline_cuda PROPERTIES - OUTPUT_NAME "lima_${PROJECT_NAME}" + OUTPUT_NAME "lima_pipeline_${PROJECT_NAME}" VERSION "${PROJECT_VERSION}" SOVERSION "${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}") diff --git a/processings/legacy/src/CMakeLists.txt b/processings/legacy/src/CMakeLists.txt index f8823117..38c12a30 100644 --- a/processings/legacy/src/CMakeLists.txt +++ b/processings/legacy/src/CMakeLists.txt @@ -34,7 +34,7 @@ target_link_libraries(pipeline_legacy # Set version and output name set_target_properties(pipeline_legacy PROPERTIES - OUTPUT_NAME "lima_${PROJECT_NAME}" + OUTPUT_NAME "lima_pipeline_${PROJECT_NAME}" VERSION "${PROJECT_VERSION}" SOVERSION "${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}") diff --git a/processings/smx/src/CMakeLists.txt b/processings/smx/src/CMakeLists.txt index 59bd447e..98c3cc15 100644 --- a/processings/smx/src/CMakeLists.txt +++ b/processings/smx/src/CMakeLists.txt @@ -34,7 +34,7 @@ target_link_libraries(pipeline_smx # Set version and output name set_target_properties(pipeline_smx PROPERTIES - OUTPUT_NAME "lima_${PROJECT_NAME}" + OUTPUT_NAME "lima_pipeline_${PROJECT_NAME}" VERSION "${PROJECT_VERSION}" SOVERSION "${PACKAGE_VERSION_MAJOR}.${PACKAGE_VERSION_MINOR}") -- GitLab From aa95e64e3890bd1c055f9e2becaa14514abd3a6a Mon Sep 17 00:00:00 2001 From: Samuel Debionne Date: Thu, 10 Feb 2022 10:13:15 +0100 Subject: [PATCH 6/9] [TANGO][SIM] Add SMX processing variant --- detectors/simulator/tango/cpp/CMakeLists.txt | 2 +- .../simulator/tango/cpp/smx/class_factory.cpp | 26 +++++++++++++++ .../simulator/tango/cpp/smx/receiver.cpp | 32 +++++++++++++++++++ .../tango/cpp/smx/receiver_class_factory.cpp | 23 +++++++++++++ 4 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 detectors/simulator/tango/cpp/smx/class_factory.cpp create mode 100644 detectors/simulator/tango/cpp/smx/receiver.cpp create mode 100644 detectors/simulator/tango/cpp/smx/receiver_class_factory.cpp diff --git a/detectors/simulator/tango/cpp/CMakeLists.txt b/detectors/simulator/tango/cpp/CMakeLists.txt index b9387b1b..7e1fc25b 100644 --- a/detectors/simulator/tango/cpp/CMakeLists.txt +++ b/detectors/simulator/tango/cpp/CMakeLists.txt @@ -5,7 +5,7 @@ # http://www.boost.org/LICENSE_1_0.txt) # Processing pipelines -set(_processings legacy cuda) +set(_processings legacy cuda smx) # Tango DS all-in-one executables foreach(proc IN LISTS _processings) diff --git a/detectors/simulator/tango/cpp/smx/class_factory.cpp b/detectors/simulator/tango/cpp/smx/class_factory.cpp new file mode 100644 index 00000000..a7c87327 --- /dev/null +++ b/detectors/simulator/tango/cpp/smx/class_factory.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include "processing.hpp" +#include "processing_class.hpp" + +#include +#include + +#include +#include + +using control_t = lima::detectors::simulator::controller; +using receiver_t = lima::detectors::simulator::receiver; +using processing_t = lima::processing::pipelines::smx::pipeline; + +/// Create Tango Class singletons and store it in DServer object. +void Tango::DServer::class_factory() +{ + add_class(lima::tango::control_class::init("LimaSimulatorControl")); + add_class(lima::tango::receiver_class::init("LimaSimulatorSmxReceiver")); + add_class(lima::tango::processing_class::init("LimaProcessingSmx")); +} diff --git a/detectors/simulator/tango/cpp/smx/receiver.cpp b/detectors/simulator/tango/cpp/smx/receiver.cpp new file mode 100644 index 00000000..eb461196 --- /dev/null +++ b/detectors/simulator/tango/cpp/smx/receiver.cpp @@ -0,0 +1,32 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include "processing.hpp" +#include "processing_class.hpp" + +#include +#include + +#include +#include + +// Explicitely instantiate template +template class lima::tango::receiver; + +// Create receiver_class singleton and +// return it in a C function for Python usage +extern "C" { + +Tango::DeviceClass* _create_receiver_class(const char* name) +{ + return lima::tango::receiver_class::init(name); +} +} //extern "C" + +// Explicitely instantiate template for DeviceClass +template class lima::tango::receiver_class; \ No newline at end of file diff --git a/detectors/simulator/tango/cpp/smx/receiver_class_factory.cpp b/detectors/simulator/tango/cpp/smx/receiver_class_factory.cpp new file mode 100644 index 00000000..5f5ab998 --- /dev/null +++ b/detectors/simulator/tango/cpp/smx/receiver_class_factory.cpp @@ -0,0 +1,23 @@ +// Copyright (C) 2021 Samuel Debionne, ESRF. + +// Use, modification and distribution is subject to the Boost Software +// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#include "processing.hpp" +#include "processing_class.hpp" + +#include + +#include +#include + +using receiver_t = lima::detectors::simulator::receiver; +using processing_t = lima::processing::pipelines::smx::pipeline; + +/// Create Tango Class singletons and store it in DServer object. +void Tango::DServer::class_factory() +{ + add_class(lima::tango::receiver_class::init("LimaSimulatorSmxReceiver")); + add_class(lima::tango::processing_class::init("LimaProcessingSmx")); +} -- GitLab From 52fd3ba6108a81a5129a497903d5aa349e40f71a Mon Sep 17 00:00:00 2001 From: Samuel Debionne Date: Wed, 16 Feb 2022 12:05:04 +0100 Subject: [PATCH 7/9] [CORE] Split frame and frame_metadata Metadata are reused by sparse_frame --- include/lima/core/frame.hpp | 16 ++-------------- include/lima/core/frame_metadata.hpp | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 14 deletions(-) create mode 100644 include/lima/core/frame_metadata.hpp diff --git a/include/lima/core/frame.hpp b/include/lima/core/frame.hpp index 4069ea89..b931294b 100644 --- a/include/lima/core/frame.hpp +++ b/include/lima/core/frame.hpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -20,6 +19,7 @@ #include #include +#include namespace lima { @@ -154,19 +154,7 @@ class frame public: using attributes_t = std::pmr::unordered_map; - - class metadata_t - { - public: - using timestamp_t = - std::chrono::steady_clock::duration; // The steady time since the beginning of the acquisition - - std::size_t idx = 0; //!< Index in the frame sequence (starting from 0) - std::size_t recv_idx = 0; //!< Index in the frame sequence (starting from 0 and monotone) - timestamp_t timestamp; //!< Acquisition timestamp from camera backend - bool is_final = false; //!< true if the frame is the last one in the sequence - bool is_valid = true; //!< true if the frame is valid (e.g. no transmission error) - }; + using metadata_t = frame_metadata; metadata_t metadata; // + +namespace lima +{ +class frame_metadata +{ + public: + using timestamp_t = std::chrono::steady_clock::duration; // The steady time since the beginning of the acquisition + + std::size_t idx = 0; //!< Index in the frame sequence (starting from 0) + std::size_t recv_idx = 0; //!< Index in the frame sequence (starting from 0 and monotone) + timestamp_t timestamp; //!< Acquisition timestamp from camera backend + bool is_final = false; //!< true if the frame is the last one in the sequence + bool is_valid = true; //!< true if the frame is valid (e.g. no transmission error) +}; + +} // namespace lima -- GitLab From f210b8bc9af30f4e274b0259c819683025aa6a14 Mon Sep 17 00:00:00 2001 From: Alejandro Homs Puron Date: Thu, 3 Mar 2022 21:52:55 +0100 Subject: [PATCH 8/9] [PROC][SMX] Protect against input other than gray16 --- .../lima/processing/pipelines/smx/peak_finder_node.hpp | 6 +++++- processings/smx/src/pipeline_smx.cpp | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp b/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp index b282140c..fe656a31 100644 --- a/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp +++ b/processings/smx/include/lima/processing/pipelines/smx/peak_finder_node.hpp @@ -13,6 +13,8 @@ #include #include +#include + #include #include @@ -126,12 +128,14 @@ namespace processing::pipelines mutable fai::vector found_d; // Found (after peak find) }; - peak_finder_node(tbb::flow::graph& g, std::size_t width, std::size_t height, + peak_finder_node(tbb::flow::graph& g, std::size_t width, std::size_t height, pixel_enum pixel, fai::kernel_params const& params, std::filesystem::path const& cl_source_path, bcl::context& context, bcl::command_queue& queue) : parent_t(g, tbb::flow::serial /*serial fo now*/, peak_finder_body(width, height, params, cl_source_path, context, queue)) { + if (pixel != pixel_enum::gray16) + LIMA_THROW_EXCEPTION(lima::invalid_argument("peak_finder_node only supports gray16 pixel type")); } }; diff --git a/processings/smx/src/pipeline_smx.cpp b/processings/smx/src/pipeline_smx.cpp index 6025d3ec..d1633c95 100644 --- a/processings/smx/src/pipeline_smx.cpp +++ b/processings/smx/src/pipeline_smx.cpp @@ -132,7 +132,8 @@ namespace processing::pipelines LIMA_LOG(trace, proc) << "Peak finder kernel parameters parsed"; auto&& [dense_x, dense_y] = frame_info.dimensions(); - peak_finder.emplace(m_graph, dense_x, dense_y, params, proc_params.fai.cl_source_path, context, queue); + peak_finder.emplace(m_graph, dense_x, dense_y, frame_info.pixel_type(), params, + proc_params.fai.cl_source_path, context, queue); checkpoint_peak_finder.emplace(m_graph, tbb::flow::unlimited, m_nb_frames_sparsified); peak_counter.emplace(m_graph, tbb::flow::unlimited, [](auto ptr) { return ptr->peak_indices.size(); }); peak_counters_buffer.emplace(m_graph, tbb::flow::unlimited, m_peak_counters_buffer, -- GitLab From d7e4a551f57e0e9ab06e0499e284115f55afb545 Mon Sep 17 00:00:00 2001 From: Alejandro Homs Puron Date: Mon, 7 Mar 2022 15:52:10 +0100 Subject: [PATCH 9/9] [PROC][SMX] Add jungfrau photon_energy, normalizing gains --- .../lima/processing/fai/kernel_params.hpp | 15 +++++++++++++++ .../processing/pipelines/smx/params.describe.hpp | 6 +++++- .../lima/processing/pipelines/smx/params.hpp | 1 + processings/smx/src/pipeline_smx.cpp | 16 ++++++++-------- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/processings/common/fai/include/lima/processing/fai/kernel_params.hpp b/processings/common/fai/include/lima/processing/fai/kernel_params.hpp index 41400540..239d9b27 100644 --- a/processings/common/fai/include/lima/processing/fai/kernel_params.hpp +++ b/processings/common/fai/include/lima/processing/fai/kernel_params.hpp @@ -14,6 +14,7 @@ #include #include +#include #include @@ -49,6 +50,7 @@ namespace processing::fai vector gain; // JungFrau Gain/Pedestal vector pedestal; // + float photon_energy; // For Jungfrau photon counting error_model_enum error_model = error_model_enum::poissonian; // FAI vector variance; // Preprocessing vector mask; // @@ -135,6 +137,7 @@ namespace processing::fai // Read the input data / parameters of the openCL kernels inline kernel_params read_kernel_params(std::filesystem::path gain_path, // JungFrau Gain/Pedestal std::filesystem::path pedestal_path, // + float photon_energy, // error_model_enum error_model, // FAI std::filesystem::path variance_path, // std::filesystem::path mask_path, // Preprocessing @@ -160,6 +163,15 @@ namespace processing::fai detail::read_h5_dset(gain_path, res.gain, "/data", queue); detail::read_h5_dset(pedestal_path, res.pedestal, "/data", queue); + // If specified, correct the Jungfrau gains with the photon energy + if (photon_energy != 0.0) { + vector factor(1, context); + boost::compute::copy(&photon_energy, &photon_energy + 1, factor.begin(), queue); + boost::compute::vector factor_d(factor, queue); + BOOST_COMPUTE_CLOSURE(float, correct_gain, (float x), (factor_d), { return x / factor_d[0]; }); + boost::compute::transform(res.gain.begin(), res.gain.end(), res.gain.begin(), correct_gain, queue); + } + res.error_model = error_model; res.dummy = dummy; res.delta_dummy = delta_dummy; @@ -226,6 +238,8 @@ namespace processing::fai std::string pedestal_path; extract(gain_path, "gain_path"); extract(pedestal_path, "pedestal_path"); + float photon_energy; + extract(photon_energy, "photon_energy"); //extract(error_model, "error_model"); error_model_enum error_model = error_model_enum::azimuthal; float dummy; @@ -271,6 +285,7 @@ namespace processing::fai return read_kernel_params(gain_path, // JungFrau Gain/Pedestal pedestal_path, // + photon_energy, // error_model, // FAI variance_path, // mask_path, // Preprocessing diff --git a/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp b/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp index c9b4eefd..1f9728fd 100644 --- a/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp +++ b/processings/smx/include/lima/processing/pipelines/smx/params.describe.hpp @@ -60,7 +60,7 @@ namespace processing::pipelines (doc, "The index of the GPU device in the list of OpenCL 1.2 devices")) // clang-format on - BOOST_DESCRIBE_STRUCT(gain_pedestal_params, (), (gain_path, pedestal_path)) + BOOST_DESCRIBE_STRUCT(gain_pedestal_params, (), (gain_path, pedestal_path, photon_energy)) // clang-format off BOOST_ANNOTATE_MEMBER(gain_pedestal_params, gain_path, @@ -70,6 +70,10 @@ namespace processing::pipelines BOOST_ANNOTATE_MEMBER(gain_pedestal_params, pedestal_path, (desc, "todo"), (doc, "TODO")) + + BOOST_ANNOTATE_MEMBER(gain_pedestal_params, photon_energy, + (desc, "todo"), + (doc, "TODO")) // clang-format on BOOST_DESCRIBE_STRUCT(fai_params, (), diff --git a/processings/smx/include/lima/processing/pipelines/smx/params.hpp b/processings/smx/include/lima/processing/pipelines/smx/params.hpp index 475a3418..905e7830 100644 --- a/processings/smx/include/lima/processing/pipelines/smx/params.hpp +++ b/processings/smx/include/lima/processing/pipelines/smx/params.hpp @@ -48,6 +48,7 @@ namespace processing::pipelines { std::filesystem::path gain_path; // JungFrau Gain/Pedestal std::filesystem::path pedestal_path; // + float photon_energy = 0.0; // Photon energy in keV }; struct fai_params diff --git a/processings/smx/src/pipeline_smx.cpp b/processings/smx/src/pipeline_smx.cpp index d1633c95..dcd70634 100644 --- a/processings/smx/src/pipeline_smx.cpp +++ b/processings/smx/src/pipeline_smx.cpp @@ -120,14 +120,14 @@ namespace processing::pipelines // Peak finder node fai::kernel_params params = fai::read_kernel_params( - proc_params.jfrau.gain_path, proc_params.jfrau.pedestal_path, proc_params.fai.error_model, - proc_params.fai.variance_path, proc_params.fai.mask_path, proc_params.fai.dark_path, - proc_params.fai.dark_variance_path, proc_params.fai.flat_path, proc_params.fai.solid_angle_path, - proc_params.fai.polarization_path, proc_params.fai.absorption_path, proc_params.fai.dummy, - proc_params.fai.delta_dummy, proc_params.fai.normalization_factor, proc_params.fai.csr_path, - proc_params.fai.cutoff_clip, proc_params.fai.cycle, proc_params.fai.empty, - proc_params.fai.radius2d_path, proc_params.fai.radius1d_path, proc_params.fai.noise, - proc_params.fai.cutoff_pick, context, queue); + proc_params.jfrau.gain_path, proc_params.jfrau.pedestal_path, proc_params.jfrau.photon_energy, + proc_params.fai.error_model, proc_params.fai.variance_path, proc_params.fai.mask_path, + proc_params.fai.dark_path, proc_params.fai.dark_variance_path, proc_params.fai.flat_path, + proc_params.fai.solid_angle_path, proc_params.fai.polarization_path, + proc_params.fai.absorption_path, proc_params.fai.dummy, proc_params.fai.delta_dummy, + proc_params.fai.normalization_factor, proc_params.fai.csr_path, proc_params.fai.cutoff_clip, + proc_params.fai.cycle, proc_params.fai.empty, proc_params.fai.radius2d_path, + proc_params.fai.radius1d_path, proc_params.fai.noise, proc_params.fai.cutoff_pick, context, queue); LIMA_LOG(trace, proc) << "Peak finder kernel parameters parsed"; -- GitLab