Commit 03767d3f authored by Alejandro Homs Puron's avatar Alejandro Homs Puron
Browse files

[IO] Refactor HDF5/Nexus to open existing files/groups/datasets

parent 26c8a6e3
......@@ -40,19 +40,19 @@ namespace io
public:
file() = default;
file(std::filesystem::path const& name, const char* nx_class = "NXroot",
h5::path nx_default = "/entry_0000", unsigned flags = H5F_ACC_TRUNC, hid_t fcpl_id = H5P_DEFAULT,
hid_t fapl_id = H5P_DEFAULT) :
h5::file(name, flags, fcpl_id, fapl_id)
/// Construct a NEXUS file
static file create(std::filesystem::path const& name, const char* nx_class = "NXroot",
h5::path nx_default = "/entry_0000", unsigned flags = H5F_ACC_TRUNC,
hid_t fcpl_id = H5P_DEFAULT, hid_t fapl_id = H5P_DEFAULT)
{
using namespace detail;
using namespace std::string_literals;
return file(h5::file::create(name, flags, fcpl_id, fapl_id), name, nx_class, nx_default);
}
attrs.create("NX_class", nx_class);
attrs.create("creator", "LIMA2-1.0.0"s);
attrs.create("default", nx_default);
attrs.create("file_name", name.c_str());
attrs.create("file_time", format_current_time());
/// Opens an existing NEXUS file
static file open(std::filesystem::path const& name, const char* nx_class = "NXroot",
unsigned flags = H5F_ACC_RDONLY, hid_t fapl_id = H5P_DEFAULT)
{
return file(h5::file::open(name, flags, fapl_id), nx_class);
}
/// Construct a NEXUS entry
......@@ -85,18 +85,51 @@ namespace io
return res;
}
private:
file(h5::file h5_file, std::filesystem::path const& name, const char* nx_class, h5::path nx_default) :
h5::file(std::move(h5_file))
{
using namespace detail;
using namespace std::string_literals;
attrs.create("NX_class", nx_class);
attrs.create("creator", "LIMA2-1.0.0"s);
attrs.create("default", nx_default);
attrs.create("file_name", name.c_str());
attrs.create("file_time", format_current_time());
}
file(h5::file h5_file, const char* nx_class) : h5::file(std::move(h5_file))
{
if (nx_class && nx_class[0] && (attrs["NX_class"].string() != nx_class))
LIMA_THROW_EXCEPTION(lima::io_error("HDF5 NX_class Attribute mismatch"));
}
};
class group : public h5::group
{
public:
template <typename Location>
group(Location const& loc, h5::path const& name, const char* nx_class, hid_t lcpl_id = H5P_DEFAULT,
hid_t gcpl_id = H5P_DEFAULT, hid_t gapl_id = H5P_DEFAULT) :
h5::group(loc, name.c_str(), lcpl_id, gcpl_id, gapl_id)
static group create(Location const& loc, h5::path const& name, const char* nx_class,
hid_t lcpl_id = H5P_DEFAULT, hid_t gcpl_id = H5P_DEFAULT, hid_t gapl_id = H5P_DEFAULT)
{
return group(h5::group::create(loc, name.c_str(), lcpl_id, gcpl_id, gapl_id), nx_class);
}
template <typename Location>
static group open(Location const& loc, h5::path const& name, hid_t gapl_id = H5P_DEFAULT)
{
return group(h5::group::open(loc, name.c_str(), gapl_id));
}
private:
group(h5::group h5_group, const char* nx_class) : h5::group(std::move(h5_group))
{
attrs.create("NX_class", nx_class);
}
group(h5::group h5_group) : h5::group(std::move(h5_group)) {}
};
inline group file::create_entry(h5::path entry_name, h5::path detector_name,
......@@ -122,7 +155,7 @@ namespace io
herr_t res;
// Create the entry group
nx::group entry = group(hid().get(), entry_name, "NXentry");
auto entry = nx::group::create(hid().get(), entry_name, "NXentry");
entry.create_dataset("title", "Lima 2D detector acquisition"s);
entry.create_dataset("start_time", format_current_time());
entry.create_dataset("end_time", format_current_time());
......@@ -132,30 +165,30 @@ namespace io
entry.attrs.create("default", "instrument"_p / detector_name / "plot"_p);
// Create beamline/instrument group
nx::group instrument(entry, "instrument"_p, "NXinstrument");
auto instrument = nx::group::create(entry, "instrument"_p, "NXinstrument");
instrument.attrs.create("default", detector_name / "plot"_p);
// Create detector group
nx::group detector(instrument, detector_name, "NXdetector");
auto detector = nx::group::create(instrument, detector_name, "NXdetector");
detector.attrs.create("default", "plot"s);
// Detector info
nx::group detector_info(detector, "detector_information"_p, "NXcollection");
auto detector_info = nx::group::create(detector, "detector_information"_p, "NXcollection");
detector_info.create_dataset("image_lima_type", pixel_type);
detector_info.create_dataset("model", model);
detector_info.create_dataset("name", name);
detector_info.create_dataset("type", type);
nx::group max_image_size(detector_info, "max_image_size"_p, "NXcollection");
auto max_image_size = nx::group::create(detector_info, "max_image_size"_p, "NXcollection");
max_image_size.create_dataset("xsize", image_xsize);
max_image_size.create_dataset("ysize", image_ysize);
nx::group pixel_size(detector_info, "pixel_size"_p, "NXcollection");
auto pixel_size = nx::group::create(detector_info, "pixel_size"_p, "NXcollection");
pixel_size.create_dataset("xsize", pixel_xsize);
pixel_size.create_dataset("ysize", pixel_ysize);
// Acquisition info
nx::group acquisition(detector, "acquisition"_p, "NXcollection");
auto acquisition = nx::group::create(detector, "acquisition"_p, "NXcollection");
acquisition.create_dataset("exposure_time", exposure_time);
acquisition.create_dataset("latency_time", latency_time);
acquisition.create_dataset("mode", acq_mode);
......
......@@ -44,10 +44,20 @@ namespace io
file() = default;
file(std::filesystem::path const& name, unsigned flags = H5F_ACC_TRUNC, hid_t fcpl_id = H5P_DEFAULT,
hid_t fapl_id = H5P_DEFAULT) :
pbase_t(H5Fcreate(name.string().c_str(), flags, fcpl_id, fapl_id)), attrs_base_t(member)
/// Creates a H5 file, flags determines the action if file exists:
/// H5F_ACC_TRUNC: tructate, H5F_ACC_EXCL: raise an exception
static file create(std::filesystem::path const& name, unsigned flags = H5F_ACC_TRUNC,
hid_t fcpl_id = H5P_DEFAULT, hid_t fapl_id = H5P_DEFAULT)
{
return H5Fcreate(name.string().c_str(), flags, fcpl_id, fapl_id);
}
/// Opens an existing H5 file, flags determines the access mode:
/// H5F_ACC_RDONLY / H5F_ACC_RDWR
static file open(std::filesystem::path const& name, unsigned flags = H5F_ACC_RDONLY,
hid_t fapl_id = H5P_DEFAULT)
{
return H5Fopen(name.string().c_str(), flags, fapl_id);
}
/// Returns true if the file is open (aka the hid is valid)
......@@ -64,6 +74,9 @@ namespace io
protected:
shared_file_hid_t hid() const { return member; }
private:
file(hid_t fid) : pbase_t(fid), attrs_base_t(member) {}
};
/// Represents an HDF5 group
......@@ -83,17 +96,18 @@ namespace io
/// Creates a new group and links it into the file
template <typename Location>
group(Location const& loc, path const& name, hid_t lcpl_id = H5P_DEFAULT, hid_t gcpl_id = H5P_DEFAULT,
hid_t gapl_id = H5P_DEFAULT) :
pbase_t(H5Gcreate2(loc, name.c_str(), lcpl_id, gcpl_id, gapl_id)), attrs_base_t(member)
static group create(Location const& loc, path const& name, hid_t lcpl_id = H5P_DEFAULT,
hid_t gcpl_id = H5P_DEFAULT, hid_t gapl_id = H5P_DEFAULT)
{
return H5Gcreate2(loc, name.c_str(), lcpl_id, gcpl_id, gapl_id);
}
//template <typename Location>
//group(Location const& loc, path const& name, hid_t gapl_id = H5P_DEFAULT) :
// pbase_t(H5Gopen2(loc, name.c_str(), gapl_id)), attrs_base_t(member)
//{
//}
/// Opens an existing group
template <typename Location>
static group open(Location const& loc, path const& name, hid_t gapl_id = H5P_DEFAULT)
{
return H5Gopen2(loc, name.c_str(), gapl_id);
}
/// Returns true if the group is open (aka the hid is valid)
bool is_open() const { return (bool) member; }
......@@ -102,6 +116,9 @@ namespace io
protected:
shared_group_hid_t hid() const { return member; }
private:
group(hid_t gid) : pbase_t(gid), attrs_base_t(member) {}
};
/// Represents an HDF5 dataset
......@@ -118,11 +135,17 @@ namespace io
dataset() = default;
template <typename Location, typename Datatype>
dataset(Location const& loc, path const& name, Datatype const& dtype_id, dataspace dspace = dataspace(),
hid_t lcpl_id = H5P_DEFAULT, hid_t dcpl_id = H5P_DEFAULT, hid_t dapl_id = H5P_DEFAULT) :
pbase_t(H5Dcreate2(loc, name.c_str(), dtype_id, dspace, lcpl_id, dcpl_id, dapl_id)),
attrs_base_t(member)
static dataset create(Location const& loc, path const& name, Datatype const& dtype_id,
dataspace dspace = dataspace::create(), hid_t lcpl_id = H5P_DEFAULT,
hid_t dcpl_id = H5P_DEFAULT, hid_t dapl_id = H5P_DEFAULT)
{
return dataset(H5Dcreate2(loc, name.c_str(), dtype_id, dspace, lcpl_id, dcpl_id, dapl_id));
}
template <typename Location>
static dataset open(Location const& loc, path const& name, hid_t dapl_id = H5P_DEFAULT)
{
return dataset(H5Dopen2(loc, name.c_str(), dapl_id));
}
/// Returns true if the dataset is open (aka the hid is valid)
......@@ -138,21 +161,23 @@ namespace io
protected:
shared_dataset_hid_t hid() const { return member; }
private:
dataset(hid_t did) : pbase_t(did), attrs_base_t(member) {}
};
inline group file::create_group(path const& name, hid_t lcpl_id, hid_t gcpl_id, hid_t gapl_id)
{
return group(member.get(), name, lcpl_id, gcpl_id, gapl_id);
return group::create(member.get(), name, lcpl_id, gcpl_id, gapl_id);
}
inline group file::require_group(path const& name)
{
// If the link exists
if (link_exists(member.get(), name))
// TODO
return group();
return group::open(member.get(), name);
else
return group(member.get(), name);
return group::create(member.get(), name);
}
} //namespace h5
......
......@@ -71,6 +71,28 @@ namespace io
return *this;
}
std::string string() const
{
H5A_info_t attr_info;
// Get the attribute info
if (H5Aget_info(m_hid.get(), &attr_info) < 0)
LIMA_THROW_EXCEPTION(lima::io_error("Error getting H5 Attribute info"));
std::string attr_data(attr_info.data_size, 0);
string_datatype dtype(attr_data);
// Read the attribute data
if (H5Aread(m_hid.get(), dtype, attr_data.data()) < 0)
LIMA_THROW_EXCEPTION(lima::io_error("Error reading H5 Attribute string data"));
// Remove trailing null terminator
if (attr_data.back() == '\0')
attr_data.erase(attr_data.size() - 1);
return attr_data;
}
shared_attribute_hid_t m_hid;
};
......@@ -85,7 +107,7 @@ namespace io
{
if (H5Aexists(m_hid.get(), attr_name) == 0) {
predef_datatype dtype = predef_datatype::create<T>();
dataspace dspace(H5S_SCALAR);
auto dspace = dataspace::create();
// Create an attribute
unique_attribute_hid_t attribute_id(
H5Acreate2(m_hid.get(), attr_name, dtype, dspace, H5P_DEFAULT, H5P_DEFAULT));
......@@ -100,7 +122,7 @@ namespace io
{
if (H5Aexists(m_hid.get(), attr_name) == 0) {
string_datatype dtype(attr_data);
dataspace dspace(H5S_SCALAR);
auto dspace = dataspace::create();
// Create an attribute
unique_attribute_hid_t attribute_id(
H5Acreate2(m_hid.get(), attr_name, dtype, dspace, H5P_DEFAULT, H5P_DEFAULT));
......@@ -147,7 +169,7 @@ namespace io
if (H5Lexists(impl(), ds_name.c_str(), H5P_DEFAULT) == 0) {
// Create a dataset
predef_datatype dtype = predef_datatype::create<T>();
dataspace dspace(H5S_SCALAR);
auto dspace = dataspace::create();
unique_dataset_hid_t dataset_id(
H5Dcreate2(impl(), ds_name.c_str(), dtype, dspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT));
......@@ -162,7 +184,7 @@ namespace io
if (H5Lexists(impl(), ds_name.c_str(), H5P_DEFAULT) == 0) {
// Create a dataset
string_datatype dtype(ds_data);
dataspace dspace(H5S_SCALAR);
auto dspace = dataspace::create();
unique_dataset_hid_t dataset_id(
H5Dcreate2(impl(), ds_name.c_str(), dtype, dspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT));
......
......@@ -11,6 +11,8 @@
#include <hdf5.h>
#include <lima/exceptions.hpp>
#include <lima/io/h5/wrapper/handle.hpp>
namespace lima
......@@ -22,10 +24,14 @@ namespace io
struct dataspace
{
dataspace(hid_t hid) : m_hid(hid) {}
dataspace(H5S_class_t type = H5S_SCALAR) : m_hid(H5Screate(type)) {}
dataspace(int rank, const hsize_t* dims, const hsize_t* maxdims = NULL) :
m_hid(H5Screate_simple(rank, dims, maxdims))
/// Create a dataspace from a given class (H5S_xxxx)
static dataspace create(H5S_class_t type = H5S_SCALAR) { return H5Screate(type); }
/// Create a simple dataspace and opens it for access
static dataspace create_simple(int rank, const hsize_t dims[], const hsize_t maxdims[] = NULL)
{
return H5Screate_simple(rank, dims, maxdims);
}
/// Determines the dimensionality of a dataspace.
......
......@@ -173,7 +173,7 @@ namespace io
//#endif //defined(LIMA_ENABLE_MPIIO)
// Open the HDF5 file
nx::file f(filename);
auto f = nx::file::create(filename);
if (!f.is_open())
LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Fcreate error")
<< boost::errinfo_filesystem_path(filename));
......@@ -252,24 +252,23 @@ namespace io
res = H5Pset_chunk(dcpl, lengthof(chunk), chunk);
// Create dataspace
dataspace file_space(lengthof(dims), dims);
auto file_space = dataspace::create_simple(lengthof(dims), dims);
// Create dataset
m_dset = dataset(entry, data_path, m_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (!m_dset)
m_dset = dataset::create(entry, data_path, m_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
if (!m_dset.is_open())
LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Dcreate2 error"));
// Add interpretation attributes
m_dset.attrs.create("interpretation", "image"s);
// Create NEXUS plot group and add a link to data from plot
nx::group plot(entry, "instrument"_p / detector_name / "plot"_p, "NXdata");
auto plot = nx::group::create(entry, "instrument"_p / detector_name / "plot"_p, "NXdata");
plot.attrs.create("signal", "data"s);
link_hard(entry, data_path, plot, "data"_p);
// Create NEXUS measurement group and add a link to data from measurement
nx::group measurement(entry, "measurement"_p, "NXcollection");
auto measurement = nx::group::create(entry, "measurement"_p, "NXcollection");
link_hard(entry, data_path, measurement, "data"_p);
}
......@@ -432,7 +431,7 @@ namespace io
using namespace h5::path_literals;
// Open the HDF5 file
nx::file f(filename);
auto f = nx::file::create(filename);
if (!f.is_open())
LIMA_THROW_EXCEPTION(lima::hdf5_error("H5Fcreate error")
<< boost::errinfo_filesystem_path(filename));
......@@ -486,7 +485,7 @@ namespace io
break;
}
nx::group data(entry, data_path, "NXdata");
auto data = nx::group::create(entry, data_path, "NXdata");
// Radius1d
{
......@@ -561,17 +560,17 @@ namespace io
res = H5Pset_chunk(dcpl, lengthof(chunk), chunk);
// Create dataspace
dataspace file_space(lengthof(dims), dims);
auto file_space = dataspace::create_simple(lengthof(dims), dims);
// Create datasets
m_background_avg =
dataset(data, "background_avg"_p, m_bckg_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
m_background_avg = dataset::create(data, "background_avg"_p, m_bckg_dtype, file_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
m_background_avg.attrs.create("interpretation", "spectrum"s);
m_background_avg.attrs.create("long_name", "Average value of background"s);
m_background_avg.attrs.create("signal", 1);
m_background_std =
dataset(data, "background_std"_p, m_bckg_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
m_background_std = dataset::create(data, "background_std"_p, m_bckg_dtype, file_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
m_background_std.attrs.create("interpretation", "spectrum"s);
m_background_std.attrs.create("long_name", "Standard deviation of background"s);
......@@ -585,10 +584,10 @@ namespace io
hsize_t dims[] = {(hsize_t) nb_frames + 1};
// Create dataspace
dataspace file_space(lengthof(dims), dims);
auto file_space = dataspace::create_simple(lengthof(dims), dims);
// Create datasets
m_frame_idx = dataset(data, "frame_ptr"_p, m_index_dtype, file_space);
m_frame_idx = dataset::create(data, "frame_ptr"_p, m_index_dtype, file_space);
// Add initial 0 index
write_frame_idx(0, 0);
......@@ -605,12 +604,13 @@ namespace io
res = H5Pset_chunk(dcpl, lengthof(chunk), chunk);
// Create dataspace
dataspace file_space(lengthof(dims), dims, max_dims);
auto file_space = dataspace::create_simple(lengthof(dims), dims, max_dims);
// Create datasets
m_pixel_idx = dataset(data, "index"_p, m_index_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
m_pixel_val =
dataset(data, "intensity"_p, m_peak_value_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
m_pixel_idx =
dataset::create(data, "index"_p, m_index_dtype, file_space, H5P_DEFAULT, dcpl, H5P_DEFAULT);
m_pixel_val = dataset::create(data, "intensity"_p, m_peak_value_dtype, file_space, H5P_DEFAULT,
dcpl, H5P_DEFAULT);
}
// Create NEXUS plot link
......@@ -618,7 +618,7 @@ namespace io
data.attrs.create("signal", "background_avg"_p);
// Create NEXUS measurement group and add a link to data from measurement
nx::group measurement(entry, "measurement"_p, "NXcollection");
auto measurement = nx::group::create(entry, "measurement"_p, "NXcollection");
link_hard(entry, data_path, measurement, "data"_p);
}
......@@ -630,7 +630,7 @@ namespace io
<< boost::errinfo_frame_idx(frame_idx));
hsize_t dims[] = {1};
dataspace mem_space(1, dims);
auto mem_space = dataspace::create_simple(1, dims);
dataspace file_space = m_frame_idx.space();
......@@ -645,7 +645,7 @@ namespace io
void write_peak_indices(std::vector<int> const& peak_indices)
{
hsize_t dims[] = {peak_indices.size()};
dataspace mem_space(1, dims);
auto mem_space = dataspace::create_simple(1, dims);
dataspace file_space = m_pixel_idx.space();
......@@ -670,7 +670,7 @@ namespace io
void write_peak_values(std::vector<float> const& peak_val)
{
hsize_t dims[] = {peak_val.size()};
dataspace mem_space(1, dims);
auto mem_space = dataspace::create_simple(1, dims);
dataspace file_space = m_pixel_val.space();
......@@ -711,7 +711,7 @@ namespace io
<< boost::errinfo_frame_idx(frame_idx));
hsize_t dims[] = {m_nb_bins};
dataspace mem_space(1, dims);
auto mem_space = dataspace::create_simple(1, dims);
dataspace file_space = m_background_avg.space();
......@@ -730,7 +730,7 @@ namespace io
<< boost::errinfo_frame_idx(frame_idx));
hsize_t dims[] = {m_nb_bins};
dataspace mem_space(1, dims);
auto mem_space = dataspace::create_simple(1, dims);
dataspace file_space = m_background_std.space();
......
......@@ -43,17 +43,19 @@ BOOST_AUTO_TEST_CASE(test_hdf5_wrapper)
namespace h5 = lima::io::h5;
using namespace h5::path_literals;
h5::file f("test_wrapper.h5");
auto f = h5::file::create("test_wrapper.h5");
f.attrs.create("NX_class", "NXroot"s);
h5::group g1(f, "root");
auto g1 = h5::group::create(f, "root");
g1.attrs.create("NX_class", "NXroot"s);
g1.attrs["NX_class"] = "NXentry"s;
h5::group g2(g1, "nested1");
h5::group g3(f, "root"_p / "nested2");
BOOST_CHECK_EQUAL(g1.attrs["NX_class"].string(), "NXentry"s);
h5::dataset ds1(g1, "outer", H5T_NATIVE_FLOAT);
h5::dataset ds2(g2, "inner", H5T_NATIVE_INT);
auto g2 = h5::group::create(g1, "nested1");
auto g3 = h5::group::create(f, "root"_p / "nested2");
auto ds1 = h5::dataset::create(g1, "outer", H5T_NATIVE_FLOAT);
auto ds2 = h5::dataset::create(g2, "inner", H5T_NATIVE_INT);
g1.create_dataset("outer2", 1.0f);
g3.create_dataset("inner2", 1);
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment