Commit 68004038 authored by payno's avatar payno
Browse files

Merge remote-tracking branch 'upstream/0.4'

parents 8f3e0db8 042547bb
Pipeline #37830 passed with stages
in 9 minutes and 20 seconds
......@@ -70,7 +70,7 @@ doc:
script:
- python -m unittest nxtomomill.test.suite -v
test:python3.5-stretch-pyqt5:
test:python3.7-stretch-pyqt5:
image: python:3.7-buster
<<: *test_linux_template
......
......@@ -5,6 +5,7 @@ Change Log
0.4.0: 2020/11/09
-----------------
* requires h5py >= 3
* utils:
* add `change_image_key_control` function to modify frame type inplace
* add `add_dark_flat_nx_file` function to add dark or flat in an existing `NXTomo` entry
......@@ -15,6 +16,7 @@ Change Log
* magnified_pixel_size will not be write anymore. If a magnified / sample pixel size is discover then this will be saved as the 'pixel_size'.
* add an option to display_advancement or not.
* split "zseries" according to z value
* add NXdata information to display detector/data from root as image
* move format version to 1.0
* app:
......
......@@ -191,6 +191,12 @@ def main(argv):
help="Define the set of frames to be mark as alignment. "
"" + _INFO_FRAME_INPUT,
)
# parser.add_argument(
# "--embed-data",
# default=False,
# action="store_true",
# help="Embed data from url in the file if not already inside",
# )
options = parser.parse_args(argv[1:])
......@@ -271,6 +277,7 @@ def main(argv):
flats_start=flat_start_url,
darks_end=darks_end_url,
flats_end=flat_end_url,
embed_data=True,
logger=_logger,
)
......
......@@ -108,20 +108,19 @@ def main(argv):
"--z_trans_key", default=EDF_Z_TRANS, help="z translation key in EDF HEADER"
)
print("******set up***********")
options = parser.parse_args(argv[1:])
conv = utils.get_tuple_of_keys_from_cmd
file_keys = converter.EDFFileKeys(
motor_mne_key=conv(options.motor_mne_key),
motor_pos_key=conv(options.motor_pos_key),
motor_mne_key=options.motor_mne_key,
motor_pos_key=options.motor_pos_key,
ref_names=conv(options.refs_name_keys),
to_ignore=conv(options.ignore_file_containing),
rot_angle_key=conv(options.rot_angle_key),
rot_angle_key=options.rot_angle_key,
dark_names=conv(options.dark_names),
x_trans_key=conv(options.x_trans_key),
y_trans_key=conv(options.y_trans_key),
z_trans_key=conv(options.z_trans_key),
x_trans_key=options.x_trans_key,
y_trans_key=options.y_trans_key,
z_trans_key=options.z_trans_key,
)
input_dir = options.scan_path
......
......@@ -154,6 +154,12 @@ def main(argv):
action="store_true",
default=False,
)
parser.add_argument(
"--entries",
help="Specify (root) entries to be converted. By default it will try "
"to convert all existing entries.",
default=None,
)
parser.add_argument(
"--raises-error",
help="Raise errors if some data are not met instead of providing some"
......@@ -172,21 +178,25 @@ def main(argv):
)
parser.add_argument(
"--x_trans_keys",
"--x-trans-keys",
default=",".join(H5_X_TRANS_KEYS),
help="x translation key in bliss HDF5 file",
)
parser.add_argument(
"--y_trans_keys",
"--y-trans-keys",
default=",".join(H5_Y_TRANS_KEYS),
help="y translation key in bliss HDF5 file",
)
parser.add_argument(
"--z_trans_keys",
"--z-trans-keys",
default=",".join(H5_Z_TRANS_KEYS),
help="z translation key in bliss HDF5 file",
)
parser.add_argument(
"--valid_camera_names",
"--valid-camera-names",
default=None,
help="Valid NXDetector dataset name to be considered. Otherwise will"
"try to deduce them from NX_class attibute (value should be"
......@@ -194,49 +204,63 @@ def main(argv):
)
parser.add_argument(
"--rot_angle_keys",
"--rot-angle-keys",
default=",".join(H5_ROT_ANGLE_KEYS),
help="Valid dataset name for rotation angle",
)
parser.add_argument(
"--acq_expo_time_keys",
"--acq-expo-time-keys",
default=",".join(H5_ACQ_EXPO_TIME_KEYS),
help="Valid dataset name for acquisition exposure time",
)
parser.add_argument(
"--x_pixel_size_key", default=H5_X_PIXEL_SIZE, help="X pixel size key to read"
"--x_pixel_size_key",
"--x-pixel-size-key",
default=H5_X_PIXEL_SIZE,
help="X pixel size key to read",
)
parser.add_argument(
"--y_pixel_size_key", default=H5_Y_PIXEL_SIZE, help="Y pixel size key to read"
"--y_pixel_size_key",
"--y-pixel-size-key",
default=H5_Y_PIXEL_SIZE,
help="Y pixel size key to read",
)
# scan titles
parser.add_argument(
"--init_titles",
"--init-titles",
default=",".join(H5_INIT_TITLES),
help="Titles corresponding to init scans",
)
parser.add_argument(
"--init_zserie_titles",
"--init-zserie-titles",
default=",".join(H5_ZSERIE_INIT_TITLES),
help="Titles corresponding to zserie init scans",
)
parser.add_argument(
"--dark_titles",
"--dark-titles",
default=",".join(H5_DARK_TITLES),
help="Titles corresponding to dark scans",
)
parser.add_argument(
"--ref_titles",
"--ref-titles",
default=",".join(H5_REF_TITLES),
help="Titles corresponding to ref scans",
)
parser.add_argument(
"--proj_titles",
"--proj-titles",
default=",".join(H5_PROJ_TITLES),
help="Titles corresponding to projection scans",
)
parser.add_argument(
"--align_titles",
"--align-titles",
default=",".join(H5_ALIGNMENT_TITLES),
help="Titles corresponding to alignment scans",
)
......@@ -294,6 +318,11 @@ def main(argv):
else:
callback_det_sel = None
if options.entries is not None:
entries = conv(options.entries)
else:
entries = None
h5_to_nx(
input_file_path=options.input_file_path,
output_file=options.output_file,
......@@ -307,6 +336,7 @@ def main(argv):
raise_error_if_issue=options.raises_error,
detector_sel_callback=callback_det_sel,
ask_before_overwrite=not options.overwrite,
entries=entries,
)
exit(0)
......
......@@ -78,6 +78,7 @@ import fabio
from nxtomomill import utils
from nxtomomill.utils import ImageKey
import fnmatch
from silx.io.utils import h5py_read_dataset
import logging
_logger = logging.getLogger(__name__)
......@@ -221,7 +222,10 @@ def edf_to_nx(
indexes = sorted(projections_urls.keys())
first_proj_file = projections_urls[indexes[0]]
fid = fabio.open(first_proj_file.file_path())
hd = fid.getHeader()
if hasattr(fid, "header"):
hd = fid.header
else:
hd = fid.getHeader()
try:
rotangle_index = (
hd[file_keys.motor_mne_key]
......@@ -249,7 +253,10 @@ def edf_to_nx(
except:
ztrans_index = -1
frame_type = fid.getByteCode()
if hasattr(fid, "bytecode"):
frame_type = fid.bytecode
else:
frame_type = fid.getByteCode()
return frame_type, rotangle_index, xtrans_index, ytrans_index, ztrans_index
(
......@@ -892,10 +899,19 @@ def _get_entry_type(
entry: h5py.Group, scan_titles
) -> typing.Union[None, AcquisitionStep]:
try:
title = entry["title"][()]
title = h5py_read_dataset(entry["title"])
except Exception as e:
_logger.error("fail to find title for %s, skip this group" % entry.name)
for step, titles in zip(AcquisitionStep, scan_titles):
step_titles = {
AcquisitionStep.INITIALIZATION: scan_titles.init_titles,
AcquisitionStep.DARK: scan_titles.dark_titles,
AcquisitionStep.REFERENCE: scan_titles.ref_titles,
AcquisitionStep.PROJECTION: scan_titles.proj_titles,
AcquisitionStep.ALIGNMENT: scan_titles.align_titles,
}
for step, titles in step_titles.items():
for title_start in titles:
if title.startswith(title_start):
return step
......@@ -904,7 +920,7 @@ def _get_entry_type(
def _is_z_series(entry: h5py.Group, scan_titles) -> bool:
try:
title = entry["title"][()]
title = h5py_read_dataset(entry["title"])
except Exception as e:
return False
else:
......@@ -1183,7 +1199,7 @@ class _BaseAcquisition:
def get_values():
for possible_key in keys:
if possible_key in node:
values = node[possible_key][()]
values = h5py_read_dataset(node[possible_key])
unit = _BaseAcquisition._get_unit(
node[possible_key], default_unit=expected_unit
)
......@@ -1691,7 +1707,7 @@ class _StandardBaseAcquisition(_BaseAcquisition):
"""return name of the acquisition"""
self._check_has_metadata()
if self._NAME_PATH in self._initialization_entry:
return self._initialization_entry[self._NAME_PATH][()]
return h5py_read_dataset(self._initialization_entry[self._NAME_PATH])
else:
_logger.warning(
"No name describing the acquisition has been found,"
......@@ -1703,7 +1719,7 @@ class _StandardBaseAcquisition(_BaseAcquisition):
"""return tuple(energy, unit)"""
self._check_has_metadata()
if self._ENERGY_PATH in self._initialization_entry:
energy = self._initialization_entry[self._ENERGY_PATH][()]
energy = h5py_read_dataset(self._initialization_entry[self._ENERGY_PATH])
unit = self._get_unit(
self._initialization_entry[self._ENERGY_PATH], default_unit="kev"
)
......@@ -1735,7 +1751,7 @@ class _StandardBaseAcquisition(_BaseAcquisition):
self._check_has_metadata()
if self._DISTANCE_PATH in self._initialization_entry:
node = self.initialization_entry[self._DISTANCE_PATH]
distance = node[()]
distance = h5py_read_dataset(node)
unit = self._get_unit(node, default_unit="cm")
# convert to meter
distance = distance * metricsystem.MetricSystem.from_value(unit).value
......@@ -1763,7 +1779,7 @@ class _StandardBaseAcquisition(_BaseAcquisition):
for key in keys:
if key in self._initialization_entry:
node = self.initialization_entry[key]
node_item = node[()]
node_item = h5py_read_dataset(node)
# if the pixel size is provided as x, y
if isinstance(node_item, numpy.ndarray):
if len(node_item) > 1 and axis == "y":
......@@ -1789,7 +1805,7 @@ class _StandardBaseAcquisition(_BaseAcquisition):
def _get_field_of_fiew(self):
if self._FOV_PATH in self._initialization_entry:
return self.initialization_entry[self._FOV_PATH][()]
return h5py_read_dataset(self.initialization_entry[self._FOV_PATH])
else:
mess = (
"unable to find information regarding field of view for"
......@@ -1805,7 +1821,9 @@ class _StandardBaseAcquisition(_BaseAcquisition):
def _get_estimated_cor_from_motor(self, pixel_size):
"""given pixel is expected to be given in meter"""
if self._file_keys.y_rot_key in self._initialization_entry:
y_rot = self.initialization_entry[self._file_keys.y_rot_key][()]
y_rot = h5py_read_dataset(
self.initialization_entry[self._file_keys.y_rot_key]
)
else:
_logger.warning(
"unable to find information on positioner {}".format(
......
......@@ -40,6 +40,7 @@ from nxtomomill.utils import add_dark_flat_nx_file
from nxtomomill.utils import change_image_key_control
from tomoscan.esrf.hdf5scan import HDF5TomoScan
from nxtomomill.utils import ImageKey
from silx.io.utils import h5py_read_dataset
class BaseTestAddDarkAndFlats(unittest.TestCase):
......@@ -65,7 +66,7 @@ class BaseTestAddDarkAndFlats(unittest.TestCase):
data_path = "/".join(
(self._simple_nx.entry, "instrument", "detector", "data")
)
self._raw_data = h5s[data_path][()]
self._raw_data = h5py_read_dataset(h5s[data_path])
nx_with_vds_path = os.path.join(self.tmpdir, "case_with_vds")
self._nx_with_virtual_dataset = MockHDF5(
scan_path=nx_with_vds_path,
......@@ -216,7 +217,7 @@ class BaseTestAddDarkAndFlats(unittest.TestCase):
with h5py.File(source_file, mode="r") as o_h5s:
if new_path in h5s:
del h5s[new_path]
h5s[new_path] = o_h5s[old_path][()]
h5s[new_path] = h5py_read_dataset(o_h5s[old_path])
elif source_file == target_file:
h5s[new_path] = h5py.SoftLink(old_path)
else:
......@@ -321,7 +322,9 @@ class TestAddDarkAtStart(BaseTestAddDarkAndFlats):
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.start_dark.shape[0]
)
......@@ -418,7 +421,9 @@ class TestAddFlatAtStart(BaseTestAddDarkAndFlats):
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.start_flat.shape[0]
)
......@@ -474,17 +479,21 @@ class TestAddFlatAtEnd(BaseTestAddDarkAndFlats):
scan.entry, "instrument", "detector", "image_key_control"
)
numpy.testing.assert_array_equal(
h5s[img_key_control_path][-2:][()], [0, 1]
h5py_read_dataset(h5s[img_key_control_path][-2:]), [0, 1]
)
img_key_path = os.path.join(
scan.entry, "instrument", "detector", "image_key"
)
numpy.testing.assert_array_equal(h5s[img_key_path][-2:][()], [0, 1])
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[img_key_path][-2:]), [0, 1]
)
# test rotation angle and count_time
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.end_flat.shape[0]
)
......@@ -544,17 +553,21 @@ class TestAddFlatAtEnd(BaseTestAddDarkAndFlats):
scan.entry, "instrument", "detector", "image_key_control"
)
numpy.testing.assert_array_equal(
h5s[img_key_control_path][-2:][()], [0, 1]
h5py_read_dataset(h5s[img_key_control_path][-2:]), [0, 1]
)
img_key_path = os.path.join(
scan.entry, "instrument", "detector", "image_key"
)
numpy.testing.assert_array_equal(h5s[img_key_path][-2:][()], [0, 1])
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[img_key_path][-2:]), [0, 1]
)
# test rotation angle and count_time
count_time_path = os.path.join(
scan.entry, "instrument", "detector", "count_time"
)
numpy.testing.assert_array_equal(h5s[count_time_path][-1][()], 1)
numpy.testing.assert_array_equal(
h5py_read_dataset(h5s[count_time_path][-1]), 1
)
self.assertEqual(
len(h5s[count_time_path]), self.nproj + self.end_flat.shape[0]
)
......@@ -717,7 +730,8 @@ class TestCompleteAddFlatAndDark(BaseTestAddDarkAndFlats):
)
numpy.testing.assert_array_almost_equal(h5s[rotation_angle_dataset][-3], 89)
numpy.testing.assert_array_almost_equal(
h5s[rotation_angle_dataset][1:4][()], numpy.array([10, 11, 12])
h5py_read_dataset(h5s[rotation_angle_dataset][1:4]),
numpy.array([10, 11, 12]),
)
numpy.testing.assert_array_almost_equal(
h5s[rotation_angle_dataset][-2], h5s[rotation_angle_dataset][-1]
......
......@@ -45,6 +45,8 @@ import logging
import h5py
from collections.abc import Iterable
from silx.utils.enum import Enum as _Enum
import uuid
from silx.io.utils import h5py_read_dataset
class ImageKey(_Enum):
......@@ -221,6 +223,7 @@ def add_dark_flat_nx_file(
flats_end: typing.Union[None, numpy.ndarray, DataUrl] = None,
extras: typing.Union[None, dict] = None,
logger: typing.Union[None, logging.Logger] = None,
embed_data: bool = True,
):
"""
This will get all data from entry@input_file and patch them with provided
......@@ -269,6 +272,8 @@ def add_dark_flat_nx_file(
* `rotation_angle`
:type extras: Union[None, dict]
:param Union[None, logging.Logger] logger: object for logs
:param bool embed_data: if True then each external data will be copy
under a 'duplicate_data' folder
"""
from nxtomomill.converter import ImageKey # avoid cyclic import
......@@ -283,6 +288,28 @@ def add_dark_flat_nx_file(
"{keys}".format(key=key, keys=valid_extra_keys)
)
if embed_data is True:
def embed_url(url):
if not isinstance(url, DataUrl):
return url
elif url.file_path() == file_path:
return url
else:
embed_data_path = "/".join(("/duplicate_data", str(uuid.uuid1())))
with cwd_context():
os.chdir(os.path.dirname(os.path.abspath(file_path)))
with HDF5File(file_path, "a") as h5s:
h5s[embed_data_path] = get_data(url)
return DataUrl(
file_path=file_path, data_path=embed_data_path, scheme="silx"
)
darks_start = embed_url(darks_start)
darks_end = embed_url(darks_end)
flats_start = embed_url(flats_start)
flats_end = embed_url(flats_end)
# !!! warning: order of dark / flat treatments import
data_names = "flats_start", "darks_end", "flats_end", "darks_start"
datas = flats_start, darks_end, flats_end, darks_start
......@@ -424,6 +451,19 @@ def _insert_frame_data(data, file_path, data_path, where, logger=None):
if data_path in h5s:
# work on an existing dataset
if h5s[data_path].is_virtual:
if (
h5py.version.hdf5_version_tuple[0] <= 1
and h5py.version.hdf5_version_tuple[1] < 12
):
if logger:
logger.warning(
"You are working on virtual dataset"
"with a hdf5 version < 12. Frame "
"you want to change might be "
"modified depending on the working "
"directory without notifying."
"See https://github.com/silx-kit/silx/issues/3277"
)
if isinstance(data, (numpy.ndarray, list, tuple)):
raise TypeError(
"Provided data is a numpy array when given"
......@@ -492,6 +532,8 @@ def _insert_frame_data(data, file_path, data_path, where, logger=None):
vds_file_path = os.path.relpath(
vds_file_path, os.path.dirname(file_path)
)
if not vds_file_path.startswith("./"):
vds_file_path = "./" + vds_file_path
new_virtual_source = h5py.VirtualSource(
path_or_dataset=vds_file_path,
......@@ -549,16 +591,20 @@ def _insert_frame_data(data, file_path, data_path, where, logger=None):
data_to_store = numpy.empty(new_shape)
if where == "start":
data_to_store[: new_data.shape[0]] = new_data
data_to_store[new_data.shape[0] :] = h5s[data_path][()]
data_to_store[new_data.shape[0] :] = h5py_read_dataset(
h5s[data_path]
)
else:
data_to_store[: h5s[data_path].shape[0]] = h5s[data_path][()]
data_to_store[: h5s[data_path].shape[0]] = h5py_read_dataset(
h5s[data_path]
)
data_to_store[h5s[data_path].shape[0] :] = new_data
else:
assert isinstance(
data, (list, tuple)
), "Unmanaged data type {}".format(type(data))
o_data = h5s[data_path]
o_data = list(o_data[()])
o_data = list(h5py_read_dataset(o_data))
if where == "start":
new_data.extend(o_data)
data_to_store = numpy.asarray(new_data)
......@@ -636,11 +682,11 @@ def change_image_key_control(
with HDF5File(file_path, mode="a") as h5s:
node = h5s[entry]
image_keys_path = "/".join(("instrument", "detector", "image_key"))
image_keys = node[image_keys_path][()]
image_keys = h5py_read_dataset(node[image_keys_path])
image_keys_control_path = "/".join(
("instrument", "detector", "image_key_control")
)
image_keys_control = node[image_keys_control_path][()]
image_keys_control = h5py_read_dataset(node[image_keys_control_path])
# filter frame indexes
if isinstance(frames_indexes, slice):
frames_indexes = list(
......
......@@ -79,7 +79,7 @@ MAJOR = 0
MINOR = 4
MICRO = 0
RELEV = "rc" # <16
SERIAL = 6 # <16
SERIAL = 7 # <16
date = __date__
......
silx
h5py<3
h5py>=3
silx>=0.14a
numpy
fabio
tomoscan >= 0.3.2
\ No newline at end of file
tomoscan >= 0.4a
\ No newline at end of file
......@@ -205,8 +205,8 @@ def get_project_configuration(dry_run):
# for the script launcher and pkg_resources
"setuptools",
"numpy",
"silx",
"h5py<3",
"silx >= 0.14a",
"h5py >= 3.0",
"tomoscan >= 0.3.2",
]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment