Commit 3a76991f authored by payno's avatar payno
Browse files

Merge branch 'compacted_projs' into 'master'

Compacted DataUrl of projections

See merge request !10
parents 2bc2c976 6156aa1d
Pipeline #22960 passed with stages
in 2 minutes and 5 seconds
......@@ -41,6 +41,7 @@ from silx.utils.enum import Enum as _Enum
from tomoscan.utils import docstring
from silx.io.utils import get_data
from ..unitsystem import metricsystem
from .utils import get_compacted_dataslices
import logging
import typing
......@@ -142,6 +143,7 @@ class HDF5TomoScan(TomoScanBase):
# data caches
self._projections = None
self._projections_compacted = None
self._flats = None
self._darks = None
self._tomo_n = None
......@@ -183,6 +185,7 @@ class HDF5TomoScan(TomoScanBase):
@docstring(TomoScanBase.clear_caches)
def clear_caches(self) -> None:
self._projections = None
self._projections_compacted = None
self._flats = None
self._darks = None
self._tomo_n = None
......@@ -613,6 +616,21 @@ class HDF5TomoScan(TomoScanBase):
else:
return None
@property
def projections_compacted(self):
"""
Return a compacted view of projection frames.
Returns
--------
merged_projections: dict
Dictionary where the key is a list of indices, and the value
is the corresponding `silx.io.url.DataUrl` with merged data_slice
"""
if self._projections_compacted is None:
self._projections_compacted = get_compacted_dataslices(self.projections)
return self._projections_compacted
def __str__(self):
return 'hdf5 scan(path: %s, master_file: %s, entry: %s)' % (self.path,
self.master_file,
......
# coding: utf-8
#/*##########################################################################
# Copyright (C) 2016-2020 European Synchrotron Radiation Facility
# Copyright (C) 2016 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
......@@ -27,7 +27,7 @@ Utils to mock scans
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "10/10/2019"
__date__ = "30/09/2019"
import h5py
import numpy
......@@ -35,7 +35,7 @@ import os
from xml.etree import cElementTree
import fabio
import fabio.edfimage
from ..scanfactory import ScanFactory # avoid cyclic import
from .hdf5scan import ImageKey, HDF5TomoScan
class _ScanMock:
......@@ -50,9 +50,7 @@ class _ScanMock:
:param scan_path:
:param n_radio:
:param n_ini_radio: number of radio existing at t time. If None,
will create all the radio and consider the
acquisition as finished
:param n_ini_radio:
:param n_extra_radio:
:param scan_range:
:param n_recons:
......@@ -63,6 +61,8 @@ class _ScanMock:
:param dark_n:
:param str scene: scene type.
* 'noise': generate radios from numpy.random
* `increase value`: first frame value will be0, then second 1...
* `arange`: arange through frames
* 'perfect-sphere: generate a sphere which just fit in the
detector dimensions
TODO: add some differente scene type.
......@@ -73,6 +73,9 @@ class _ScanMock:
self.n_radio = n_radio
self.scene = scene
if not os.path.exists(scan_path):
os.mkdir(scan_path)
self.write_metadata(n_radio=n_radio, scan_range=scan_range,
ref_n=ref_n, dark_n=dark_n)
......@@ -117,6 +120,12 @@ class _ScanMock:
def _get_radio_data(self, index):
if self.scene == 'noise':
return numpy.random.random((self.det_height * self.det_width)).reshape((self.det_width, self.det_height))
elif self.scene == 'increasing value':
return numpy.zeros((self.det_width, self.det_height)) + index
elif self.scene == 'arange':
start = index * (self.det_height * self.det_width)
stop = (index + 1) * (self.det_height * self.det_width)
return numpy.arange(start=start, stop=stop).reshape(self.det_width, self.det_height)
elif self.scene == 'perfect-sphere':
background = numpy.zeros((self.det_height * self.det_width))
radius = min(background.shape)
......@@ -133,74 +142,45 @@ class _ScanMock:
scale = 1
background[radii < radius * scale] = 1.0
return background
else:
raise ValueError('selected scene %s is no managed' % self.scene)
class MockHDF5(_ScanMock):
"""
Mock an acquisition in a hdf5 file
"""
def __init__(self, scan_path, n_radio, n_ini_radio=None, n_extra_radio=0,
scan_range=360, n_recons=0, n_pag_recons=0, recons_vol=False,
dim=200):
self.scan_master_file = os.path.join(scan_path, os.path.basename((scan_path)) + '.h5')
super(MockHDF5, self).__init__(scan_path=scan_path, n_radio=n_radio,
n_ini_radio=n_ini_radio,
n_extra_radio=n_extra_radio,
scan_range=scan_range,
n_recons=n_recons,
n_pag_recons=n_pag_recons,
recons_vol=recons_vol, dim=dim)
def add_radio(self, index=None):
radio = self._get_radio_data(index=index)
radio = radio.reshape((1, self.det_height, self.det_width))
with h5py.File(self.scan_master_file, 'r+') as h5_file:
if '1_tomo/measurement/pcoedge64:image' in h5_file:
current_dataset = h5_file['1_tomo/measurement/pcoedge64:image'][()]
del h5_file['1_tomo/measurement/pcoedge64:image']
new_dataset = numpy.append(current_dataset, radio)
shape = list(current_dataset.shape)
shape[0] += 1
new_dataset = new_dataset.reshape(shape)
else:
new_dataset = radio
with h5py.File(self.scan_master_file, 'a') as h5_file:
h5_file['1_tomo/measurement/pcoedge64:image'] = new_dataset
def write_metadata(self, n_radio, scan_range, ref_n, dark_n):
with h5py.File(self.scan_master_file, 'a') as h5_file:
h5_file['1_tomo/scan_meta/technique/scan/tomo_n'] = n_radio
h5_file['1_tomo/scan_meta/technique/scan/scan_range'] = scan_range
h5_file['1_tomo/scan_meta/technique/scan/ref_n'] = ref_n
h5_file['1_tomo/scan_meta/technique/scan/dark_n'] = dark_n
h5_file['1_tomo/scan_meta/technique/detector/size'] = (self.det_width, self.det_height)
h5_file['1_tomo/scan_meta/technique/detector/pixel_size'] = _ScanMock.PIXEL_SIZE
h5_file['1_tomo/scan_meta/technique/detector/ref_on'] = str(n_radio)
def end_acquisition(self):
# no specific operation to do
pass
class MockEDF(_ScanMock):
"""Mock a EDF acquisition"""
"""Mock a EDF acquisition
"""
_RECONS_PATTERN = 'slice_'
_RECONS_PATTERN = '_slice_'
_PAG_RECONS_PATTERN = 'slice_pag_'
_PAG_RECONS_PATTERN = '_slice_pag_'
def __init__(self, scan_path, n_radio, n_ini_radio=None, n_extra_radio=0,
scan_range=360, n_recons=0, n_pag_recons=0, recons_vol=False,
dim=200):
dim=200, scene='noise'):
self._last_radio_index = -1
super(MockEDF, self).__init__(scan_path=scan_path, n_radio=n_radio,
super(MockEDF, self).__init__(scan_path=scan_path,
n_radio=n_radio,
n_ini_radio=n_ini_radio,
n_extra_radio=n_extra_radio,
scan_range=scan_range,
n_recons=n_recons,
n_pag_recons=n_pag_recons,
recons_vol=recons_vol, dim=dim)
recons_vol=recons_vol,
dim=dim,
scene=scene)
if n_ini_radio:
for i_radio in range(n_ini_radio):
self.add_radio(i_radio)
for i_extra_radio in range(n_extra_radio):
self.add_radio(i_extra_radio)
for i_recons in range(n_recons):
self.add_reconstruction(i_recons)
for i_recons in range(n_pag_recons):
self.add_pag_reconstruction(i_recons)
if recons_vol is True:
self.add_recons_vol()
def get_info_file(self):
return os.path.join(self.scan_path, os.path.basename(self.scan_path) + '.info')
......@@ -233,17 +213,17 @@ class MockEDF(_ScanMock):
info_file.write('PixelSize= ' + str(_ScanMock.PIXEL_SIZE) + '\n')
def add_radio(self, index=None):
if index:
if index is not None:
self._last_radio_index = index
index_ = index
else:
self._last_radio_index += 1
index_ = self._last_radio_index
file_name = os.path.basename(self.scan_path) + '_{0:04d}'.format(index_) + ".edf"
f = os.path.join(self.scan_path, file_name)
if not os.path.exists(f):
data = self._get_radio_data(index=index_)
assert data is not None
assert data.shape == (self.det_width, self.det_height)
edf_writer = fabio.edfimage.EdfImage(data=data,
header={"tata": "toto"})
......@@ -374,6 +354,7 @@ class MockEDF(_ScanMock):
assert type(nRadio) is int
assert type(nRecons) is int
assert type(dim) is int
from ..scanfactory import ScanFactory # avoid cyclic import
MockEDF.fastMockAcquisition(folder=scanID, n_radio=nRadio,
scan_range=scan_range, n_extra_radio=n_extra_radio)
......
......@@ -202,6 +202,21 @@ class TestHDF5Scan(HDF5TestBaseClass):
self.assertTrue(numpy.isclose(self.scan.get_distance(unit='cm'), -1997.35))
def testCompactedProjs(self):
projs_compacted = self.scan.projections_compacted
self.assertTrue(projs_compacted.keys() == self.scan.projections.keys())
# ~ print(set(map(str, list(projs_compacted.values()))))
for i in range(22, 1520+1):
self.assertTrue(
projs_compacted[i].data_slice() == slice(22, 1521, None)
)
for i in range(1542, 1543):
self.assertTrue(
projs_compacted[i].data_slice() == slice(1542, 1543, None)
)
def suite():
test_suite = unittest.TestSuite()
for ui in (TestHDF5Scan, ):
......
......@@ -32,6 +32,7 @@ import os
import fabio
from silx.io.url import DataUrl
from typing import Union
import numpy
def get_parameters_frm_par_or_info(file_: str) -> dict:
......@@ -86,3 +87,78 @@ def extract_urls_from_edf(file_: str, start_index: Union[None, int], n_frames: U
data_slice=[i_frame, ])
index += 1
return res
def get_compacted_dataslices(urls):
"""
Regroup urls to get the data more efficiently.
Build a structure mapping files indices to information on
how to load the data: `{indices_set: data_location}`
where `data_location` contains contiguous indices.
Parameters
-----------
urls: dict
Dictionary where the key is an integer and the value is a silx `DataUrl`.
Returns
--------
merged_urls: dict
Dictionary where the key is a list of indices, and the value
is the corresponding `silx.io.url.DataUrl` with merged data_slice
"""
def _convert_to_slice(idx):
if numpy.isscalar(idx):
return slice(idx, idx+1)
# otherwise, assume already slice object
return idx
def is_contiguous_slice(slice1, slice2):
if numpy.isscalar(slice1):
slice1 = slice(slice1, slice1+1)
if numpy.isscalar(slice2):
slice2 = slice(slice2, slice2+1)
return slice2.start == slice1.stop
def merge_slices(slice1, slice2):
return slice(slice1.start, slice2.stop)
sorted_files_indices = sorted(urls.keys())
idx0 = sorted_files_indices[0]
first_url = urls[idx0]
merged_indices = [
[idx0]
]
data_location = [
[
first_url.file_path(),
first_url.data_path(),
_convert_to_slice(first_url.data_slice())
]
]
pos = 0
curr_fp, curr_dp, curr_slice = data_location[pos]
for idx in sorted_files_indices[1:]:
url = urls[idx]
next_slice = _convert_to_slice(url.data_slice())
if (url.file_path() == curr_fp) and (url.data_path() == curr_dp) and is_contiguous_slice(curr_slice, next_slice):
merged_indices[pos].append(idx)
merged_slices = merge_slices(curr_slice, next_slice)
data_location[pos][-1] = merged_slices
curr_slice = merged_slices
else: # "jump"
pos += 1
merged_indices.append([idx])
data_location.append([
url.file_path(), url.data_path(), _convert_to_slice(url.data_slice())
])
curr_fp, curr_dp, curr_slice = data_location[pos]
# Format result
res = {}
for ind, dl in zip(merged_indices, data_location):
# res[tuple(ind)] = DataUrl(file_path=dl[0], data_path=dl[1], data_slice=dl[2])
res.update(dict.fromkeys(ind, DataUrl(file_path=dl[0], data_path=dl[1], data_slice=dl[2])))
return res
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment