Commit 1ed43984 authored by payno's avatar payno
Browse files

[Dataset] move to fabio.FileSeries

- now series is given as a file pattern (give file xxx_0000.edf) and we can browse through this dataset
- rework the Dataset __data to return a fabio `FileSeries`
- postpone the DataReduction to be proceeded by the Dataset to avoid loading of all frames.
- limited to single frame edf file.
parent a535a5b5
......@@ -31,8 +31,9 @@ __date__ = "01/10/2018"
from fabio.fabio import file_series
except ImportError:
from id06workflow.third_party.fabio.file_serie import FileSeries
from id06workflow.third_party.fabio.file_serie import FileSeries, filename_series
import numpy
import os
class Dataset(object):
......@@ -59,16 +60,16 @@ class Dataset(object):
"""Flag to avoid loading flat field from files every time"""
"""Region Of Interest if any defined"""
self.__data = None
self.__data_series = None
self.__darks = None
self.__ff = None
def data_files(self):
def data_files_pattern(self):
return self._data_files
def data_files(self, data_files):
def data_files_pattern(self, data_files):
self._data_files = data_files
self.__data_has_changed = True
......@@ -112,13 +113,13 @@ class Dataset(object):
def __eq__(self, other):
if isinstance(other, Dataset) is False:
return False
return (sorted(list(self.data_files)) == sorted(list(other.data_files)) and
return (sorted(list(self.data_files_pattern)) == sorted(list(other.data_files_pattern)) and
sorted(list(self.dark_files)) == sorted(list(other.dark_files)) and
sorted(list(self.flat_fields_files)) == sorted(list(other.flat_fields_files)))
def __str__(self):
return "\n".join(
("data_files: " + (str(self.data_files) or ""),
("data_files_pattern: " + (str(self.data_files_pattern) or ""),
"dark_files: " + (str(self.dark_files) or ""),
"flat field images: " + (str(self.flat_fields_files) or "")))
......@@ -128,18 +129,22 @@ class Dataset(object):
self.__ff_has_changed = False
return self.__ff
def getData(self):
def getDataFileSeries(self):
Return a fabio :FileSeries: to iterate over frame.
if self.__data_has_changed is True:
self.__data = self._loadFiles(self.data_files)
assert os.path.exists(self.data_files_pattern)
# TODO: warning, for now only deal with single frame file
filenames = filename_series(self.data_files_pattern)
self.__data_series = FileSeries(filenames=filenames,
self.__data_has_changed = False
return self.__data
return self.__data_series
def _loadFiles(self, files_list):
# TODO: add information if the number of frame is fixed...
if len(files_list) is 0:
return None
series = FileSeries(files_list)
framelist = []
for frame in series.iterframes():
return numpy.asarray(framelist)
return self._loadSeries(series)
......@@ -60,6 +60,8 @@ class Experiment(object):
self.__data = None
"""current data. The one updated with treatment, roi...
Which is different from the raw data contained in the dataset"""
self.__metadata = None
"""Metadata associated to the data. One per slice edf file header"""
self.__ndim = 1
"""Number of dimension of the experiment (motors scanned/rocked)
For now limited to 1
......@@ -106,26 +108,38 @@ class Experiment(object):
# TODO: cache should probably be used in the future to deal wuth data
if self.dataset is None:
return None
rawdata = self.dataset.getData()
fileseries = self.dataset.getDataFileSeries()
reductionStep = self._getDataReductionOperations()
if reductionStep in (None, []):
steps = (1, 1, 1)
reductionStep = [(1, 1, 1)]
if len(reductionStep) is 1:
steps = reductionStep[0].steps
if len(reductionStep) is 1:
steps = reductionStep[0]
raise NotImplementedError('cannot manage several reduction steps for the moment')
raise NotImplementedError('cannot manage several reduction steps for the moment')
# load only files we want to keep. Apply the z reduction
data_with_z_reduction, self.__metadata = self._loadFileSeries(fileseries=fileseries,
if self.roi is not None:
assert isinstance(self.roi, tuple)
origin, size = self.roi
assert rawdata.ndim is 3
assert data_with_z_reduction.ndim is 3
# TODO: make sure origin and size are given as y, x
ymin, ymax = int(origin[1]), int(origin[1] + size[1])
xmin, xmax = int(origin[0]), int(origin[0] + size[0])
return rawdata[::steps[0], ymin:ymax:steps[1], xmin:xmax:steps[2]]
return data_with_z_reduction[:, ymin:ymax:steps[1], xmin:xmax:steps[0]]
return rawdata[::steps[0], ::steps[1], ::steps[2]]
return data_with_z_reduction[:, ::steps[1], ::steps[0]]
def _loadFileSeries(self, fileseries, z_step):
data = []
headers = []
for iFrame in numpy.arange(start=0, stop=fileseries.nframes, step=z_step):
# TODO: deal with motors
frame = fileseries.getframe(iFrame)
return numpy.asarray(data), headers
def data(self):
......@@ -33,7 +33,11 @@ from . import OverwritingOperation
class DataReduction(OverwritingOperation):
Set the experiment to use only part of the dataset.
This operation is a 'one shot' operation ofr now as it is postpone in the
def __init__(self, experiment, x_factor, y_factor, z_factor):
Apply a data reduction to the experiment
......@@ -50,24 +54,16 @@ class DataReduction(OverwritingOperation):
self._cache_data = None
def dry_run(self, cache_data=None):
if cache_data is None:
self._cache_data =[...][::self._z_factor, ::self._y_factor, ::self._x_factor]
self._cache_data = cache_data[::self._z_factor, ::self._y_factor, ::self._x_factor]
return self._cache_data
raise NotImplementedError('Not possible for data reduction since'
'managed by the Experiment class')
def compute(self): =[::self._z_factor, ::self._y_factor, ::self._x_factor]
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache') = self._cache_data
raise NotImplementedError('Not possible for data reduction since'
'managed by the Experiment class')
def clear_cache(self):
self._cache_data = None
......@@ -75,3 +71,7 @@ class DataReduction(OverwritingOperation):
def key(self):
return ' '.join((self._name, 'x:', str(self._x_factor), 'y:',
str(self._y_factor), 'z:', str(self._z_factor)))
def steps(self):
return (self._x_factor, self._y_factor, self._z_factor)
......@@ -57,7 +57,7 @@ class DatasetSelectionDialog(qt.QDialog):
# expose API
self.getDataFiles = self.mainWindow.getDataFiles
self.getDataFilesPattern = self.mainWindow.getDataFilesPattern
self.getDarkFiles = self.mainWindow.getDarkFiles
self.getFlatFieldFiles = self.mainWindow.getFlatFieldFiles
......@@ -68,15 +68,15 @@ class DatasetSelection(qt.QTabWidget):
def __init__(self, parent):
qt.QTabWidget.__init__(self, parent)
self._dataFiles = DataListWidget(parent=self)
self.addTab(self._dataFiles, 'data files')
self._dataFiles = FilePatternWidget(parent=self)
self.addTab(self._dataFiles, 'data files pattern:')
self._darkFiles = DataListWidget(parent=self)
self.addTab(self._darkFiles, 'dark files')
self._ffFiles = DataListWidget(parent=self)
self.addTab(self._ffFiles, 'flat field files')
# expose API
self.getDataFiles = self._dataFiles.getFiles
self.getDataFilesPattern = self._dataFiles.getPattern
self.getDarkFiles = self._darkFiles.getFiles
self.getFlatFieldFiles = self._ffFiles.getFiles
......@@ -86,7 +86,7 @@ class DatasetSelection(qt.QTabWidget):
:return: dataset defined on the GUI
:rtype: :class:`Dataset`
return Dataset(data_files=self.getDataFiles(),
return Dataset(data_files=self.getDataFilesPattern(),
......@@ -97,11 +97,33 @@ class DatasetSelection(qt.QTabWidget):
:param :class:`Dataset` dataset:
assert isinstance(dataset, Dataset)
class FilePatternWidget(qt.QWidget):
"""A simple interface to select files from a pattern
.. warning: the widget won't check for scan validity and will only
emit the path to folders to the next widgets
:param parent: the parent widget
def __init__(self, parent=None):
qt.QWidget.__init__(self, parent)
self.layout().addWidget(qt.QLabel('file pattern:', parent=self))
self._file_pattern = qt.QLineEdit('', parent=self)
def getPattern(self):
return str(self._file_pattern.text())
def setPattern(self, pattern):
return self._file_pattern.setText(str(pattern))
class DataListWidget(qt.QWidget):
"""A simple list of dataset path.
......@@ -75,7 +75,7 @@ class DataReductionOW(OWWidget):
self._widget.layout().addWidget(self._yreduc, 1, 1)
self._widget.layout().addWidget(qt.QLabel('z reduction:'), 2, 0)
self._zreduc = qt.QLineEdit('1', parent=self)
self._zreduc = qt.QLineEdit('10', parent=self)
self._widget.layout().addWidget(self._zreduc, 2, 1)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment