Merge branch 'add_gradient_removal' into 'master'

Add gradient removal

Closes #5

See merge request !3

id06workflow/core/Dataset/__init__.py

@@ -31,8 +31,9 @@ __date__ = "01/10/2018"
 try:
     from fabio.fabio import file_series
 except ImportError:
-    from id06workflow.third_party.fabio.file_serie import FileSeries
+    from id06workflow.third_party.fabio.file_serie import FileSeries, filename_series
 import numpy
+import os
 
 
 class Dataset(object):
@@ -59,16 +60,16 @@ class Dataset(object):
         """Flag to avoid loading flat field from files every time"""
 
         """Region Of Interest if any defined"""
-        self.__data = None
+        self.__data_series = None
         self.__darks = None
         self.__ff = None
 
     @property
-    def data_files(self):
+    def data_files_pattern(self):
         return self._data_files
 
-    @data_files.setter
-    def data_files(self, data_files):
+    @data_files_pattern.setter
+    def data_files_pattern(self, data_files):
         self._data_files = data_files
         self.__data_has_changed = True
 
@@ -112,13 +113,13 @@ class Dataset(object):
     def __eq__(self, other):
         if isinstance(other, Dataset) is False:
             return False
-        return (sorted(list(self.data_files)) == sorted(list(other.data_files)) and
+        return (sorted(list(self.data_files_pattern)) == sorted(list(other.data_files_pattern)) and
                 sorted(list(self.dark_files)) == sorted(list(other.dark_files)) and
                 sorted(list(self.flat_fields_files)) == sorted(list(other.flat_fields_files)))
 
     def __str__(self):
         return "\n".join(
-                ("data_files: " + (str(self.data_files) or ""),
+                ("data_files_pattern: " + (str(self.data_files_pattern) or ""),
                 "dark_files: " + (str(self.dark_files) or ""),
                 "flat field images: " + (str(self.flat_fields_files) or "")))
 
@@ -128,18 +129,22 @@ class Dataset(object):
             self.__ff_has_changed = False
         return self.__ff
 
-    def getData(self):
+    def getDataFileSeries(self):
+        """
+        Return a fabio :FileSeries: to iterate over frame.
+        """
         if self.__data_has_changed is True:
-            self.__data = self._loadFiles(self.data_files)
+            assert os.path.exists(self.data_files_pattern)
+            # TODO: warning, for now only deal with single frame file
+            filenames = filename_series(self.data_files_pattern)
+            self.__data_series = FileSeries(filenames=filenames,
+                                            single_frame=True)
             self.__data_has_changed = False
-        return self.__data
+        return self.__data_series
 
     def _loadFiles(self, files_list):
         # TODO: add information if the number of frame is fixed...
         if len(files_list) is 0:
             return None
         series = FileSeries(files_list)
-        framelist = []
-        for frame in series.iterframes():
-            framelist.append(frame.data)
-        return numpy.asarray(framelist)
+        return self._loadSeries(series)

id06workflow/core/experiment/__init__.py

@@ -60,6 +60,8 @@ class Experiment(object):
         self.__data = None
         """current data. The one updated with treatment, roi...
         Which is different from the raw data contained in the dataset"""
+        self.__metadata = None
+        """Metadata associated to the data. One per slice edf file header"""
         self.__ndim = 1
         """Number of dimension of the experiment (motors scanned/rocked)
         For now limited to 1
@@ -106,26 +108,38 @@ class Experiment(object):
         # TODO: cache should probably be used in the future to deal wuth data
         if self.dataset is None:
             return None
-        rawdata = self.dataset.getData()
+        fileseries = self.dataset.getDataFileSeries()
         reductionStep = self._getDataReductionOperations()
         if reductionStep in (None, []):
-            steps = (1, 1, 1)
+            reductionStep = [(1, 1, 1)]
+        if len(reductionStep) is 1:
+            steps = reductionStep[0].steps
         else:
-            if len(reductionStep) is 1:
-                steps = reductionStep[0]
-            else:
-                raise NotImplementedError('cannot manage several reduction steps for the moment')
+            raise NotImplementedError('cannot manage several reduction steps for the moment')
 
+        # load only files we want to keep. Apply the z reduction
+        data_with_z_reduction, self.__metadata = self._loadFileSeries(fileseries=fileseries,
+                                                                      z_step=steps[2])
         if self.roi is not None:
             assert isinstance(self.roi, tuple)
             origin, size = self.roi
-            assert rawdata.ndim is 3
+            assert data_with_z_reduction.ndim is 3
             # TODO: make sure origin and size are given as y, x
             ymin, ymax = int(origin[1]), int(origin[1] + size[1])
             xmin, xmax = int(origin[0]), int(origin[0] + size[0])
-            return rawdata[::steps[0], ymin:ymax:steps[1], xmin:xmax:steps[2]]
+            return data_with_z_reduction[:, ymin:ymax:steps[1], xmin:xmax:steps[0]]
         else:
-            return rawdata[::steps[0], ::steps[1], ::steps[2]]
+            return data_with_z_reduction[:, ::steps[1], ::steps[0]]
+
+    def _loadFileSeries(self, fileseries, z_step):
+        data = []
+        headers = []
+        for iFrame in numpy.arange(start=0, stop=fileseries.nframes, step=z_step):
+            # TODO: deal with motors
+            frame = fileseries.getframe(iFrame)
+            data.append(frame.data)
+            headers.append(frame.header)
+        return numpy.asarray(data), headers
 
     @property
     def data(self):
@@ -216,6 +230,12 @@ class Experiment(object):
         if isinstance(operation, DataReduction):
             self._data_reduction_operations.append(operation)
 
+    def getOperation(self, id):
+        if id in self.operations:
+            return self.operations[id]
+        else:
+            return None
+
     def _getRoiOperations(self):
         """
         

id06workflow/core/experiment/operation/datareduction.py

@@ -33,7 +33,11 @@ from . import OverwritingOperation
 
 
 class DataReduction(OverwritingOperation):
-
+    """
+    Set the experiment to use only part of the dataset.
+    This operation is a 'one shot' operation ofr now as it is postpone in the
+    :class:`Experiment`
+    """
     def __init__(self, experiment, x_factor, y_factor, z_factor):
         """
         Apply a data reduction to the experiment
@@ -50,25 +54,24 @@ class DataReduction(OverwritingOperation):
         self._cache_data = None
 
     def dry_run(self, cache_data=None):
-        if cache_data is None:
-            self._cache_data = self.data[...][::self._z_factor, ::self._y_factor, ::self._x_factor]
-        else:
-            self._cache_data = cache_data
-        return self._cache_data
+        raise NotImplementedError('Not possible for data reduction since'
+                                  'managed by the Experiment class')
 
     def compute(self):
-        self.data = self.data[::self._z_factor, ::self._y_factor, ::self._x_factor]
         self.registerOperation()
         return self.data
 
     def apply(self):
-        if self._cache_data is None:
-            raise ValueError('No data in cache')
-
-        self.data = self._cache_data
-        self.clear_cache()
-        self.registerOperation()
-        return self.data
+        raise NotImplementedError('Not possible for data reduction since'
+                                  'managed by the Experiment class')
 
     def clear_cache(self):
         self._cache_data = None
+
+    def key(self):
+        return ' '.join((self._name, 'x:', str(self._x_factor), 'y:',
+                         str(self._y_factor), 'z:', str(self._z_factor)))
+
+    @property
+    def steps(self):
+        return (self._x_factor, self._y_factor, self._z_factor)

id06workflow/core/experiment/operation/mapping.py

@@ -35,7 +35,24 @@ import logging
 _logger = logging.getLogger(__file__)
 
 
-class IntensityMapping(AdditiveOperation):
+class _MappingBase(AdditiveOperation):
+    """
+    Base class used for mapping
+    """
+    @property
+    def dim(self):
+        raise NotImplementedError('Base class')
+
+    @property
+    def intensity_map(self):
+        raise NotImplementedError('Base class')
+
+    @property
+    def ndim(self):
+        raise NotImplementedError('Base class')
+
+
+class IntensityMapping(_MappingBase):
     """
     for each detector pixel (x,y), each data dimension (motors scanned/rocked)
     are evaluated: 1) intensity is summed along all other dimensions than the
@@ -45,9 +62,10 @@ class IntensityMapping(AdditiveOperation):
     :param float or None threshold: if the threshold is None then the mean
                                 value of the current data will be take
     """
+    KEY = 'intensity mapping'
+
     def __init__(self, experiment, threshold=None):
-        AdditiveOperation.__init__(self, experiment=experiment,
-                                   name='intensity mapping')
+        _MappingBase.__init__(self, experiment=experiment, name=self.KEY)
         self._dim = []
         if threshold is None:
             self._threshold = numpy.median(self.data)
@@ -58,7 +76,7 @@ class IntensityMapping(AdditiveOperation):
         self.__dim = []
 
     def key(self):
-        return self._name
+        return self.KEY
 
     def compute(self):
         _logger.info('start computing the dx2d map')
@@ -107,7 +125,6 @@ class IntensityMapping(AdditiveOperation):
                             self.dim[iDim][y, x, 2] = angles_skewness
                             self.dim[iDim][y, x, 3] = angles_kurtosis
 
-            print('should register mapping operation')
             self.registerOperation()
             return self.dim
 
@@ -126,3 +143,83 @@ class IntensityMapping(AdditiveOperation):
     @property
     def dim(self):
         return self.__dim
+
+    @property
+    def ndim(self):
+        return len(self.__dim)
+
+
+class GradientRemoval(_MappingBase):
+    """
+    Compute gradient for the current intensity and remove it from the initial
+    intensity calculation
+    """
+    def __init__(self, experiment):
+        _MappingBase.__init__(self, experiment=experiment,
+                              name='gradient removal')
+        self._gradients = None
+        self.__dim = []
+        self.__intensity_map = None
+
+    def dry_run(self, cache_data=None):
+        raise ValueError('Not available for Gradient removal yet')
+
+    def compute(self):
+        # TODO: why delta = 0.567
+        delta = 0.567
+
+        # TODO: for now we ask experiment for the intensity mapping if computed
+        mapping = self._experiment.getOperation(IntensityMapping.KEY)
+        if mapping is None:
+            _logger.error('unable to process gradient correction, mapping '
+                          'needs to be proceed before.')
+            return
+
+        self.__dim = self.apply_gradient_correction(self.data, delta=delta, mapping=mapping)
+        self.__intensity_map = mapping.intensity_map[...]
+        self.registerOperation()
+
+    @staticmethod
+    def apply_gradient_correction(data, delta, mapping):
+        assert isinstance(mapping, IntensityMapping)
+        # TODO: number of element: for now only square matrices but if evolve ?
+        corr2, corr1 = numpy.meshgrid(
+            numpy.linspace(-delta, delta, data.shape[2]),
+            numpy.linspace(0, 0, data.shape[1]))
+        _gradients = []
+
+        # TODO: if necessary compute mean...
+
+        for iDim in range(mapping.ndim):
+            gradient = mapping.dim[iDim][...]
+
+            # TODO: for now gradient corrcetion is only apply on mean, why ?
+            # TODO: this should be in enumerate(Mode) but intensity is stored
+            # in an other struct, why ?
+            _gradient = numpy.asarray((
+                    gradient[:, :, 0] + corr2,
+                    gradient[:, :, 1] + corr2,
+                    gradient[:, :, 2] + corr2,
+                    gradient[:, :, 3] + corr2)
+            )
+            _gradients.append(_gradient)
+
+        return _gradients
+
+    def gradients(self):
+        return self._gradients
+
+    def key(self):
+        return self._name
+
+    @property
+    def dim(self):
+        return self.__dim
+
+    @property
+    def intensity_map(self):
+        return self.__intensity_map
+
+    @property
+    def ndim(self):
+        return len(self.__dim)

id06workflow/core/experiment/operation/noisereduction.py

@@ -31,61 +31,12 @@ __date__ = "15/10/2018"
 
 import logging
 from . import AdditiveOperation, OverwritingOperation
+from .mapping import IntensityMapping
 import numpy
 
 _logger = logging.getLogger(__file__)
 
 
-class GradientCalculation(AdditiveOperation):
-    """
-    Compute gradient for the current intensity
-    """
-    def __init__(self, experiment):
-        AdditiveOperation.__init__(self, experiment=experiment,
-                                   name='gradient removal')
-        self._gradients = None
-
-    def compute(self):
-        try:
-            # TODO: in the future create the operation if not existing
-            intensityOperation = self._experiment.getOperation('intensity')
-        except Exception as e:
-            _logger.error(e)
-            return
-
-        # TODO: how to get delta ?
-        delta = 0.567
-        corr2, corr1 = numpy.meshgrid(
-            numpy.linspace(-delta, delta, self.data.shape[0]),
-            numpy.linspace(0, 0,
-                           self.data.shape[0]))
-        self._gradients = []
-
-        # TODO: if necessary compute mean...
-
-        for iDim in self._experiment.ndim:
-            gradient = intensityOperation.dim[iDim][...]
-            gradient = gradient + corr2
-            self._gradients.append(gradient)
-
-        self.registerOperation()
-        return self._gradients
-
-    def gradients(self):
-        return self._gradients
-
-    def key(self):
-        return self._name
-
-
-class GradientRemoval(OverwritingOperation):
-    """Apply a gradient removal to the data"""
-    pass
-
-    def key(self):
-        return self._name
-
-
 class ThresholdRemoval(OverwritingOperation):
     """
     Apply a simple threshold filtering on data
@@ -129,7 +80,7 @@ class ThresholdRemoval(OverwritingOperation):
 
     @replacement_value.setter
     def replacement_value(self, value):
-        self.replacement_value = value
+        self._replacement_value = value
 
     def compute(self):
         """

id06workflow/core/experiment/operation/roi.py

@@ -45,7 +45,7 @@ class RoiOperation(OverwritingOperation):
         """ numpy.ndarray used for caching"""
 
     def key(self):
-        return ' '.join((self._name, 'origin:', str(self._origin), 'origin:',
+        return ' '.join((self._name, 'origin:', str(self._origin), 'size:',
                          str(self._size)))
 
     def compute(self):

id06workflow/gui/datasetselection.py

@@ -57,7 +57,7 @@ class DatasetSelectionDialog(qt.QDialog):
         _buttons.rejected.connect(self.reject)
 
         # expose API
-        self.getDataFiles = self.mainWindow.getDataFiles
+        self.getDataFilesPattern = self.mainWindow.getDataFilesPattern
         self.getDarkFiles = self.mainWindow.getDarkFiles
         self.getFlatFieldFiles = self.mainWindow.getFlatFieldFiles
 
@@ -68,15 +68,15 @@ class DatasetSelection(qt.QTabWidget):
     """
     def __init__(self, parent):
         qt.QTabWidget.__init__(self, parent)
-        self._dataFiles = DataListWidget(parent=self)
-        self.addTab(self._dataFiles, 'data files')
+        self._dataFiles = FilePatternWidget(parent=self)
+        self.addTab(self._dataFiles, 'data files pattern:')
         self._darkFiles = DataListWidget(parent=self)
         self.addTab(self._darkFiles, 'dark files')
         self._ffFiles = DataListWidget(parent=self)
         self.addTab(self._ffFiles, 'flat field files')
 
         # expose API
-        self.getDataFiles = self._dataFiles.getFiles
+        self.getDataFilesPattern = self._dataFiles.getPattern
         self.getDarkFiles = self._darkFiles.getFiles
         self.getFlatFieldFiles = self._ffFiles.getFiles
 
@@ -86,7 +86,7 @@ class DatasetSelection(qt.QTabWidget):
         :return: dataset defined on the GUI
         :rtype: :class:`Dataset`
         """
-        return Dataset(data_files=self.getDataFiles(),
+        return Dataset(data_files=self.getDataFilesPattern(),
                        dark_files=self.getDarkFiles(),
                        ff_files=self.getFlatFieldFiles())
 
@@ -97,11 +97,33 @@ class DatasetSelection(qt.QTabWidget):
         :param :class:`Dataset` dataset: 
         """
         assert isinstance(dataset, Dataset)
-        self._dataFiles.setFiles(dataset.data_files)
+        self._dataFiles.setFiles(dataset.data_files_pattern)
         self._darkFiles.setFiles(dataset.dark_files)
         self._ffFiles.setFiles(dataset._ff_files)
 
 
+class FilePatternWidget(qt.QWidget):
+    """A simple interface to select files from a pattern
+
+    .. warning: the widget won't check for scan validity and will only
+        emit the path to folders to the next widgets
+
+    :param parent: the parent widget
+    """
+    def __init__(self, parent=None):
+        qt.QWidget.__init__(self, parent)
+        self.setLayout(qt.QHBoxLayout())
+        self.layout().addWidget(qt.QLabel('file pattern:', parent=self))
+        self._file_pattern = qt.QLineEdit('', parent=self)
+        self.layout().addWidget(self._file_pattern)
+
+    def getPattern(self):
+        return str(self._file_pattern.text())
+
+    def setPattern(self, pattern):
+        return self._file_pattern.setText(str(pattern))
+
+
 class DataListWidget(qt.QWidget):
     """A simple list of dataset path.
 
@@ -204,7 +226,6 @@ class FileList(qt.QTableWidget):
         itemIndex = self.row(item)
         self.takeItem(itemIndex, 0)
         self._selection.remove(item.text())
-        # ScanList.remove(self, item.text())
         self.removeRow(item.row())
         self.setRowCount(self.rowCount() - 1)
         self._update()

id06workflow/gui/mapping.py

@@ -32,7 +32,7 @@ from silx.gui import qt
 from silx.gui.plot import Plot2D
 from id06workflow.gui import icons
 from id06workflow.core.mapping import MEAN, VARIANCE, SKENESS, KURTOSIS, MODES, INTENSITY
-from id06workflow.core.experiment.operation.mapping import IntensityMapping
+from id06workflow.core.experiment.operation.mapping import _MappingBase
 from id06workflow.gui.settings import DEFAULT_COLORMAP
 from collections import OrderedDict
 import functools
@@ -124,7 +124,7 @@ class MappingPlot(qt.QMainWindow):
         self._data[SKENESS] = self._data[KURTOSIS] = None
 
         if operation is not None:
-            assert isinstance(operation, IntensityMapping)
+            assert isinstance(operation, _MappingBase)
             self._data[INTENSITY] = operation.intensity_map
             self._data[MEAN] = operation.dim[0][:, :, 0]
             self._data[VARIANCE] = operation.dim[0][:, :, 1]

id06workflow/gui/noisereduction.py

@@ -99,6 +99,7 @@ class NoiseReductionWidget(qt.QWidget):
                 threshold = self._paramsWidget.getUpperThreshold()
                 if threshold is not None:
                     self._upperThreshold.threshold = threshold
+                    self._upperThreshold.replacement_value = 0.0
                     self._cache_data = self._upperThreshold.dry_run(self._cache_data)
                 threshold = self._paramsWidget.getLowerThreshold()
                 if threshold is not None:

id06workflow/gui/roiselection.py

@@ -100,6 +100,8 @@ class ROIOnStackView(qt.QWidget):
                                   size=stack.shape[-2:])
         else:
             self._clipROI()
+        # activate ROI edition by default
+        self._roi.setEditable(True)
 
     def setROI(self, origin, size):
         self._roi.setGeometry(origin=origin, size=size)

orangecontrib/id06workflow/widgets/datareduction.py

@@ -75,7 +75,7 @@ class DataReductionOW(OWWidget):
         self._widget.layout().addWidget(self._yreduc, 1, 1)
 
         self._widget.layout().addWidget(qt.QLabel('z reduction:'), 2, 0)
-        self._zreduc = qt.QLineEdit('1', parent=self)
+        self._zreduc = qt.QLineEdit('10', parent=self)
         self._zreduc.setValidator(qt.QIntValidator(parent=self))
         self._widget.layout().addWidget(self._zreduc, 2, 1)
         layout.addWidget(self._widget)
@@ -92,12 +92,9 @@ class DataReductionOW(OWWidget):
     def _process(self, experiment):
         if experiment is None:
             return
-        try:
-            operation = DataReduction(experiment=experiment,
-                                      x_factor=self.getXReduction(),
-                                      y_factor=self.getYReduction(),
-                                      z_factor=self.getZReduction())
-            operation.compute()
-            self.send("data", experiment)
-        except Exception as e:
-            _logger.error(e)
+        operation = DataReduction(experiment=experiment,
+                                  x_factor=self.getXReduction(),
+                                  y_factor=self.getYReduction(),
+                                  z_factor=self.getZReduction())
+        operation.compute()
+        self.send("data", experiment)

orangecontrib/id06workflow/widgets/gradientremoval.py

+# coding: utf-8
+# /*##########################################################################
+#
+# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# ###########################################################################*/
+
+__authors__ = ["H. Payno"]
+__license__ = "MIT"
+__date__ = "16/10/2018"