[operation] Adapt several operation to multiple dimension

- adapt shift (wok with data_flatten but should probably be extended latter)
- adapt noise reduction (work with data_flatten)
- remove the data reduction widget because should be highly modify since the numpy.ndarra.view in the experiement.data appear. Should be re-done

id06workflow/core/experiment/__init__.py

@@ -178,16 +178,14 @@ class Experiment(object):
 
     @property
     def data(self):
-        if self.__data is None:
-            self.__data = self.getRawData()
         if self.__dims.ndim > 1:
             # TODO: create view or adapt view on raw data
             shape = list(self.__dims.shape)
-            shape.append(self.__data.shape[-2])
-            shape.append(self.__data.shape[-1])
-            return self.__data.view().reshape(shape)
+            shape.append(self.data_flatten.shape[-2])
+            shape.append(self.data_flatten.shape[-1])
+            return self.data_flatten.view().reshape(shape)
         else:
-            return self.__data
+            return self.data_flatten
 
     @property
     def data_flatten(self):
@@ -250,7 +248,7 @@ class Experiment(object):
                 _logger.warning(
                     'No flat field defined for background, getting it '
                     'directly from data files')
-                return numpy.median(numpy.nan_to_num(self.__data))
+                return numpy.median(numpy.nan_to_num(self.data_flatten))
             else:
                 return None
         else:
@@ -327,7 +325,6 @@ class Experiment(object):
         self.__dims.clear()
         _fail = False, None  # register fail status and latest reason
         for axis, dim in dims.items():
-            print(" ##### axis %s" % axis)
             if dim.size is None:
                 assert isinstance(dim, _Dim)
                 # try to deduce dimension size
@@ -338,7 +335,6 @@ class Experiment(object):
                                                                relative_prev_val=dim.relative_prev_val,
                                                                cycle_length=dim.size,
                                                                axis=axis)
-                    print('unique_dim_value: %s' % unique_dim_value)
                 except Exception as e:
                     if _fail[0] is False:
                         _fail = True, e

id06workflow/core/experiment/operation/__init__.py

@@ -98,7 +98,7 @@ class OverwritingOperation(_BaseOperation):
         _BaseOperation.__init__(self, experiment, name, can_overwrite_data=True)
 
     @_BaseOperation.data.setter
-    def data_flatten(self, data):
+    def data(self, data):
         self._experiment.data = data
 
     def dry_run(self, cache_data=None):

id06workflow/core/experiment/operation/noisereduction.py

@@ -142,6 +142,7 @@ class BackgroundSubtraction(OverwritingOperation):
 
     def compute(self):
         if self._compute(self.data_flatten):
+            assert self.data_flatten.ndim is 3
             self.registerOperation()
         return self.data_flatten
 
@@ -169,6 +170,8 @@ class BackgroundSubtraction(OverwritingOperation):
         else:
             background = self._experiment.getMedianBackground()
             if isinstance(background, numpy.ndarray):
+                assert data.ndim is 3
+                assert background.ndim is 2
                 data[:] = data[:] - background
             else:
                 data = data - background

id06workflow/core/experiment/operation/roi.py

@@ -49,21 +49,22 @@ class RoiOperation(OverwritingOperation):
                          str(self._size)))
 
     def compute(self):
-        self.data_flatten = self._compute(self.data_flatten)
+        self.data = self._compute(self.data)
         self.registerOperation()
         return self.data
 
     def dry_run(self, cache_data=None):
         if cache_data is None:
-            self._cache_data = self.data[...]
+            self._cache_data = self.data_flatten[...]
         else:
+            assert cache_data.ndim is 3
             self._cache_data = cache_data
         return self._compute(self._cache_data)
 
     def apply(self):
         if self._cache_data is None:
             raise ValueError('No data in cache')
-        self.data_flatten = self._cache_data
+        self.data = self._cache_data
         self.clear_cache()
         self.registerOperation()
         return self.data
@@ -71,8 +72,12 @@ class RoiOperation(OverwritingOperation):
     def _compute(self, data):
         ymin, ymax = int(self._origin[1]), int(self._origin[1] + self._size[1])
         xmin, xmax = int(self._origin[0]), int(self._origin[0] + self._size[0])
-
-        data = data[:, ymin:ymax, xmin:xmax]
+        if data.ndim is 3:
+            data = data[:, ymin:ymax, xmin:xmax]
+        elif data.ndim is 4:
+            data = data[:, :, ymin:ymax, xmin:xmax]
+        else:
+            raise ValueError('data dimension is not managed (%s)' % data.ndim)
         return data
 
     def clear_cache(self):

id06workflow/core/experiment/operation/shift.py

@@ -55,7 +55,7 @@ class Shift(OverwritingOperation):
         OverwritingOperation.__init__(self, experiment, name='shift')
         if dz != 0:
             raise NotImplementedError('z shift not taken into account yet')
-        assert self.data.ndim is 3
+        assert self.data_flatten.ndim is 3
         self.dx = dx
         self.dy = dy
         self.dz = dz
@@ -86,7 +86,7 @@ class Shift(OverwritingOperation):
         self._dz = dz
 
     def compute(self):
-        self.data_flatten = self._compute(self.data)
+        self.data_flatten = self._compute(self.data_flatten)
         self.registerOperation()
         return self.data
 
@@ -112,7 +112,7 @@ class Shift(OverwritingOperation):
     def _compute(self, data):
         res = []
         nImg = data.shape[0]
-        for iImg, img in enumerate(self.data[:]):
+        for iImg, img in enumerate(self.data_flatten[:]):
             self.updateProgress(int(iImg / nImg * 100.0))
             _dx = self.dx * iImg
             _dy = self.dy * iImg

id06workflow/core/test/__init__.py

@@ -33,6 +33,7 @@ from . import test_geometry
 from . import test_image
 from . import test_mask
 from . import test_operation
+from . import test_stream_operations
 
 
 def suite():
@@ -43,5 +44,6 @@ def suite():
         test_image.suite(),
         test_mask.suite(),
         test_operation.suite(),
+        test_stream_operations.suite(),
     ])
     return test_suite

id06workflow/core/test/test_experiment.py

@@ -85,9 +85,9 @@ class TestExperiement(unittest.TestCase):
         back_sub_data = experiment.apply_background_subtraction()
         self.assertTrue(numpy.array_equal(back_sub_data[3], numpy.zeros(dims)))
 
+    @unittest.skipUnless(os.path.exists('/nobackup/linazimov/payno/dev/esrf/ID06/dataset/for_mapping'))
     def testMapping(self):
         # TODO: store datasets on the web and retrieve them using utils.getDataset()
-        # from fabio or silx or tmower
         _dir = '/nobackup/linazimov/payno/dev/esrf/ID06/dataset/for_mapping'
         files = []
         [files.append(os.path.join(_dir, _file)) for _file in os.listdir(_dir)]

id06workflow/core/test/test_stream_operations.py

@@ -35,7 +35,7 @@ from id06workflow.core.experiment import Experiment, _Dim
 import os
 
 
-
+@unittest.skipIf(os.path.exists('/nobackup/linazimov/payno/datasets/id06/strain_scan') is False, reason='data files not available')
 class TestOperationStream(unittest.TestCase):
     """Test some default operation streaming"""
     def setUp(self):

id06workflow/gui/dimensions.py

@@ -98,7 +98,6 @@ class DimensionMapping(qt.QWidget):
             widget.setDim(dim)
         widget.axis = axis
         self._dims[row] = widget
-        print('add a widget to the dim')
         return widget
 
     def setDims(self, dims):
@@ -164,7 +163,6 @@ class DimensionWidget(DimensionMapping):
 
     def __init__(self, parent):
         DimensionMapping.__init__(self, parent)
-        print('Create Diemsnion widget')
         self.__experiment = None
         self._fitButton = qt.QPushButton('fit', parent=self)
         self.layout().addWidget(self._fitButton, 6, 0, 1, 1)
@@ -183,7 +181,6 @@ class DimensionWidget(DimensionMapping):
                            dimensions.
         :type experiment: :class:`Experiment`
         """
-        print('set experiment')
         assert isinstance(experiment, Experiment)
         self.__experiment = experiment
         if len(self.experiment.metadata) > 0:
@@ -212,6 +209,17 @@ class DimensionWidget(DimensionMapping):
             self.fitSucceed.emit()
             return True, None
 
+    def addDim(self, axis=None, dim=None):
+        """
+
+        :param axis: which axis is defining this dimension
+        :param :class:_Dim dim: definition of the dimension to add
+        """
+        widget = super().addDim(axis, dim)
+        if self.__experiment is not None and len(self.__experiment.metadata) > 0:
+            widget._setMetadata(self.__experiment.metadata[0])
+        return widget
+
 
 class _DimensionItem(_Dim, qt.QWidget):
     """Widget use to define a dimension"""
@@ -337,16 +345,15 @@ class _DimensionItem(_Dim, qt.QWidget):
 
     def setDim(self, dim):
         assert isinstance(dim, _Dim)
-        print('should set %s' % dim.name)
+        _kind = _METADATA_TYPES_I[dim.kind]
+        idx = self._kindCB.findText(_kind)
+        assert idx >= 0
+        self._kindCB.setCurrentIndex(idx)
         idx = self._namesCB.findText(dim.name)
         if idx >= 0:
             self._namesCB.setCurrentIndex(idx)
         else:
             self._namesCB.addItem(dim.name)
-        _kind = _METADATA_TYPES_I[dim.kind]
-        idx = self._kindCB.findText(_kind)
-        assert idx >= 0
-        self._namesCB.setCurrentIndex(idx)
         if dim.size is not None:
             self._sizeWidget.setValue(dim.size)
         self._relativeCB.setChecked(dim.relative_prev_val)
@@ -366,7 +373,6 @@ class _DimensionItem(_Dim, qt.QWidget):
 
     @property
     def kind(self):
-        print(self._kindCB.currentText())
         assert self._kindCB.currentText() in _METADATA_TYPES
         return _METADATA_TYPES[self._kindCB.currentText()]
 
@@ -391,8 +397,6 @@ class _DimensionItem(_Dim, qt.QWidget):
         return self._namesCB.currentText()
 
     def _setSize(self, size):
-        print('Should update size to %s' % size)
-        print(self.relative)
         _Dim._setSize(self, size)
         self._sizeWidget.setValue(size=size, editable=self.relative)
 

id06workflow/gui/shifcorrection.py

@@ -123,7 +123,7 @@ class ShiftCorrectionWidget(qt.QWidget):
 
         self._rawData = self._operation.data
         self._lastShift = (0, 0, 0)
-        self._stack.setStack(self._operation.data)
+        self._stack.setStack(self._operation.data_flatten)
         self._sumPlot.addImage(self._operation.data.sum(axis=0))
         self._updateShift()
 

orangecontrib/id06workflow/test/test_example.py

-import unittest
-
-
-class ExampleTests(unittest.TestCase):
-    def test_addition(self):
-        self.assertEqual(1 + 1, 2)

orangecontrib/id06workflow/widgets/datareduction.py

-# coding: utf-8
-# /*##########################################################################
-#
-# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the "Software"), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in
-# all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-# THE SOFTWARE.
-#
-# ###########################################################################*/
-
-__authors__ = ["H. Payno"]
-__license__ = "MIT"
-__date__ = "02/10/2018"
-
-
-from Orange.widgets import gui
-from Orange.widgets.widget import OWWidget
-from Orange.canvas.registry.description import OutputSignal, InputSignal
-from id06workflow.core.experiment import Experiment
-from id06workflow.core.experiment.operation.datareduction import DataReduction
-from silx.gui import qt
-import logging
-_logger = logging.getLogger(__file__)
-
-
-class DataReductionOW(OWWidget):
-    """
-    TODO: define the graph and the possible acquisition types.
-    """
-    name = "data reduction"
-    id = "orange.widgets.id06workflow.datareduction"
-    description = "Select data to define the dataset to treat"
-    icon = "icons/data_reduction.svg"
-    priority = 100
-    category = "esrfWidgets"
-    keywords = ["dataset", "data", "reduction"]
-
-    inputs = [InputSignal(name="data", handler='_process', type=Experiment)]
-    outputs = [OutputSignal(name="data", type=Experiment)]
-
-    want_main_area = True
-    resizing_enabled = True
-    compress_signal = False
-
-
-    def __init__(self):
-        # TODO: add drag and drop stuff on it
-        super().__init__()
-        layout = gui.vBox(self.mainArea, 'data reduction').layout()
-        self._widget = qt.QWidget(parent=self)
-        self._widget.setLayout(qt.QGridLayout())
-        self._widget.layout().addWidget(qt.QLabel('x reduction:'), 0, 0)
-        self._xreduc = qt.QLineEdit('2', parent=self)
-        self._xreduc.setValidator(qt.QIntValidator(parent=self))
-        self._widget.layout().addWidget(self._xreduc, 0, 1)
-
-        self._widget.layout().addWidget(qt.QLabel('y reduction:'), 1, 0)
-        self._yreduc = qt.QLineEdit('2', parent=self)
-        self._yreduc.setValidator(qt.QIntValidator(parent=self))
-        self._widget.layout().addWidget(self._yreduc, 1, 1)
-
-        self._widget.layout().addWidget(qt.QLabel('z reduction:'), 2, 0)
-        self._zreduc = qt.QLineEdit('10', parent=self)
-        self._zreduc.setValidator(qt.QIntValidator(parent=self))
-        self._widget.layout().addWidget(self._zreduc, 2, 1)
-        layout.addWidget(self._widget)
-
-    def getZReduction(self):
-        return int(self._zreduc.text())
-
-    def getYReduction(self):
-        return int(self._yreduc.text())
-
-    def getXReduction(self):
-        return int(self._xreduc.text())
-
-    def _process(self, experiment):
-        if experiment is None:
-            return
-        operation = DataReduction(experiment=experiment,
-                                  x_factor=self.getXReduction(),
-                                  y_factor=self.getYReduction(),
-                                  z_factor=self.getZReduction())
-        operation.compute()
-        self.send("data", experiment)

orangecontrib/id06workflow/widgets/dimensions.py

@@ -95,7 +95,6 @@ class DimensionOW(OWWidget):
 
     def _process(self, experiment):
         if experiment is not None:
-            print('**** set experiment ***')
             self._widget.setExperiment(experiment)
             self.show()
 
@@ -106,3 +105,4 @@ class DimensionOW(OWWidget):
                                    'Fail to define coherent diemnsion values. Reason is %s' % msg)
         else:
             self.send("data", self._widget.experiment)
+            OWWidget.accept(self)

orangecontrib/id06workflow/widgets/roiselection.py

@@ -110,5 +110,6 @@ class RoiSelectionOW(OWWidget):
             self.send("data", self._editedExperiment)
         except Exception as e:
             _logger.error(e)
-        OWWidget.accept(self)
+        else:
+            OWWidget.accept(self)
         # TODO: what can be the image to send to show roi selection (z sum ?)

orangecontrib/id06workflow/widgets/shiftcorrection.py

@@ -65,6 +65,7 @@ class ShiftCorrectionOW(OWWidget):
 
     def __init__(self):
         super().__init__()
+        self._editedExperiment = None
         self._widget = ShiftCorrectionWidget(parent=self)
         self._progress = gui.ProgressBar(self, 100)
         self._widget.sigProgress.connect(self._setProgressValue)
@@ -119,4 +120,4 @@ class ShiftCorrectionOW(OWWidget):
             self.send("image",
                       image)
             # TODO: clear widget to avoid reprocessing ?
-        OWWidget.accept(self)
+            OWWidget.accept(self)

test_real_dataset.py

@@ -42,6 +42,7 @@ logging.disable(logging.INFO)
 app = qt.QApplication.instance() or qt.QApplication([])
 
 
+@unittest.skipIf(os.path.exists('/users/payno/datasets/id06/strain_scan'))
 class TestTrueData(OrangeWorflowTest):
     """
     Create dummy workflow just to make sure all the widget can interact together
@@ -56,8 +57,6 @@ class TestTrueData(OrangeWorflowTest):
                 'orangecontrib.id06workflow.widgets.geometry.GeometryOW')
         dimDefNode = self.addWidget(
                 'orangecontrib.id06workflow.widgets.dimensions.DimensionOW')
-        dataReductionNode = self.addWidget(
-                'orangecontrib.id06workflow.widgets.datareduction.DataReductionOW')
         roiSelectionNode = self.addWidget(
                 'orangecontrib.id06workflow.widgets.roiselection.RoiSelectionOW')
         noiseReductionNode = self.addWidget(
@@ -74,8 +73,7 @@ class TestTrueData(OrangeWorflowTest):
 
         self.link(dataSelectionNode, "data", geometryNode, "data")
         self.link(geometryNode, "data", dimDefNode, "data")
-        self.link(dimDefNode, "data", dataReductionNode, "data")
-        self.link(dataReductionNode, "data", roiSelectionNode, "data")
+        self.link(dimDefNode, "data", roiSelectionNode, "data")
         self.link(roiSelectionNode, "data", noiseReductionNode, "data")
         self.link(noiseReductionNode, "data", shiftCorrectionNode, "data")
         self.link(shiftCorrectionNode, "data", mappingNode, "data")
@@ -85,7 +83,6 @@ class TestTrueData(OrangeWorflowTest):
         self._selectionWidget = self.getWidgetForNode(dataSelectionNode)
         self._geometryWidget = self.getWidgetForNode(geometryNode)
         self._dimDefWidget = self.getWidgetForNode(dimDefNode)
-        self._dataReductionWidget = self.getWidgetForNode(dataSelectionNode)
         self._roiSelectionWidget = self.getWidgetForNode(roiSelectionNode)
         self._shiftCorrectionWidget = self.getWidgetForNode(shiftCorrectionNode)
         self._noiseReductionWidget = self.getWidgetForNode(noiseReductionNode)
@@ -140,42 +137,35 @@ class TestTrueData(OrangeWorflowTest):
 
         # define dimension definition
         self._dimDefWidget._experiment.metadata[0].get_keys(POSITIONER_METADATA)
-        while(self._dimDefWidget._experiment is None and len(self._dimDefWidget._experiment.metadata) < 0):
-            self.qWait(100)
         self._dimDefWidget.setDims(self._dims)
-        app.exec_()
         self._moveToNextStep()
         self.assertTrue(self._dimDefWidget._widget.dims[0].kind == POSITIONER_METADATA)
         self.assertTrue(self._dimDefWidget._widget.dims[0].name == 'diffry')
         self.assertTrue(self._dimDefWidget._widget.dims[0]._sizeWidget._active)
         self.assertFalse(self._dimDefWidget._widget.dims[1]._sizeWidget._active)
-        self.assertTrue(self._dimDefWidget._ndim is 2)
+        self.assertTrue(self._dimDefWidget._widget.dims[0].size is 31)
+        self.assertTrue(self._dimDefWidget._widget.dims[1].size is None)
         self._dimDefWidget._widget.fit()
         self.assertTrue(self._dimDefWidget._widget.dims[0].size is 31)
-        self.assertFalse(self._dimDefWidget._widget.dims[1].size is 2)
-        print(self._dimDefWidget._experiment.data.shape)
+        self.assertTrue(self._dimDefWidget._widget.dims[1].size is 2)
         self.assertTrue(self._dimDefWidget._experiment.data.shape == (31, 2, 2048, 2048))
-        self._selectionWidget.validate()
+        self._dimDefWidget.validate()
         self._moveToNextStep()
 
         # define noise reduction
         self._noiseReductionWidget.setBackgroundSubtraction(True)
         self._noiseReductionWidget.setLowerThreshold(5.0)
         self._noiseReductionWidget.setUpperThreshold(5000)
+        self._noiseReductionWidget.validate()
+        self._moveToNextStep()
 
         # manage roi selection
-        self._moveToNextStep()
-        # self._moveToNextStep()
-        # self._moveToNextStep()
         self._roiSelectionWidget.setROI(origin=(750, 750), size=(600, 600))
         self._roiSelectionWidget.validate()
         self._moveToNextStep()
-        self._roiSelectionWidget.hide()
-        self._moveToNextStep()
 
         # manage noise reduction
         self._noiseReductionWidget.validate()
-        self._shiftCorrectionWidget.hide()
         self._moveToNextStep()
 
         # manage shift correction