Merge branch 'small_fixes' into 'master'

Small fixes + jupyter notebook  tutorial on Experiment and operations

See merge request !10

.gitignore

@@ -43,4 +43,7 @@ dataset/
 datasets/
 *.egg-info*
 *.log
-*.log.*
\ No newline at end of file
+*.log.*
+
+# notebook checkpoints
+*.ipynb_checkpoints/

id06workflow/core/Dataset/__init__.py

@@ -104,10 +104,6 @@ class Dataset(object):
         self._ff_files.append(flat_field_file)
         self.__ff_has_changed = True
 
-    def applyShiftCorrection(self, correction):
-        # TODO: move shift corrcetion code in here
-        raise NotImplementedError('')
-
     def is_valid(self):
         return len(self._data_files_pattern) > 0 or self._data is not None
 

id06workflow/core/experiment/__init__.py

@@ -356,7 +356,7 @@ class AcquisitionDims(object):
         self.__dims = {}
 
     def add_dim(self, axis, dim):
-        assert isinstance(dim, _Dim)
+        assert isinstance(dim, Dim)
         self.__dims[axis] = dim
 
     def clear(self):
@@ -400,16 +400,16 @@ class AcquisitionDims(object):
             _logger.error('axis %s is not defined yet, cannot defined a size '
                           'for it' % axis)
         else:
-            self.__dims[axis] = _Dim(name=self.__dims[axis].name,
-                                     kind=self.__dims[axis].kind,
-                                     size=size)
+            self.__dims[axis] = Dim(name=self.__dims[axis].name,
+                                    kind=self.__dims[axis].kind,
+                                    size=size)
 
     def __iter__(self):
         for iAxis, dim in self.__dims.items():
             yield (iAxis, dim)
 
 
-class _Dim(object):
+class Dim(object):
     def __init__(self, kind, name, size=None, relative_prev_val=False):
         """
         Define a dimension used during the experiment
@@ -467,7 +467,7 @@ class _Dim(object):
     def _setSize(self, size):
         """
         .. note: having a setter was needed for GUI and Signal?SLOT stuff
-                (see :class:`_DimensionItem`) 
+                (see :class:`DimensionItem`)
         """
         self._size = size
 

id06workflow/core/operation/com.py

@@ -30,7 +30,7 @@ __date__ = "15/10/2018"
 
 
 from . import AdditiveOperation
-from .mapping import _MappingBase
+from .mapping import _MappingBase, _IMap
 from collections import namedtuple
 import numpy
 import logging
@@ -53,22 +53,23 @@ class COM(AdditiveOperation):
         AdditiveOperation.__init__(self, experiment=map.experiment,
                                    name=COM._NAME)
         self._map = map
-        self.__maps = []
+        self.__maps = {}
 
     def key(self):
         return COM._NAME
 
     def compute(self):
-        assert self._map.ndim is 1
-        self.__maps = []
-
-        for dim in self._map.dims:
-            _map1 = dim[:, :, 0]
-            _map1 = _map1 - numpy.nan_to_num(_map1.mean())
-
-            _map2 = dim[:, :, 1]
-            _map2 = _map2 - numpy.nan_to_num(_map2.mean())
-            self.__maps.append(_MapsCom(map1=_map1, map2=_map2))
+        self.__maps = {}
+        for axis, _map in self._map.dims.items():
+            assert _map.mean.ndim is 2
+            _mean = _map.mean - numpy.nan_to_num(_map.mean.mean())
+            _variance = _map.variance - numpy.nan_to_num(_map.variance.mean())
+            _skewness = _map.skewness - numpy.nan_to_num(_map.skewness.mean())
+            _kurtosis = _map.kurtosis - numpy.nan_to_num(_map.kurtosis.mean())
+            self.__maps[axis] = _IMap(name=_map.name, kind=_map.kind,
+                                      mean=_mean, variance=_variance,
+                                      skewness=_skewness, kurtosis=_kurtosis)
+        self.registerOperation()
 
     @property
     def maps(self):

id06workflow/core/operation/mapping.py

@@ -46,6 +46,11 @@ class _MappingBase(AdditiveOperation):
     """
     @property
     def dims(self):
+        """
+
+        :return:
+        :rtype: dict with axis index as key, _IMap as value
+        """
         raise NotImplementedError('Base class')
 
     @property
@@ -98,7 +103,7 @@ class IntensityMapping(_MappingBase):
             self._resetIntensityMap()
             self._resetDim()
             for x in range(self.data.shape[-1]):
-                self.updateProgress(int(x / self.data.shape[2] * 100.0))
+                self.updateProgress(int(x / self.data.shape[-2] * 100.0))
                 for y in range(self.data.shape[-2]):
                     # TODO: do this operation using a grid or something close
                     # but would have to find a way to write at the same position...
@@ -149,6 +154,7 @@ class IntensityMapping(_MappingBase):
 
     @property
     def dims(self):
+        assert type(self.__dim) is dict
         return self.__dim
 
     @property
@@ -185,6 +191,7 @@ class GradientRemoval(_MappingBase):
 
     def apply_gradient_correction(self, data, mapping):
         assert isinstance(mapping, IntensityMapping)
+        _gradients = {}
 
         for axis, dim in mapping.dims.items():
             unique_values = numpy.array(self._experiment.dims.get(axis).unique_values)
@@ -193,7 +200,6 @@ class GradientRemoval(_MappingBase):
             corr2, corr1 = numpy.meshgrid(
                     numpy.linspace(-_delta, _delta, data.shape[2]),
                     numpy.linspace(0, 0, data.shape[1]))
-            _gradients = {}
 
             _mean = dim.mean + corr2
             _variance = dim.variance + corr2
@@ -211,6 +217,7 @@ class GradientRemoval(_MappingBase):
 
     @property
     def dims(self):
+        assert type(self.__dim) is dict
         return self.__dim
 
     @property

id06workflow/core/test/test_experiment.py

@@ -29,7 +29,7 @@ __date__ = "09/10/2018"
 
 
 import unittest
-from id06workflow.core.experiment import AcquisitionDims, _Dim, DEFAULT_METADATA
+from id06workflow.core.experiment import AcquisitionDims, Dim, DEFAULT_METADATA
 
 
 class TestDataset(unittest.TestCase):
@@ -45,15 +45,15 @@ class TestAcquisitionDims(unittest.TestCase):
         """global test of the AcquisitionDims APi"""
         acquiDim = AcquisitionDims()
         self.assertTrue(acquiDim.shape == (1, ))
-        acquiDim.add_dim(axis=0, dim=_Dim(kind=DEFAULT_METADATA, name='0', size=10))
-        acquiDim.add_dim(axis=1, dim=_Dim(kind=DEFAULT_METADATA, name='1'))
+        acquiDim.add_dim(axis=0, dim=Dim(kind=DEFAULT_METADATA, name='0', size=10))
+        acquiDim.add_dim(axis=1, dim=Dim(kind=DEFAULT_METADATA, name='1'))
         self.assertTrue(acquiDim.shape == (10, -1))
         acquiDim.set_size(axis=1, size=2)
         self.assertTrue(acquiDim.shape == (10, 2))
         acquiDim.clear()
         self.assertTrue(acquiDim.shape == (1, ))
-        acquiDim.add_dim(axis=1, dim=_Dim(kind=DEFAULT_METADATA, name='0', size=10))
-        acquiDim.add_dim(axis=4, dim=_Dim(kind=DEFAULT_METADATA, name='1'))
+        acquiDim.add_dim(axis=1, dim=Dim(kind=DEFAULT_METADATA, name='0', size=10))
+        acquiDim.add_dim(axis=4, dim=Dim(kind=DEFAULT_METADATA, name='1'))
         self.assertTrue(acquiDim.shape == (1, 10, 1, 1, -1))
 
 

id06workflow/core/test/test_image.py

@@ -64,11 +64,11 @@ class TestShift(unittest.TestCase):
             [ff_files.append(os.path.join(dir_ff, _file)) for _file in
              os.listdir(dir_ff)]
 
-            dim1 = experiment._Dim(kind=experiment.POSITIONER_METADATA,
-                                   name='diffry', relative_prev_val=True,
-                                   size=31)
-            dim2 = experiment._Dim(kind=experiment.POSITIONER_METADATA,
-                                   name='obpitch')
+            dim1 = experiment.Dim(kind=experiment.POSITIONER_METADATA,
+                                  name='diffry', relative_prev_val=True,
+                                  size=31)
+            dim2 = experiment.Dim(kind=experiment.POSITIONER_METADATA,
+                                  name='obpitch')
             self._dims = {0: dim1, 1: dim2}
 
             return experiment.Dataset(data_files_pattern=data_file_pattern,

id06workflow/core/test/test_mapping.py

@@ -32,7 +32,7 @@ import os
 import unittest
 
 from id06workflow.core.experiment import Experiment, Dataset, \
-    POSITIONER_METADATA, _Dim
+    POSITIONER_METADATA, Dim
 from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
 from id06workflow.core.operation.mapping import IntensityMapping
 
@@ -69,9 +69,9 @@ class TestMappingOperation(unittest.TestCase):
         self.experiment = Experiment(dataset=self.dataset, geometry=geometry)
 
         # define dimensions
-        dim1 = _Dim(kind=POSITIONER_METADATA, name='diffry',
-                    relative_prev_val=True, size=31)
-        dim2 = _Dim(kind=POSITIONER_METADATA, name='obpitch')
+        dim1 = Dim(kind=POSITIONER_METADATA, name='diffry',
+                   relative_prev_val=True, size=31)
+        dim2 = Dim(kind=POSITIONER_METADATA, name='obpitch')
         self.experiment.set_dims(dims={0: dim1, 1: dim2})
 
     def testIntensityMapping(self):

id06workflow/core/test/test_stream_operations.py

@@ -30,12 +30,11 @@ __date__ = "21/11/2018"
 
 import os
 import unittest
-
 from id06workflow.core.operation.mapping import IntensityMapping, \
     GradientRemoval
-
+from id06workflow.core.operation.com import COM
 from id06workflow.core.experiment import Dataset, POSITIONER_METADATA
-from id06workflow.core.experiment import Experiment, _Dim
+from id06workflow.core.experiment import Experiment, Dim
 from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
 from id06workflow.core.operation import noisereduction
 from id06workflow.core.utils import metadata as metadatautils
@@ -70,9 +69,9 @@ class TestOperationStream(unittest.TestCase):
 
     def applyDimsDef(self):
         """Apply experimentation dimension definition"""
-        dim1 = _Dim(kind=POSITIONER_METADATA, name='diffry',
-                    relative_prev_val=True, size=31)
-        dim2 = _Dim(kind=POSITIONER_METADATA, name='obpitch')
+        dim1 = Dim(kind=POSITIONER_METADATA, name='diffry',
+                   relative_prev_val=True, size=31)
+        dim2 = Dim(kind=POSITIONER_METADATA, name='obpitch')
         self.experiment.set_dims(dims={0: dim1, 1: dim2})
 
     def testDimensionManual(self):
@@ -114,9 +113,9 @@ class TestOperationStream(unittest.TestCase):
         ope_intensity_map.compute()
         ope_gradient_removal = GradientRemoval(self.experiment)
         ope_gradient_removal.compute()
-        ope_intensity_map_2 = IntensityMapping(experiment=self.experiment)
-        ope_intensity_map_2.compute()
 
+        com_operation = COM(map=ope_gradient_removal)
+        com_operation.compute()
         # TODO: should be call at one moment if the 'project file' is not storing
         # each results
         # self.experiment.save('myProjectFile.h5')

id06workflow/gui/com.py

@@ -29,11 +29,11 @@ __date__ = "04/10/2018"
 
 
 import logging
-from collections import OrderedDict
 from silx.gui import qt
 from silx.gui.plot import Plot2D
-
+from id06workflow.core.mapping import MEAN, VARIANCE, SKEWNESS, KURTOSIS
 from id06workflow.core.operation.com import COM as COMOperation
+from id06workflow.gui.settings import DEFAULT_COLORMAP
 
 _logger = logging.getLogger(__file__)
 
@@ -48,6 +48,7 @@ class ComWidget(qt.QWidget):
         self._control = _ComControlWidget(parent=self)
         self.layout().addWidget(self._control)
         self._plot = Plot2D(parent=self)
+        self._plot.setDefaultColormap(DEFAULT_COLORMAP)
         self.layout().addWidget(self._plot)
 
         # connect signal
@@ -64,30 +65,20 @@ class ComWidget(qt.QWidget):
     def _update(self):
         self._plot.clear()
         dim = self._control.getDim()
-        _iMap = self._control.getMapType()
-        if dim >= 0:
-            # TODO: maps[dims] should be also a list with key values: 'value',
-            # 'mean', 'gradients' ...
-            if len(self._operation.maps[dim]) <= _iMap:
-                _logger.error('map of index %s has not been created' % _iMap)
-                self._plot.clear()
-                return
-            else:
-                _map = self._operation.maps[dim][_iMap]
-                self._plot.addImage(_map)
+        map_type = self._control.getMapType()
 
+        if dim is None:
+            return
 
-class _ComControlWidget(qt.QWidget):
+        self._plot.clear()
+        try:
+            _map = getattr(self._operation.maps[dim], map_type)
+            self._plot.addImage(_map)
+        except Exception as e:
+            _logger.error(e)
 
-    _COM_VALUE = 'com value'
-    _COM_VARIANCE = 'com mean variance'
-    _COM_GRADIENTS = 'com gradients'
 
-    _MAP_TYPES = OrderedDict([
-        (_COM_VALUE, 0),
-        (_COM_VARIANCE, 1),
-        (_COM_GRADIENTS, 2)
-    ])
+class _ComControlWidget(qt.QWidget):
 
     sigSelectionChanged = qt.Signal()
     """Signal emitted when the selection of the com change"""
@@ -102,9 +93,9 @@ class _ComControlWidget(qt.QWidget):
         self._map = qt.QComboBox(parent=self)
         self.layout().addWidget(self._map)
         # TODO: are those name correct ?
-        for map_type in (_ComControlWidget._MAP_TYPES):
+        for map_type in (MEAN, VARIANCE, SKEWNESS, KURTOSIS):
             self._map.addItem(map_type)
-        index = self._map.findText(_ComControlWidget._COM_VALUE)
+        index = self._map.findText(MEAN)
         assert index >= 0
         self._map.setCurrentIndex(index)
 
@@ -124,14 +115,12 @@ class _ComControlWidget(qt.QWidget):
     def getDim(self):
         currentDim = self._dim.currentText()
         if currentDim == '':
-            return -1
+            return None
         else:
             return int(currentDim)
 
     def getMapType(self):
-        currentMapType = self._map.currentText()
-        assert currentMapType in _ComControlWidget._MAP_TYPES
-        return _ComControlWidget._MAP_TYPES[currentMapType]
+        return self._map.currentText()
 
     def __selectionChanged(self, *args, **kwargs):
         self.sigSelectionChanged.emit()

id06workflow/gui/dimensions.py

@@ -29,7 +29,7 @@ __date__ = "03/10/2018"
 
 
 from silx.gui import qt
-from id06workflow.core.experiment import _Dim, Experiment
+from id06workflow.core.experiment import Dim, Experiment
 from id06workflow.core.experiment import _METADATA_TYPES, _METADATA_TYPES_I, DEFAULT_METADATA
 import logging
 
@@ -87,7 +87,7 @@ class DimensionMapping(qt.QWidget):
         """
 
         :param axis: which axis is defining this dimension
-        :param :class:_Dim dim: definition of the dimension to add
+        :param :class:Dim dim: definition of the dimension to add
         """
         if axis is None:
             axis = self._getNextFreeAxis()
@@ -104,13 +104,13 @@ class DimensionMapping(qt.QWidget):
     def setDims(self, dims):
         """
 
-        :param dict dims: axis as key and :class:`_Dim` as value  
+        :param dict dims: axis as key and :class:`Dim` as value
         :return: 
         """
         self.clear()
         for axis, dim in dims.items():
             assert type(axis) is int
-            assert isinstance(dim, _Dim)
+            assert isinstance(dim, Dim)
             self.addDim(axis=axis, dim=dim)
 
     def removeDim(self, row):
@@ -214,7 +214,7 @@ class DimensionWidget(DimensionMapping):
         """
 
         :param axis: which axis is defining this dimension
-        :param :class:_Dim dim: definition of the dimension to add
+        :param :class:`Dim` dim: definition of the dimension to add
         """
         widget = super().addDim(axis, dim)
         if self.__experiment is not None and len(self.__experiment.metadata) > 0:
@@ -222,7 +222,7 @@ class DimensionWidget(DimensionMapping):
         return widget
 
 
-class _DimensionItem(_Dim, qt.QWidget):
+class _DimensionItem(Dim, qt.QWidget):
     """Widget use to define a dimension"""
     removed = qt.Signal(qt.QObject)
     """Signal emitted when the Item should be removed"""
@@ -294,7 +294,7 @@ class _DimensionItem(_Dim, qt.QWidget):
         :param int row: row position in the QTableWidget. Also used as ID
         """
         qt.QWidget.__init__(self, parent)
-        _Dim.__init__(self, kind=DEFAULT_METADATA, name='')
+        Dim.__init__(self, kind=DEFAULT_METADATA, name='')
 
         self.__metadata = None
 
@@ -331,7 +331,7 @@ class _DimensionItem(_Dim, qt.QWidget):
         self._kindCB.currentIndexChanged.connect(self._updateNames)
         self._infoButton.pressed.connect(self.showUniqueNames)
 
-        # update values from _Dim
+        # update values from `Dim` class
         self._kindCB.currentTextChanged.connect(self._setKind)
         self._namesCB.currentTextChanged.connect(self._setName)
         self._sizeWidget.valueChanged.connect(self._setSize)
@@ -363,7 +363,7 @@ class _DimensionItem(_Dim, qt.QWidget):
             self._infoButton.show()
 
     def setDim(self, dim):
-        assert isinstance(dim, _Dim)
+        assert isinstance(dim, Dim)
         _kind = _METADATA_TYPES_I[dim.kind]
         idx = self._kindCB.findText(_kind)
         assert idx >= 0
@@ -397,8 +397,8 @@ class _DimensionItem(_Dim, qt.QWidget):
 
     @property
     def dim(self):
-        return _Dim(name=self.name, kind=self.kind, size=self.dimsize,
-                    relative_prev_val=self.relative)
+        return Dim(name=self.name, kind=self.kind, size=self.dimsize,
+                   relative_prev_val=self.relative)
 
     @property
     def dimsize(self):
@@ -416,7 +416,7 @@ class _DimensionItem(_Dim, qt.QWidget):
         return self._namesCB.currentText()
 
     def _setSize(self, size):
-        _Dim._setSize(self, size)
+        Dim._setSize(self, size)
         self._sizeWidget.setValue(size=size, editable=self.relative)
 
     def setNames(self, names):

id06workflow/gui/test/test_mapping_gui.py

@@ -33,7 +33,7 @@ import unittest
 from silx.gui import qt
 
 from id06workflow.core.experiment import Experiment, Dataset, \
-    POSITIONER_METADATA, _Dim
+    POSITIONER_METADATA, Dim
 from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
 from id06workflow.core.operation.mapping import IntensityMapping
 from id06workflow.gui.mapping import MappingPlot
@@ -73,9 +73,9 @@ class TestMappingOperation(unittest.TestCase):
         self.experiment = Experiment(dataset=self.dataset, geometry=geometry)
 
         # define dimensions
-        dim1 = _Dim(kind=POSITIONER_METADATA, name='diffry',
-                    relative_prev_val=True, size=31)
-        dim2 = _Dim(kind=POSITIONER_METADATA, name='obpitch')
+        dim1 = Dim(kind=POSITIONER_METADATA, name='diffry',
+                   relative_prev_val=True, size=31)
+        dim2 = Dim(kind=POSITIONER_METADATA, name='obpitch')
         self.experiment.set_dims(dims={0: dim1, 1: dim2})
 
         self._widget = MappingPlot(parent=None)

orangecontrib/id06workflow/test/test_real_dataset.py

@@ -31,7 +31,7 @@ import unittest
 from silx.gui import qt
 from orangecontrib.id06workflow.test.OrangeWorkflowTest import OrangeWorflowTest
 from id06workflow.core.experiment import Dataset, POSITIONER_METADATA
-from id06workflow.core.experiment import _Dim
+from id06workflow.core.experiment import Dim
 from id06workflow.unitsystem import metricsystem
 from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
 import os
@@ -124,9 +124,9 @@ class TestTrueData(OrangeWorflowTest):
         self.dataset = Dataset(data_files_pattern=data_file_pattern,
                                ff_files=data_bb_files)
 
-        dim1 = _Dim(kind=POSITIONER_METADATA, name='diffry',
-                    relative_prev_val=True, size=31)
-        dim2 = _Dim(kind=POSITIONER_METADATA, name='obpitch')
+        dim1 = Dim(kind=POSITIONER_METADATA, name='diffry',
+                   relative_prev_val=True, size=31)
+        dim2 = Dim(kind=POSITIONER_METADATA, name='obpitch')
         self._dims = {0: dim1, 1: dim2}
 
         return self.dataset