Commit 9c9af232 authored by payno's avatar payno

[operation] Adapt several operation to multiple dimension

- adapt shift (wok with data_flatten but should probably be extended latter)
- adapt noise reduction (work with data_flatten)
- remove the data reduction widget because should be highly modify since the numpy.ndarra.view in the experiement.data appear. Should be re-done
parent 00338127
......@@ -178,16 +178,14 @@ class Experiment(object):
@property
def data(self):
if self.__data is None:
self.__data = self.getRawData()
if self.__dims.ndim > 1:
# TODO: create view or adapt view on raw data
shape = list(self.__dims.shape)
shape.append(self.__data.shape[-2])
shape.append(self.__data.shape[-1])
return self.__data.view().reshape(shape)
shape.append(self.data_flatten.shape[-2])
shape.append(self.data_flatten.shape[-1])
return self.data_flatten.view().reshape(shape)
else:
return self.__data
return self.data_flatten
@property
def data_flatten(self):
......@@ -250,7 +248,7 @@ class Experiment(object):
_logger.warning(
'No flat field defined for background, getting it '
'directly from data files')
return numpy.median(numpy.nan_to_num(self.__data))
return numpy.median(numpy.nan_to_num(self.data_flatten))
else:
return None
else:
......@@ -327,7 +325,6 @@ class Experiment(object):
self.__dims.clear()
_fail = False, None # register fail status and latest reason
for axis, dim in dims.items():
print(" ##### axis %s" % axis)
if dim.size is None:
assert isinstance(dim, _Dim)
# try to deduce dimension size
......@@ -338,7 +335,6 @@ class Experiment(object):
relative_prev_val=dim.relative_prev_val,
cycle_length=dim.size,
axis=axis)
print('unique_dim_value: %s' % unique_dim_value)
except Exception as e:
if _fail[0] is False:
_fail = True, e
......
......@@ -98,7 +98,7 @@ class OverwritingOperation(_BaseOperation):
_BaseOperation.__init__(self, experiment, name, can_overwrite_data=True)
@_BaseOperation.data.setter
def data_flatten(self, data):
def data(self, data):
self._experiment.data = data
def dry_run(self, cache_data=None):
......
......@@ -142,6 +142,7 @@ class BackgroundSubtraction(OverwritingOperation):
def compute(self):
if self._compute(self.data_flatten):
assert self.data_flatten.ndim is 3
self.registerOperation()
return self.data_flatten
......@@ -169,6 +170,8 @@ class BackgroundSubtraction(OverwritingOperation):
else:
background = self._experiment.getMedianBackground()
if isinstance(background, numpy.ndarray):
assert data.ndim is 3
assert background.ndim is 2
data[:] = data[:] - background
else:
data = data - background
......
......@@ -49,21 +49,22 @@ class RoiOperation(OverwritingOperation):
str(self._size)))
def compute(self):
self.data_flatten = self._compute(self.data_flatten)
self.data = self._compute(self.data)
self.registerOperation()
return self.data
def dry_run(self, cache_data=None):
if cache_data is None:
self._cache_data = self.data[...]
self._cache_data = self.data_flatten[...]
else:
assert cache_data.ndim is 3
self._cache_data = cache_data
return self._compute(self._cache_data)
def apply(self):
if self._cache_data is None:
raise ValueError('No data in cache')
self.data_flatten = self._cache_data
self.data = self._cache_data
self.clear_cache()
self.registerOperation()
return self.data
......@@ -71,8 +72,12 @@ class RoiOperation(OverwritingOperation):
def _compute(self, data):
ymin, ymax = int(self._origin[1]), int(self._origin[1] + self._size[1])
xmin, xmax = int(self._origin[0]), int(self._origin[0] + self._size[0])
if data.ndim is 3:
data = data[:, ymin:ymax, xmin:xmax]
elif data.ndim is 4:
data = data[:, :, ymin:ymax, xmin:xmax]
else:
raise ValueError('data dimension is not managed (%s)' % data.ndim)
return data
def clear_cache(self):
......
......@@ -55,7 +55,7 @@ class Shift(OverwritingOperation):
OverwritingOperation.__init__(self, experiment, name='shift')
if dz != 0:
raise NotImplementedError('z shift not taken into account yet')
assert self.data.ndim is 3
assert self.data_flatten.ndim is 3
self.dx = dx
self.dy = dy
self.dz = dz
......@@ -86,7 +86,7 @@ class Shift(OverwritingOperation):
self._dz = dz
def compute(self):
self.data_flatten = self._compute(self.data)
self.data_flatten = self._compute(self.data_flatten)
self.registerOperation()
return self.data
......@@ -112,7 +112,7 @@ class Shift(OverwritingOperation):
def _compute(self, data):
res = []
nImg = data.shape[0]
for iImg, img in enumerate(self.data[:]):
for iImg, img in enumerate(self.data_flatten[:]):
self.updateProgress(int(iImg / nImg * 100.0))
_dx = self.dx * iImg
_dy = self.dy * iImg
......
......@@ -33,6 +33,7 @@ from . import test_geometry
from . import test_image
from . import test_mask
from . import test_operation
from . import test_stream_operations
def suite():
......@@ -43,5 +44,6 @@ def suite():
test_image.suite(),
test_mask.suite(),
test_operation.suite(),
test_stream_operations.suite(),
])
return test_suite
......@@ -85,9 +85,9 @@ class TestExperiement(unittest.TestCase):
back_sub_data = experiment.apply_background_subtraction()
self.assertTrue(numpy.array_equal(back_sub_data[3], numpy.zeros(dims)))
@unittest.skipUnless(os.path.exists('/nobackup/linazimov/payno/dev/esrf/ID06/dataset/for_mapping'))
def testMapping(self):
# TODO: store datasets on the web and retrieve them using utils.getDataset()
# from fabio or silx or tmower
_dir = '/nobackup/linazimov/payno/dev/esrf/ID06/dataset/for_mapping'
files = []
[files.append(os.path.join(_dir, _file)) for _file in os.listdir(_dir)]
......
......@@ -35,7 +35,7 @@ from id06workflow.core.experiment import Experiment, _Dim
import os
@unittest.skipIf(os.path.exists('/nobackup/linazimov/payno/datasets/id06/strain_scan') is False, reason='data files not available')
class TestOperationStream(unittest.TestCase):
"""Test some default operation streaming"""
def setUp(self):
......
......@@ -98,7 +98,6 @@ class DimensionMapping(qt.QWidget):
widget.setDim(dim)
widget.axis = axis
self._dims[row] = widget
print('add a widget to the dim')
return widget
def setDims(self, dims):
......@@ -164,7 +163,6 @@ class DimensionWidget(DimensionMapping):
def __init__(self, parent):
DimensionMapping.__init__(self, parent)
print('Create Diemsnion widget')
self.__experiment = None
self._fitButton = qt.QPushButton('fit', parent=self)
self.layout().addWidget(self._fitButton, 6, 0, 1, 1)
......@@ -183,7 +181,6 @@ class DimensionWidget(DimensionMapping):
dimensions.
:type experiment: :class:`Experiment`
"""
print('set experiment')
assert isinstance(experiment, Experiment)
self.__experiment = experiment
if len(self.experiment.metadata) > 0:
......@@ -212,6 +209,17 @@ class DimensionWidget(DimensionMapping):
self.fitSucceed.emit()
return True, None
def addDim(self, axis=None, dim=None):
"""
:param axis: which axis is defining this dimension
:param :class:_Dim dim: definition of the dimension to add
"""
widget = super().addDim(axis, dim)
if self.__experiment is not None and len(self.__experiment.metadata) > 0:
widget._setMetadata(self.__experiment.metadata[0])
return widget
class _DimensionItem(_Dim, qt.QWidget):
"""Widget use to define a dimension"""
......@@ -337,16 +345,15 @@ class _DimensionItem(_Dim, qt.QWidget):
def setDim(self, dim):
assert isinstance(dim, _Dim)
print('should set %s' % dim.name)
_kind = _METADATA_TYPES_I[dim.kind]
idx = self._kindCB.findText(_kind)
assert idx >= 0
self._kindCB.setCurrentIndex(idx)
idx = self._namesCB.findText(dim.name)
if idx >= 0:
self._namesCB.setCurrentIndex(idx)
else:
self._namesCB.addItem(dim.name)
_kind = _METADATA_TYPES_I[dim.kind]
idx = self._kindCB.findText(_kind)
assert idx >= 0
self._namesCB.setCurrentIndex(idx)
if dim.size is not None:
self._sizeWidget.setValue(dim.size)
self._relativeCB.setChecked(dim.relative_prev_val)
......@@ -366,7 +373,6 @@ class _DimensionItem(_Dim, qt.QWidget):
@property
def kind(self):
print(self._kindCB.currentText())
assert self._kindCB.currentText() in _METADATA_TYPES
return _METADATA_TYPES[self._kindCB.currentText()]
......@@ -391,8 +397,6 @@ class _DimensionItem(_Dim, qt.QWidget):
return self._namesCB.currentText()
def _setSize(self, size):
print('Should update size to %s' % size)
print(self.relative)
_Dim._setSize(self, size)
self._sizeWidget.setValue(size=size, editable=self.relative)
......
......@@ -123,7 +123,7 @@ class ShiftCorrectionWidget(qt.QWidget):
self._rawData = self._operation.data
self._lastShift = (0, 0, 0)
self._stack.setStack(self._operation.data)
self._stack.setStack(self._operation.data_flatten)
self._sumPlot.addImage(self._operation.data.sum(axis=0))
self._updateShift()
......
import unittest
class ExampleTests(unittest.TestCase):
def test_addition(self):
self.assertEqual(1 + 1, 2)
# coding: utf-8
# /*##########################################################################
#
# Copyright (c) 2016-2017 European Synchrotron Radiation Facility
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ###########################################################################*/
__authors__ = ["H. Payno"]
__license__ = "MIT"
__date__ = "02/10/2018"
from Orange.widgets import gui
from Orange.widgets.widget import OWWidget
from Orange.canvas.registry.description import OutputSignal, InputSignal
from id06workflow.core.experiment import Experiment
from id06workflow.core.experiment.operation.datareduction import DataReduction
from silx.gui import qt
import logging
_logger = logging.getLogger(__file__)
class DataReductionOW(OWWidget):
"""
TODO: define the graph and the possible acquisition types.
"""
name = "data reduction"
id = "orange.widgets.id06workflow.datareduction"
description = "Select data to define the dataset to treat"
icon = "icons/data_reduction.svg"
priority = 100
category = "esrfWidgets"
keywords = ["dataset", "data", "reduction"]
inputs = [InputSignal(name="data", handler='_process', type=Experiment)]
outputs = [OutputSignal(name="data", type=Experiment)]
want_main_area = True
resizing_enabled = True
compress_signal = False
def __init__(self):
# TODO: add drag and drop stuff on it
super().__init__()
layout = gui.vBox(self.mainArea, 'data reduction').layout()
self._widget = qt.QWidget(parent=self)
self._widget.setLayout(qt.QGridLayout())
self._widget.layout().addWidget(qt.QLabel('x reduction:'), 0, 0)
self._xreduc = qt.QLineEdit('2', parent=self)
self._xreduc.setValidator(qt.QIntValidator(parent=self))
self._widget.layout().addWidget(self._xreduc, 0, 1)
self._widget.layout().addWidget(qt.QLabel('y reduction:'), 1, 0)
self._yreduc = qt.QLineEdit('2', parent=self)
self._yreduc.setValidator(qt.QIntValidator(parent=self))
self._widget.layout().addWidget(self._yreduc, 1, 1)
self._widget.layout().addWidget(qt.QLabel('z reduction:'), 2, 0)
self._zreduc = qt.QLineEdit('10', parent=self)
self._zreduc.setValidator(qt.QIntValidator(parent=self))
self._widget.layout().addWidget(self._zreduc, 2, 1)
layout.addWidget(self._widget)
def getZReduction(self):
return int(self._zreduc.text())
def getYReduction(self):
return int(self._yreduc.text())
def getXReduction(self):
return int(self._xreduc.text())
def _process(self, experiment):
if experiment is None:
return
operation = DataReduction(experiment=experiment,
x_factor=self.getXReduction(),
y_factor=self.getYReduction(),
z_factor=self.getZReduction())
operation.compute()
self.send("data", experiment)
......@@ -95,7 +95,6 @@ class DimensionOW(OWWidget):
def _process(self, experiment):
if experiment is not None:
print('**** set experiment ***')
self._widget.setExperiment(experiment)
self.show()
......@@ -106,3 +105,4 @@ class DimensionOW(OWWidget):
'Fail to define coherent diemnsion values. Reason is %s' % msg)
else:
self.send("data", self._widget.experiment)
OWWidget.accept(self)
......@@ -110,5 +110,6 @@ class RoiSelectionOW(OWWidget):
self.send("data", self._editedExperiment)
except Exception as e:
_logger.error(e)
else:
OWWidget.accept(self)
# TODO: what can be the image to send to show roi selection (z sum ?)
......@@ -65,6 +65,7 @@ class ShiftCorrectionOW(OWWidget):
def __init__(self):
super().__init__()
self._editedExperiment = None
self._widget = ShiftCorrectionWidget(parent=self)
self._progress = gui.ProgressBar(self, 100)
self._widget.sigProgress.connect(self._setProgressValue)
......
......@@ -42,6 +42,7 @@ logging.disable(logging.INFO)
app = qt.QApplication.instance() or qt.QApplication([])
@unittest.skipIf(os.path.exists('/users/payno/datasets/id06/strain_scan'))
class TestTrueData(OrangeWorflowTest):
"""
Create dummy workflow just to make sure all the widget can interact together
......@@ -56,8 +57,6 @@ class TestTrueData(OrangeWorflowTest):
'orangecontrib.id06workflow.widgets.geometry.GeometryOW')
dimDefNode = self.addWidget(
'orangecontrib.id06workflow.widgets.dimensions.DimensionOW')
dataReductionNode = self.addWidget(
'orangecontrib.id06workflow.widgets.datareduction.DataReductionOW')
roiSelectionNode = self.addWidget(
'orangecontrib.id06workflow.widgets.roiselection.RoiSelectionOW')
noiseReductionNode = self.addWidget(
......@@ -74,8 +73,7 @@ class TestTrueData(OrangeWorflowTest):
self.link(dataSelectionNode, "data", geometryNode, "data")
self.link(geometryNode, "data", dimDefNode, "data")
self.link(dimDefNode, "data", dataReductionNode, "data")
self.link(dataReductionNode, "data", roiSelectionNode, "data")
self.link(dimDefNode, "data", roiSelectionNode, "data")
self.link(roiSelectionNode, "data", noiseReductionNode, "data")
self.link(noiseReductionNode, "data", shiftCorrectionNode, "data")
self.link(shiftCorrectionNode, "data", mappingNode, "data")
......@@ -85,7 +83,6 @@ class TestTrueData(OrangeWorflowTest):
self._selectionWidget = self.getWidgetForNode(dataSelectionNode)
self._geometryWidget = self.getWidgetForNode(geometryNode)
self._dimDefWidget = self.getWidgetForNode(dimDefNode)
self._dataReductionWidget = self.getWidgetForNode(dataSelectionNode)
self._roiSelectionWidget = self.getWidgetForNode(roiSelectionNode)
self._shiftCorrectionWidget = self.getWidgetForNode(shiftCorrectionNode)
self._noiseReductionWidget = self.getWidgetForNode(noiseReductionNode)
......@@ -140,42 +137,35 @@ class TestTrueData(OrangeWorflowTest):
# define dimension definition
self._dimDefWidget._experiment.metadata[0].get_keys(POSITIONER_METADATA)
while(self._dimDefWidget._experiment is None and len(self._dimDefWidget._experiment.metadata) < 0):
self.qWait(100)
self._dimDefWidget.setDims(self._dims)
app.exec_()
self._moveToNextStep()
self.assertTrue(self._dimDefWidget._widget.dims[0].kind == POSITIONER_METADATA)
self.assertTrue(self._dimDefWidget._widget.dims[0].name == 'diffry')
self.assertTrue(self._dimDefWidget._widget.dims[0]._sizeWidget._active)
self.assertFalse(self._dimDefWidget._widget.dims[1]._sizeWidget._active)
self.assertTrue(self._dimDefWidget._ndim is 2)
self.assertTrue(self._dimDefWidget._widget.dims[0].size is 31)
self.assertTrue(self._dimDefWidget._widget.dims[1].size is None)
self._dimDefWidget._widget.fit()
self.assertTrue(self._dimDefWidget._widget.dims[0].size is 31)
self.assertFalse(self._dimDefWidget._widget.dims[1].size is 2)
print(self._dimDefWidget._experiment.data.shape)
self.assertTrue(self._dimDefWidget._widget.dims[1].size is 2)
self.assertTrue(self._dimDefWidget._experiment.data.shape == (31, 2, 2048, 2048))
self._selectionWidget.validate()
self._dimDefWidget.validate()
self._moveToNextStep()
# define noise reduction
self._noiseReductionWidget.setBackgroundSubtraction(True)
self._noiseReductionWidget.setLowerThreshold(5.0)
self._noiseReductionWidget.setUpperThreshold(5000)
self._noiseReductionWidget.validate()
self._moveToNextStep()
# manage roi selection
self._moveToNextStep()
# self._moveToNextStep()
# self._moveToNextStep()
self._roiSelectionWidget.setROI(origin=(750, 750), size=(600, 600))
self._roiSelectionWidget.validate()
self._moveToNextStep()
self._roiSelectionWidget.hide()
self._moveToNextStep()
# manage noise reduction
self._noiseReductionWidget.validate()
self._shiftCorrectionWidget.hide()
self._moveToNextStep()
# manage shift correction
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment