Commit aa10a212 authored by payno's avatar payno

[core/operation/mapping] rework mapping

- _MappingBase.dim -> _MappingBase.dims
- rework IntensityMap and gradient removal, the value of the delta has still to be given mannually (but should come from the range of the unique values)
- add testFullSteam to check the full treatment without GUI
- rework DimensionGUI to display if asked the unique values for each dimensions once fit has been done
parent 4994fca4
Pipeline #6651 passed with stage
in 2 minutes and 34 seconds
......@@ -135,8 +135,9 @@ class Dataset(object):
Return a fabio :FileSeries: to iterate over frame.
"""
if self.__data_has_changed is True:
assert os.path.exists(self.data_files_pattern)
# TODO: warning, for now only deal with single frame file
if not os.path.exists(self.data_files_pattern):
raise ValueError('Given file path does not exists (%s)' % self.data_files_pattern)
filenames = filename_series(self.data_files_pattern)
self.__data_series = FileSeries(filenames=filenames,
single_frame=True)
......
......@@ -45,7 +45,7 @@ class _MappingBase(AdditiveOperation):
Base class used for mapping
"""
@property
def dim(self):
def dims(self):
raise NotImplementedError('Base class')
@property
......@@ -102,7 +102,7 @@ class IntensityMapping(_MappingBase):
for y in range(self.data.shape[-2]):
# TODO: do this operation using a grid or something close
# but would have to find a way to write at the same position...
reciprocalVol = numpy.squeeze(self.data[:, y, x])
reciprocalVol = numpy.squeeze(self.data_flatten[:, y, x]).astype(numpy.float128)
reciprocalVol = reciprocalVol - reciprocalVol.min()
maxIntensity = reciprocalVol.max()
if maxIntensity > self._threshold:
......@@ -112,7 +112,7 @@ class IntensityMapping(_MappingBase):
for axis, dim in self._experiment.dims:
intensity = numpy.squeeze(reciprocalVol.sum())
# TODO: move calculation of angle_mean, angle_variance,
dim_unique_values = self.expriment.get_unique_values(axis)
dim_unique_values = dim.unique_values.astype(numpy.float128)
# TODO: why mean is obtained by the sum ?
# angles_mean = (intensity * angles).mean()
_mean = (intensity * dim_unique_values).sum()
......@@ -120,16 +120,16 @@ class IntensityMapping(_MappingBase):
_variance = (intensity * diff ** 2).sum()
_skewness = (intensity * diff ** 3).sum()
_kurtosis = (intensity * diff ** 4).sum()
self.dim[axis].mean[y, x] = _mean
self.dim[axis].variance[y, x] = _variance
self.dim[axis].skewness[y, x] = _skewness
self.dim[axis].kurtosis[y, x] = _kurtosis
self.dims[axis].mean[y, x] = _mean
self.dims[axis].variance[y, x] = _variance
self.dims[axis].skewness[y, x] = _skewness
self.dims[axis].kurtosis[y, x] = _kurtosis
self.registerOperation()
return self.dim
return self.dims
def _resetIntensityMap(self):
self.__intensity_map = numpy.zeros(self.data.shape[1:])
self.__intensity_map = numpy.zeros(self.data.shape[-2:], dtype=numpy.float128)
def _resetDim(self):
self.__dim = {}
......@@ -148,7 +148,7 @@ class IntensityMapping(_MappingBase):
return self.__intensity_map
@property
def dim(self):
def dims(self):
return self.__dim
@property
......@@ -164,8 +164,7 @@ class GradientRemoval(_MappingBase):
def __init__(self, experiment):
_MappingBase.__init__(self, experiment=experiment,
name='gradient removal')
self._gradients = None
self.__dim = []
self.__dim = {}
self.__intensity_map = None
def dry_run(self, cache_data=None):
......@@ -182,8 +181,10 @@ class GradientRemoval(_MappingBase):
'needs to be proceed before.')
return
self.__dim = self.apply_gradient_correction(self.data, delta=delta, mapping=mapping)
self.__intensity_map = mapping.intensity_map[...]
self.__dim = self.apply_gradient_correction(self.data_flatten,
delta=delta,
mapping=mapping)
self.__intensity_map = mapping.intensity_map
self.registerOperation()
@staticmethod
......@@ -193,34 +194,25 @@ class GradientRemoval(_MappingBase):
corr2, corr1 = numpy.meshgrid(
numpy.linspace(-delta, delta, data.shape[2]),
numpy.linspace(0, 0, data.shape[1]))
_gradients = []
# TODO: if necessary compute mean...
_gradients = {}
for iDim in range(mapping.ndim):
gradient = mapping.dims[iDim][...]
for axis, dim in mapping.dims.items():
_mean = dim.mean + corr2
_variance = dim.variance + corr2
_skewness = dim.skewness + corr2
_kurtosis = dim.kurtosis + corr2
gradient = _IMap(mean=_mean, variance=_variance, skewness=_skewness,
kurtosis=_kurtosis, name=dim.name, kind=dim.kind)
# TODO: for now gradient corrcetion is only apply on mean, why ?
# TODO: this should be in enumerate(Mode) but intensity is stored
# in an other struct, why ?
_gradient = numpy.asarray((
gradient[:, :, 0] + corr2,
gradient[:, :, 1] + corr2,
gradient[:, :, 2] + corr2,
gradient[:, :, 3] + corr2)
)
_gradients.append(_gradient)
_gradients[axis] = gradient
return _gradients
def gradients(self):
return self._gradients
def key(self):
return self._name
@property
def dim(self):
def dims(self):
return self.__dim
@property
......
......@@ -93,7 +93,9 @@ class ThresholdRemoval(OverwritingOperation):
if None given, will set the value of the
threshold
"""
if self._compute(self.data_flatten):
_res = self._compute(self.data_flatten)
if _res is not None:
self.data_flatten = _res
self.registerOperation()
return self.data_flatten
......@@ -141,8 +143,10 @@ class BackgroundSubtraction(OverwritingOperation):
return self._name
def compute(self):
if self._compute(self.data_flatten):
assert self.data_flatten.ndim is 3
_data_sub = self._compute(self.data_flatten)
if _data_sub is not None:
assert _data_sub.ndim is 3
self.data_flatten = _data_sub
self.registerOperation()
return self.data_flatten
......
......@@ -56,9 +56,9 @@ class TestMappingOperation(unittest.TestCase):
# define geometry
geometry = TwoThetaGeometry(
twotheta=0.03, # for now those are defined in degree but should
twotheta=0.03, # for now those are defined in degree but should
# be defined in radians
xmag=1.0, # what is the unit of this ?
xmag=1.0, # what is the unit of this ?
xpixelsize=1.0,
ypixelsize=1.0,
orientation=TwoThetaGeometry.VERTICAL)
......
......@@ -30,7 +30,10 @@ __date__ = "21/11/2018"
import unittest
from id06workflow.core.experiment import Experiment, Dataset, POSITIONER_METADATA
from id06workflow.core.geometry.TwoThetaGeometry import TwoThetaGeometry
from id06workflow.core.utils import metadata as metadatautils
from id06workflow.core.experiment.operation.mapping import IntensityMapping, GradientRemoval
from id06workflow.core.experiment.operation import noisereduction
from id06workflow.core.experiment import Experiment, _Dim
import os
......@@ -50,7 +53,17 @@ class TestOperationStream(unittest.TestCase):
data_bb_files.append(os.path.join(bb_folder, _file))
self.dataset = Dataset(data_files_pattern=data_file_pattern,
ff_files=data_bb_files)
self.experiment = Experiment(dataset=self.dataset, geometry=None)
# define geometry
geometry = TwoThetaGeometry(
twotheta=0.03, # for now those are defined in degree but should
# be defined in radians
xmag=1.0, # what is the unit of this ?
xpixelsize=1.0,
ypixelsize=1.0,
orientation=TwoThetaGeometry.VERTICAL)
self.experiment = Experiment(dataset=self.dataset, geometry=geometry)
def applyDimsDef(self):
"""Apply experimentation dimension definition"""
......@@ -81,6 +94,30 @@ class TestOperationStream(unittest.TestCase):
self.assertTrue(self.experiment.dims.shape == (31, 2))
self.assertTrue(self.experiment.data.shape == (31, 2, 2048, 2048))
def testFullStream(self):
"""Apply the entire set of operation on the """
self.applyDimsDef()
ope_mask_sub = noisereduction.MaskSubstraction(self.experiment)
ope_mask_sub.compute()
ope_background_sub = noisereduction.BackgroundSubtraction(self.experiment)
ope_background_sub.compute()
ope_low_threshold = noisereduction.ThresholdRemoval(self.experiment,
lower=True, threshold=0.2)
ope_low_threshold.compute()
ope_high_threshold = noisereduction.ThresholdRemoval(self.experiment,
lower=False, threshold=10000.0)
ope_high_threshold.compute()
ope_intensity_map = IntensityMapping(experiment=self.experiment)
ope_intensity_map.compute()
ope_gradient_removal = GradientRemoval(self.experiment)
ope_gradient_removal.compute()
ope_intensity_map_2 = IntensityMapping(experiment=self.experiment)
ope_intensity_map_2.compute()
# TODO: should be call at one moment if the 'project file' is not storing
# each results
# self.experiment.save('myProjectFile.h5')
def suite():
test_suite = unittest.TestSuite()
......
......@@ -41,24 +41,25 @@ class DimensionMapping(qt.QWidget):
Widget used to define the number of dimension and with which values they are
mapped
"""
_V_HEADERS = ['Axis', 'Kind', 'Name', 'Size', 'Is relative', '']
_V_HEADERS = ['Axis', 'Kind', 'Name', 'Size', 'Is relative', '', '']
def __init__(self, parent):
qt.QWidget.__init__(self, parent)
self.setLayout(qt.QGridLayout())
self._table = qt.QTableWidget(parent=self)
self._table.setColumnCount(6)
self._table.setColumnCount(len(self._V_HEADERS))
self._table.setHorizontalHeaderLabels(self._V_HEADERS)
header = self._table.horizontalHeader()
if qt.qVersion() < "5.0":
setResizeMode = header.setResizeMode
else:
setResizeMode = header.setSectionResizeMode
for iColumn in (0, 3, 4, 5):
setResizeMode(iColumn, qt.QHeaderView.ResizeToContents)
for iColumn in (1, 2):
setResizeMode(iColumn, qt.QHeaderView.Stretch)
for iColumn in range(len(self._V_HEADERS)):
if iColumn in (1, 2):
setResizeMode(iColumn, qt.QHeaderView.Stretch)
else:
setResizeMode(iColumn, qt.QHeaderView.ResizeToContents)
self._table.verticalHeader().hide()
self._dims = {}
......@@ -315,6 +316,9 @@ class _DimensionItem(_Dim, qt.QWidget):
style = qt.QApplication.style()
icon = style.standardIcon(qt.QStyle.SP_BrowserStop)
self._rmButton = qt.QPushButton(icon=icon, parent=self)
icon = style.standardIcon(qt.QStyle.SP_FileDialogContentsView)
self._infoButton = qt.QPushButton(icon=icon, parent=self)
self._infoButton.hide()
# connect Signal/slot
self._rmButton.pressed.connect(self.remove)
......@@ -325,6 +329,8 @@ class _DimensionItem(_Dim, qt.QWidget):
self._namesCB.currentIndexChanged.connect(self._dimHasChanged)
self._relativeCB.toggled.connect(self._dimHasChanged)
self._kindCB.currentIndexChanged.connect(self._updateNames)
self._infoButton.pressed.connect(self.showUniqueNames)
# update values from _Dim
self._kindCB.currentTextChanged.connect(self._setKind)
self._namesCB.currentTextChanged.connect(self._setName)
......@@ -343,6 +349,19 @@ class _DimensionItem(_Dim, qt.QWidget):
def remove(self):
self.removed.emit(self)
def showUniqueNames(self):
title = "%s - %s unique names" % (_METADATA_TYPES_I[self.kind], self.name)
unique_values_str = []
[unique_values_str.append(str(val)) for val in self.unique_values]
qt.QMessageBox.information(self, title, ', '.join(unique_values_str))
def _setUniqueValues(self, values):
super()._setUniqueValues(values)
if values is None:
self._infoButton.hide()
else:
self._infoButton.show()
def setDim(self, dim):
assert isinstance(dim, _Dim)
_kind = _METADATA_TYPES_I[dim.kind]
......@@ -402,23 +421,21 @@ class _DimensionItem(_Dim, qt.QWidget):
def setNames(self, names):
self._namesCB.clear()
for name in names:
for name in sorted(list(names)):
self._namesCB.addItem(name)
def embedInTable(self, table, row):
self.__row = row
for column, widget in enumerate((self._axis, self._kindCB, self._namesCB,
self._sizeWidget, self._relativeCB, self._rmButton)):
self._sizeWidget, self._relativeCB,
self._infoButton, self._rmButton)):
table.setCellWidget(row, column, widget)
def _updateNames(self, *args, **kwargs):
"""Update names for the current kind"""
if self.__metadata is not None:
_lastActiveName = self.name
self._namesCB.clear()
for key in self.__metadata.get_keys(self.kind):
self._namesCB.addItem(key)
self.setNames(self.__metadata.get_keys(self.kind))
idx = self._namesCB.findText(_lastActiveName)
if idx >= 0:
self._namesCB.setCurrentIndex(idx)
......
......@@ -124,6 +124,8 @@ class MappingPlot(qt.QMainWindow):
return
mode = self._getMode()
dim = self._getDim()
if dim == '':
return
assert dim in self._operation.dims
if mode == INTENSITY:
......
......@@ -134,11 +134,11 @@ class _DummyMap(_MappingBase):
for iDim in range(ndim):
self.__dim.append(numpy.zeros((*(shape), 4)))
for imap in range(4):
self.dim[iDim][:, :, imap] = numpy.random.random(shape)
self.dims[iDim][:, :, imap] = numpy.random.random(shape)
self.__intensity_map = numpy.random.random(shape)
@property
def dim(self):
def dims(self):
return self.__dim
@property
......
......@@ -110,7 +110,7 @@ class TestTrueData(OrangeWorflowTest):
if self._previous_workflow_output is not None:
os.environ['WORKFLOW_OUTPUT'] = self._previous_workflow_output
# shutil.rmtree(self._screenshot_output)
shutil.rmtree(self._screenshot_output)
OrangeWorflowTest.tearDown(self)
def createDataset(self):
......@@ -192,8 +192,12 @@ class TestTrueData(OrangeWorflowTest):
# screenshot saving
# make sur all screenshots are processed
for i in range(9):
max_iter = 200
iter = 0
while iter < max_iter and os.path.exists(os.path.join(self._screenshot_output, 'intensity.png')) is False:
self._moveToNextStep()
iter = iter + 1
self.qWait(200)
self.assertTrue(os.path.exists(os.path.join(self._screenshot_output, 'intensity.png')))
self.assertTrue(os.path.exists(os.path.join(self._screenshot_output, 'positioner_diffry_mean.png')))
self.assertTrue(os.path.exists(os.path.join(self._screenshot_output, 'positioner_diffry_variance.png')))
......
......@@ -95,8 +95,14 @@ class DimensionOW(OWWidget):
def _process(self, experiment):
if experiment is not None:
self._widget.setExperiment(experiment)
self.show()
try:
self._widget.setExperiment(experiment)
except ValueError as e:
qt.QMessageBox.warning(self,
'Fail to setup dimension definition',
str(e))
else:
self.show()
def validate(self):
succeed, msg = self._widget.fit()
......
......@@ -31,6 +31,8 @@ __date__ = "16/10/2018"
from Orange.widgets import gui
from Orange.widgets.widget import OWWidget
from Orange.canvas.registry.description import InputSignal, OutputSignal
from id06workflow.core.experiment import _METADATA_TYPES_I
from id06workflow.core.mapping import MEAN, VARIANCE, SKEWNESS, KURTOSIS
from id06workflow.gui.mapping import MappingPlot
from id06workflow.core.experiment import Experiment
from id06workflow.core.experiment.operation.mapping import GradientRemoval, _MappingBase
......@@ -72,15 +74,16 @@ class GradientRemovalOW(OWWidget):
if experiment is None:
return
# TODO: as for com calculation the input should be a _Mapping base to
# assert isinstance(mapping, _MappingBase)
operation = GradientRemoval(experiment=experiment)
operation.compute()
self.send("image",
_Image(img=operation.dim[0][:, :, 0], name='mean'))
for iDim, dim in operation.dims.items():
name = dim.name
kind = _METADATA_TYPES_I[dim.kind]
for mode in (MEAN, VARIANCE, SKEWNESS, KURTOSIS):
img = getattr(dim, mode)
img_name = '_'.join((kind, name, mode, 'gradient_removal'))
self.send("image", _Image(img=img, name=img_name))
self._plot.setOperation(operation)
self.send("data", experiment)
self.send("map", operation)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment